From 3abdc44e7b623107c97593077f228a861fd69a56 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 2 Jul 2015 18:16:02 +0000
Subject: [PATCH 01/40] Add fts5_source_id() SQL function, similar to
 sqlite_source_id().

FossilOrigin-Name: ec2573487cb86664f7f056168a764c28ca8236fc
---
 ext/fts5/fts5_main.c            | 17 +++++++++++++++++
 ext/fts5/test/fts5unicode2.test |  2 +-
 ext/fts5/tool/mkfts5c.tcl       | 22 ++++++++++++++++++++++
 main.mk                         | 10 ++--------
 manifest                        | 20 ++++++++++----------
 manifest.uuid                   |  2 +-
 6 files changed, 53 insertions(+), 20 deletions(-)

diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c
index b52b2e9ec4..775c027756 100644
--- a/ext/fts5/fts5_main.c
+++ b/ext/fts5/fts5_main.c
@@ -2223,6 +2223,18 @@ static void fts5Fts5Func(
   sqlite3_result_blob(pCtx, buf, sizeof(pGlobal), SQLITE_TRANSIENT);
 }
 
+/*
+** Implementation of fts5_source_id() function.
+*/
+static void fts5SourceIdFunc(
+  sqlite3_context *pCtx,          /* Function call context */
+  int nArg,                       /* Number of args */
+  sqlite3_value **apVal           /* Function arguments */
+){
+  assert( nArg==0 );
+  sqlite3_result_text(pCtx, "--FTS5-SOURCE-ID--", -1, SQLITE_TRANSIENT);
+}
+
 #ifdef _WIN32
 __declspec(dllexport)
 #endif
@@ -2284,6 +2296,11 @@ int sqlite3_fts5_init(
           db, "fts5", 0, SQLITE_UTF8, p, fts5Fts5Func, 0, 0
       );
     }
+    if( rc==SQLITE_OK ){
+      rc = sqlite3_create_function(
+          db, "fts5_source_id", 0, SQLITE_UTF8, p, fts5SourceIdFunc, 0, 0
+      );
+    }
   }
   return rc;
 }
diff --git a/ext/fts5/test/fts5unicode2.test b/ext/fts5/test/fts5unicode2.test
index e34bc840a5..d3ff5128da 100644
--- a/ext/fts5/test/fts5unicode2.test
+++ b/ext/fts5/test/fts5unicode2.test
@@ -380,7 +380,7 @@ proc do_isspace_test {tn tokenizer lCp} {
 }
 
 set tokenizers [list unicode61]
-ifcapable icu { lappend tokenizers icu }
+#ifcapable icu { lappend tokenizers icu }
 
 # Some tests to check that the tokenizers can both identify white-space 
 # codepoints. All codepoints tested below are of type "Zs" in the
diff --git a/ext/fts5/tool/mkfts5c.tcl b/ext/fts5/tool/mkfts5c.tcl
index 34777e3915..e38101e210 100644
--- a/ext/fts5/tool/mkfts5c.tcl
+++ b/ext/fts5/tool/mkfts5c.tcl
@@ -40,6 +40,9 @@ set G(footer) {
 #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS5) */
 }
 
+#-------------------------------------------------------------------------
+# Read and return the entire contents of text file $zFile from disk.
+#
 proc readfile {zFile} {
   set fd [open $zFile]
   set data [read $fd]
@@ -47,6 +50,22 @@ proc readfile {zFile} {
   return $data
 }
 
+#-------------------------------------------------------------------------
+# This command returns a string identifying the current sqlite version -
+# the equivalent of the SQLITE_SOURCE_ID string.
+#
+proc fts5_source_id {zDir} {
+  set top [file dirname [file dirname $zDir]]
+  set uuid [string trim [readfile [file join $top manifest.uuid]]]
+
+  set L [split [readfile [file join $top manifest]]] 
+  set date [lindex $L [expr [lsearch -exact $L D]+1]]
+  set date [string range $date 0 [string last . $date]-1]
+  set date [string map {T { }} $date]
+
+  return "fts5: $date $uuid"
+}
+
 proc fts5c_init {zOut} {
   global G
   set G(fd) stdout
@@ -59,11 +78,14 @@ proc fts5c_printfile {zIn} {
   global G
   set data [readfile $zIn]
   puts $G(fd) "#line 1 \"[file tail $zIn]\""
+
+  set srcid_map [list --FTS5-SOURCE-ID-- [fts5_source_id $::srcdir]]
   foreach line [split $data "\n"] {
     if {[regexp {^#include.*fts5} $line]} continue
     if {[regexp {^(const )?[a-zA-Z][a-zA-Z0-9]* [*]?sqlite3Fts5} $line]} {
       set line "static $line"
     }
+    set line [string map $srcid_map $line]
     puts $G(fd) $line
   }
 }
diff --git a/main.mk b/main.mk
index 1f658440ab..2d087e6184 100644
--- a/main.mk
+++ b/main.mk
@@ -72,9 +72,6 @@ LIBOBJ+= vdbe.o parse.o \
 	 vdbetrace.o wal.o walker.o where.o wherecode.o whereexpr.o \
          utf.o vtab.o
 
-LIBOBJ += fts5.o
-
-
 
 # All of the source code files.
 #
@@ -307,7 +304,8 @@ TESTSRC += \
   $(TOP)/ext/misc/totype.c \
   $(TOP)/ext/misc/wholenumber.c \
   $(TOP)/ext/misc/vfslog.c \
-  $(TOP)/ext/fts5/fts5_tcl.c 
+  $(TOP)/ext/fts5/fts5_tcl.c \
+  fts5.c
 
 
 #TESTSRC += $(TOP)/ext/fts2/fts2_tokenizer.c
@@ -666,10 +664,6 @@ fts5parse.h: fts5parse.c
 fts5.c: $(FTS5_SRC)
 	tclsh $(TOP)/ext/fts5/tool/mkfts5c.tcl
 
-fts5.o:	fts5.c $(HDR) $(EXTHDR)
-	$(TCCX) -DSQLITE_CORE -c fts5.c
-
-
 
 userauth.o:	$(TOP)/ext/userauth/userauth.c $(HDR) $(EXTHDR)
 	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/userauth/userauth.c
diff --git a/manifest b/manifest
index dfc7d5e26a..2793a6a664 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sharmless\scompiler\swarnings\sand\sone\smacro\sVVA_ONLY\smacro\susage\sissue.
-D 2015-07-02T16:52:01.447
+C Add\sfts5_source_id()\sSQL\sfunction,\ssimilar\sto\ssqlite_source_id().
+D 2015-07-02T18:16:02.364
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 1f525f24e2d3a4defd0ce819c10980caeec967fe
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -113,7 +113,7 @@ F ext/fts5/fts5_config.c b2456e9625bca41c51d54c363e369c6356895c90
 F ext/fts5/fts5_expr.c d2e148345639c5a5583e0daa39a639bf298ae6a7
 F ext/fts5/fts5_hash.c 219f4edd72e5cf95b19c33f1058809a18fad5229
 F ext/fts5/fts5_index.c fb1f0de6b4cd02a212c0c9c5580daa64a5634035
-F ext/fts5/fts5_main.c c24ee96e7b97178d02e66e1fe1d43f6145aab8f8
+F ext/fts5/fts5_main.c 3390a455109b3d2f1341d45bda0556d23348a339
 F ext/fts5/fts5_storage.c 4cae85b5287b159d9d98174a4e70adf872b0930a
 F ext/fts5/fts5_tcl.c 85eb4e0d0fefa9420b78151496ad4599a1783e20
 F ext/fts5/fts5_tokenize.c 30f97a8c74683797b4cd233790444fbefb3b0708
@@ -172,13 +172,13 @@ F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17
 F ext/fts5/test/fts5rowid.test f7674e19a40987bf59624d8db9827114cb7f7a3e
 F ext/fts5/test/fts5tokenizer.test 83e7e01a21ec7fdf814d51f6184cc26bb77d7695
 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841
-F ext/fts5/test/fts5unicode2.test 84282d4a6dd34370dc19a3486dd6fecc89c7ed0b
+F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59
 F ext/fts5/test/fts5unicode3.test 35c3d02aa7acf7d43d8de3bfe32c15ba96e8928e
 F ext/fts5/test/fts5unindexed.test e9539d5b78c677315e7ed8ea911d4fd25437c680
 F ext/fts5/test/fts5version.test bed59038e937c40d3c0056d08076db7874c6cd4a
 F ext/fts5/test/fts5vocab.test cdf97b9678484e9bad5062edf9c9106e5c3b0c5c
 F ext/fts5/tool/loadfts5.tcl 7ef3e62131f0434a78e4f5c5b056b09d221710a8
-F ext/fts5/tool/mkfts5c.tcl 7174fce13c9d4b11c702eef3767344066cffe87e
+F ext/fts5/tool/mkfts5c.tcl a154b057664fabf5c9a0a202a99f2be7b96e545a
 F ext/fts5/tool/showfts5.tcl 921f33b30c3189deefd2b2cc81f951638544aaf1
 F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43
 F ext/icu/icu.c b2732aef0b076e4276d9b39b5a33cec7a05e1413
@@ -248,7 +248,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 8a1aded277e90ea7c10d0100d1a38f980e7d12a1
+F main.mk 439afc754d942e614e70fbde1d16e29d6dab18a9
 F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea
 F mkopcodeh.awk 0e7f04a8eb90f92259e47d80110e4e98d7ce337a
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
@@ -1364,7 +1364,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P f022e04343fc166a71b315d5aabbc5cc075a4767
-R 5c911bb6fc3e16cbfd6e76bce0768816
-U mistachkin
-Z 616700ef04b7deb7995d1138c3fd92ed
+P bcc8a75509aafda61feb6dcc074668c79611a662
+R 7e05f8767a887a2725fcaaa0e38ca6ef
+U dan
+Z a6f281b613f2ad2a3748391e96894e41
diff --git a/manifest.uuid b/manifest.uuid
index e4a6a04493..3e74c54eb3 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-bcc8a75509aafda61feb6dcc074668c79611a662
\ No newline at end of file
+ec2573487cb86664f7f056168a764c28ca8236fc
\ No newline at end of file

From 6544b8643fcd45917d135f691e6edd11b9dbd9d0 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 2 Jul 2015 18:52:16 +0000
Subject: [PATCH 02/40] Minor change to mkfts5c.tcl so that fts5.c can be used
 according to the instructions in loadext.html.

FossilOrigin-Name: edbcccd349090efff3f975e27451608136a429fc
---
 ext/fts5/tool/mkfts5c.tcl |  4 ++--
 manifest                  | 12 ++++++------
 manifest.uuid             |  2 +-
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/ext/fts5/tool/mkfts5c.tcl b/ext/fts5/tool/mkfts5c.tcl
index e38101e210..c21243fd13 100644
--- a/ext/fts5/tool/mkfts5c.tcl
+++ b/ext/fts5/tool/mkfts5c.tcl
@@ -24,7 +24,7 @@ set G(src) [string map [list %dir% $srcdir] {
 
 set G(hdr) {
 
-#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS5) 
+#if !defined(SQLITE_TEST) || defined(SQLITE_ENABLE_FTS5) 
 
 #if !defined(NDEBUG) && !defined(SQLITE_DEBUG) 
 # define NDEBUG 1
@@ -37,7 +37,7 @@ set G(hdr) {
 
 set G(footer) {
     
-#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS5) */
+#endif /* !defined(SQLITE_TEST) || defined(SQLITE_ENABLE_FTS5) */
 }
 
 #-------------------------------------------------------------------------
diff --git a/manifest b/manifest
index 2793a6a664..da9a81acb6 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sfts5_source_id()\sSQL\sfunction,\ssimilar\sto\ssqlite_source_id().
-D 2015-07-02T18:16:02.364
+C Minor\schange\sto\smkfts5c.tcl\sso\sthat\sfts5.c\scan\sbe\sused\saccording\sto\sthe\sinstructions\sin\sloadext.html.
+D 2015-07-02T18:52:16.632
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 1f525f24e2d3a4defd0ce819c10980caeec967fe
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -178,7 +178,7 @@ F ext/fts5/test/fts5unindexed.test e9539d5b78c677315e7ed8ea911d4fd25437c680
 F ext/fts5/test/fts5version.test bed59038e937c40d3c0056d08076db7874c6cd4a
 F ext/fts5/test/fts5vocab.test cdf97b9678484e9bad5062edf9c9106e5c3b0c5c
 F ext/fts5/tool/loadfts5.tcl 7ef3e62131f0434a78e4f5c5b056b09d221710a8
-F ext/fts5/tool/mkfts5c.tcl a154b057664fabf5c9a0a202a99f2be7b96e545a
+F ext/fts5/tool/mkfts5c.tcl f9ada2ab7a96a471c7e28a2e390cf4f098348cff
 F ext/fts5/tool/showfts5.tcl 921f33b30c3189deefd2b2cc81f951638544aaf1
 F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43
 F ext/icu/icu.c b2732aef0b076e4276d9b39b5a33cec7a05e1413
@@ -1364,7 +1364,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P bcc8a75509aafda61feb6dcc074668c79611a662
-R 7e05f8767a887a2725fcaaa0e38ca6ef
+P ec2573487cb86664f7f056168a764c28ca8236fc
+R 8f8fb58b016bb7e374a57938c502e4cf
 U dan
-Z a6f281b613f2ad2a3748391e96894e41
+Z f04dde703cf9595bec71c1e298c8382a
diff --git a/manifest.uuid b/manifest.uuid
index 3e74c54eb3..8110ba7c6b 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ec2573487cb86664f7f056168a764c28ca8236fc
\ No newline at end of file
+edbcccd349090efff3f975e27451608136a429fc
\ No newline at end of file

From d2dc87f6d52da7f018ff8228cb49552cc1cba1d6 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 2 Jul 2015 19:47:08 +0000
Subject: [PATCH 03/40] When doing coverage analysis of a btree page for pragma
 integrity_check, make the first entry (that covers the header, cell index,
 and gap) implied, for a performance boost and size reduction.

FossilOrigin-Name: e53d497c2d2fbc5014b39f5624b52ce207800698
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/btree.c   | 20 +++++++++++++-------
 3 files changed, 21 insertions(+), 15 deletions(-)

diff --git a/manifest b/manifest
index da9a81acb6..d4f2070257 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Minor\schange\sto\smkfts5c.tcl\sso\sthat\sfts5.c\scan\sbe\sused\saccording\sto\sthe\sinstructions\sin\sloadext.html.
-D 2015-07-02T18:52:16.632
+C When\sdoing\scoverage\sanalysis\sof\sa\sbtree\spage\sfor\spragma\sintegrity_check,\nmake\sthe\sfirst\sentry\s(that\scovers\sthe\sheader,\scell\sindex,\sand\sgap)\simplied,\nfor\sa\sperformance\sboost\sand\ssize\sreduction.
+D 2015-07-02T19:47:08.924
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 1f525f24e2d3a4defd0ce819c10980caeec967fe
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -269,7 +269,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240
 F src/backup.c 4d9134dc988a87838c06056c89c0e8c4700a0452
 F src/bitvec.c d1f21d7d91690747881f03940584f4cc548c9d3d
 F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79
-F src/btree.c cf3375bf706d15d3b8b7ac1fc86b0bf5603324d5
+F src/btree.c 7705ce3e17c2b0cb8d4c3d935b269270d3c2a548
 F src/btree.h 969adc948e89e449220ff0ff724c94bb2a52e9f1
 F src/btreeInt.h 2ad754dd4528baa8d0946a593cc373b890bf859e
 F src/build.c b3f15255d5b16e42dafeaa638fd4f8a47c94ed70
@@ -1364,7 +1364,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P ec2573487cb86664f7f056168a764c28ca8236fc
-R 8f8fb58b016bb7e374a57938c502e4cf
-U dan
-Z f04dde703cf9595bec71c1e298c8382a
+P edbcccd349090efff3f975e27451608136a429fc
+R cdbc7a7113ac1ac4a12173f49bd7408c
+U drh
+Z 652b7ed6ab7d80fc7ccd06705dfdfa60
diff --git a/manifest.uuid b/manifest.uuid
index 8110ba7c6b..77cf6612a3 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-edbcccd349090efff3f975e27451608136a429fc
\ No newline at end of file
+e53d497c2d2fbc5014b39f5624b52ce207800698
\ No newline at end of file
diff --git a/src/btree.c b/src/btree.c
index ca0dd0e596..30f24e9129 100644
--- a/src/btree.c
+++ b/src/btree.c
@@ -8955,7 +8955,7 @@ static int checkTreePage(
   u32 usableSize;          /* Usable size of the page */
   u32 contentOffset;       /* Offset to the start of the cell content area */
   u32 *heap = 0;           /* Min-heap used for checking cell coverage */
-  u32 x, prev = 0;
+  u32 x, prev = 0;         /* Next and previous entry on the min-heap */
   const char *saved_zPfx = pCheck->zPfx;
   int saved_v1 = pCheck->v1;
   int saved_v2 = pCheck->v2;
@@ -9018,7 +9018,6 @@ static int checkTreePage(
     ** as the other cell checks, so initialize the heap.  */
     heap = pCheck->heap;
     heap[0] = 0;
-    btreeHeapInsert(heap, contentOffset-1);
   }
 
   /* EVIDENCE-OF: R-02776-14802 The cell pointer array consists of K 2-byte
@@ -9099,7 +9098,6 @@ static int checkTreePage(
     if( !pPage->leaf ){
       heap = pCheck->heap;
       heap[0] = 0;
-      btreeHeapInsert(heap, contentOffset-1);
       for(i=nCell-1; i>=0; i--){
         u32 size;
         pc = get2byteAligned(&data[cellStart+i*2]);
@@ -9133,13 +9131,21 @@ static int checkTreePage(
     }
     /* Analyze the min-heap looking for overlap between cells and/or 
     ** freeblocks, and counting the number of untracked bytes in nFrag.
+    ** 
+    ** Each min-heap entry is of the form:    (start_address<<16)|end_address.
+    ** There is an implied first entry the covers the page header, the cell
+    ** pointer index, and the gap between the cell pointer index and the start
+    ** of cell content.  
+    **
+    ** The loop below pulls entries from the min-heap in order and compares
+    ** the start_address against the previous end_address.  If there is an
+    ** overlap, that means bytes are used multiple times.  If there is a gap,
+    ** that gap is added to the fragmentation count.
     */
     nFrag = 0;
-    assert( heap[0]>0 );
-    assert( (heap[1]>>16)==0 );
-    btreeHeapPull(heap,&prev);
+    prev = contentOffset - 1;   /* Implied first min-heap entry */
     while( btreeHeapPull(heap,&x) ){
-      if( (prev&0xffff)+1>(x>>16) ){
+      if( (prev&0xffff)>=(x>>16) ){
         checkAppendMsg(pCheck,
           "Multiple uses for byte %u of page %d", x>>16, iPage);
         break;

From a153bbc4c28f4404eaffc8f23e63f55f5d4f20ee Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 2 Jul 2015 20:12:34 +0000
Subject: [PATCH 04/40] Copy fts5.h into the build directory as part of
 building fts5.c. Also remove code to edit fts5parse.c from various Makefiles
 and add it to mkfts5c.tcl.

FossilOrigin-Name: 4d8f03dc7b6529cf8d46fc0fcc82112c0394dc13
---
 Makefile.in               |  4 +---
 Makefile.msc              |  6 +-----
 ext/fts5/tool/mkfts5c.tcl | 11 ++++++++---
 main.mk                   |  4 +---
 manifest                  | 20 ++++++++++----------
 manifest.uuid             |  2 +-
 6 files changed, 22 insertions(+), 25 deletions(-)

diff --git a/Makefile.in b/Makefile.in
index 24c414f515..63d311d375 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -999,14 +999,12 @@ fts5parse.c:	$(TOP)/ext/fts5/fts5parse.y lemon
 	cp $(TOP)/ext/fts5/fts5parse.y .
 	rm -f fts5parse.h
 	./lemon $(OPTS) fts5parse.y
-	mv fts5parse.c fts5parse.c.orig
-	cat fts5parse.c.orig | sed 's/yy/fts5yy/g' | sed 's/YY/fts5YY/g' \
-		| sed 's/TOKEN/FTS5TOKEN/g' >> fts5parse.c
 
 fts5parse.h: fts5parse.c
 
 fts5.c: $(FTS5_SRC)
 	$(TCLSH_CMD) $(TOP)/ext/fts5/tool/mkfts5c.tcl
+	cp $(TOP)/ext/fts5/fts5.h .
 
 fts5.lo:	fts5.c $(HDR) $(EXTHDR)
 	$(LTCOMPILE) -DSQLITE_CORE -c fts5.c
diff --git a/Makefile.msc b/Makefile.msc
index ec553a742d..ba00856180 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -1679,16 +1679,12 @@ fts5parse.c:	$(TOP)\ext\fts5\fts5parse.y lemon.exe
 	copy $(TOP)\ext\fts5\fts5parse.y .
 	del /Q fts5parse.h 2>NUL
 	.\lemon.exe $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(OPTS) fts5parse.y
-	move fts5parse.c fts5parse.c.orig
-	type fts5parse.c.orig \
-		| $(NAWK) "/.*/ { gsub(/yy/,\"fts5yy\");print }" \
-		| $(NAWK) "/.*/ { gsub(/YY/,\"fts5YY\");print }" \
-		| $(NAWK) "/.*/ { gsub(/TOKEN/,\"FTS5TOKEN\");print }" > $@
 
 fts5parse.h: fts5parse.c
 
 fts5.c: $(FTS5_SRC)
 	$(TCLSH_CMD) $(TOP)\ext\fts5\tool\mkfts5c.tcl
+	copy $(TOP)\ext\fts5\fts5.h .
 
 fts5.lo:	fts5.c $(HDR) $(EXTHDR)
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c fts5.c
diff --git a/ext/fts5/tool/mkfts5c.tcl b/ext/fts5/tool/mkfts5c.tcl
index c21243fd13..5d292e95d1 100644
--- a/ext/fts5/tool/mkfts5c.tcl
+++ b/ext/fts5/tool/mkfts5c.tcl
@@ -77,15 +77,20 @@ proc fts5c_init {zOut} {
 proc fts5c_printfile {zIn} {
   global G
   set data [readfile $zIn]
-  puts $G(fd) "#line 1 \"[file tail $zIn]\""
+  set zTail [file tail $zIn]
+  puts $G(fd) "#line 1 \"$zTail\""
+
+  set sub_map [list --FTS5-SOURCE-ID-- [fts5_source_id $::srcdir]]
+  if {$zTail=="fts5parse.c"} {
+    lappend sub_map yy fts5yy YY fts5YY TOKEN FTS5TOKEN
+  }
 
-  set srcid_map [list --FTS5-SOURCE-ID-- [fts5_source_id $::srcdir]]
   foreach line [split $data "\n"] {
     if {[regexp {^#include.*fts5} $line]} continue
     if {[regexp {^(const )?[a-zA-Z][a-zA-Z0-9]* [*]?sqlite3Fts5} $line]} {
       set line "static $line"
     }
-    set line [string map $srcid_map $line]
+    set line [string map $sub_map $line]
     puts $G(fd) $line
   }
 }
diff --git a/main.mk b/main.mk
index 2d087e6184..b3a240f4ec 100644
--- a/main.mk
+++ b/main.mk
@@ -655,14 +655,12 @@ fts5parse.c:	$(TOP)/ext/fts5/fts5parse.y lemon
 	cp $(TOP)/ext/fts5/fts5parse.y .
 	rm -f fts5parse.h
 	./lemon $(OPTS) fts5parse.y
-	mv fts5parse.c fts5parse.c.orig
-	cat fts5parse.c.orig | sed 's/yy/fts5yy/g' | sed 's/YY/fts5YY/g' \
-		| sed 's/TOKEN/FTS5TOKEN/g' >> fts5parse.c
 
 fts5parse.h: fts5parse.c
 
 fts5.c: $(FTS5_SRC)
 	tclsh $(TOP)/ext/fts5/tool/mkfts5c.tcl
+	cp $(TOP)/ext/fts5/fts5.h .
 
 
 userauth.o:	$(TOP)/ext/userauth/userauth.c $(HDR) $(EXTHDR)
diff --git a/manifest b/manifest
index d4f2070257..391ee1d4fa 100644
--- a/manifest
+++ b/manifest
@@ -1,9 +1,9 @@
-C When\sdoing\scoverage\sanalysis\sof\sa\sbtree\spage\sfor\spragma\sintegrity_check,\nmake\sthe\sfirst\sentry\s(that\scovers\sthe\sheader,\scell\sindex,\sand\sgap)\simplied,\nfor\sa\sperformance\sboost\sand\ssize\sreduction.
-D 2015-07-02T19:47:08.924
+C Copy\sfts5.h\sinto\sthe\sbuild\sdirectory\sas\spart\sof\sbuilding\sfts5.c.\sAlso\sremove\scode\sto\sedit\sfts5parse.c\sfrom\svarious\sMakefiles\sand\sadd\sit\sto\smkfts5c.tcl.
+D 2015-07-02T20:12:34.556
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
-F Makefile.in 1f525f24e2d3a4defd0ce819c10980caeec967fe
+F Makefile.in 78db7e3b643002849258892ab2a9df10c24ee63d
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
-F Makefile.msc 704da8dafbe56e3a734262d07003f203a7e490b6
+F Makefile.msc 8c5911bcee5867bd594f99cae130b2796520b6f4
 F Makefile.vxworks e1b65dea203f054e71653415bd8f96dcaed47858
 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7
 F VERSION ce0ae95abd7121c534f6917c1c8f2b70d9acd4db
@@ -178,7 +178,7 @@ F ext/fts5/test/fts5unindexed.test e9539d5b78c677315e7ed8ea911d4fd25437c680
 F ext/fts5/test/fts5version.test bed59038e937c40d3c0056d08076db7874c6cd4a
 F ext/fts5/test/fts5vocab.test cdf97b9678484e9bad5062edf9c9106e5c3b0c5c
 F ext/fts5/tool/loadfts5.tcl 7ef3e62131f0434a78e4f5c5b056b09d221710a8
-F ext/fts5/tool/mkfts5c.tcl f9ada2ab7a96a471c7e28a2e390cf4f098348cff
+F ext/fts5/tool/mkfts5c.tcl 6efea6d3fad978a18e2514971f7b16e00a495275
 F ext/fts5/tool/showfts5.tcl 921f33b30c3189deefd2b2cc81f951638544aaf1
 F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43
 F ext/icu/icu.c b2732aef0b076e4276d9b39b5a33cec7a05e1413
@@ -248,7 +248,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 439afc754d942e614e70fbde1d16e29d6dab18a9
+F main.mk a5b0c90e724860fdab98dd54d3a73d9e32cb0702
 F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea
 F mkopcodeh.awk 0e7f04a8eb90f92259e47d80110e4e98d7ce337a
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
@@ -1364,7 +1364,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P edbcccd349090efff3f975e27451608136a429fc
-R cdbc7a7113ac1ac4a12173f49bd7408c
-U drh
-Z 652b7ed6ab7d80fc7ccd06705dfdfa60
+P e53d497c2d2fbc5014b39f5624b52ce207800698
+R 1584f35d5dde8b2b947b977e6cc7cb5d
+U dan
+Z 1aebafb090eb6eb9b114980658da8eef
diff --git a/manifest.uuid b/manifest.uuid
index 77cf6612a3..8adf4ef83c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e53d497c2d2fbc5014b39f5624b52ce207800698
\ No newline at end of file
+4d8f03dc7b6529cf8d46fc0fcc82112c0394dc13
\ No newline at end of file

From 2e43e96107a9789ae67625c34a0b566fe3875384 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 3 Jul 2015 14:34:25 +0000
Subject: [PATCH 05/40] New command-line options for speedtest1: --multithread,
 --nomemstat, --serialized, and --singlethread

FossilOrigin-Name: 2b7567229e3ba7e22271bb909ba977b3de19cca5
---
 manifest          | 14 +++++++-------
 manifest.uuid     |  2 +-
 test/speedtest1.c | 12 ++++++++++++
 3 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 391ee1d4fa..d282824079 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Copy\sfts5.h\sinto\sthe\sbuild\sdirectory\sas\spart\sof\sbuilding\sfts5.c.\sAlso\sremove\scode\sto\sedit\sfts5parse.c\sfrom\svarious\sMakefiles\sand\sadd\sit\sto\smkfts5c.tcl.
-D 2015-07-02T20:12:34.556
+C New\scommand-line\soptions\sfor\sspeedtest1:\s--multithread,\s--nomemstat,\n--serialized,\sand\s--singlethread
+D 2015-07-03T14:34:25.319
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 78db7e3b643002849258892ab2a9df10c24ee63d
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -1005,7 +1005,7 @@ F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523
 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715
 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa
 F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b
-F test/speedtest1.c a2834483e435cf6017b0fead53c5a68c6efc67e8
+F test/speedtest1.c 2e8f367c5fe5e0c3df24a157e17cb2f75c1b88c7
 F test/spellfix.test 0597065ff57042df1f138e6a2611ae19c2698135
 F test/sqldiff1.test 8f6bc7c6a5b3585d350d779c6078869ba402f8f5
 F test/sqllimits1.test e05786eaed7950ff6a2d00031d001d8a26131e68
@@ -1364,7 +1364,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P e53d497c2d2fbc5014b39f5624b52ce207800698
-R 1584f35d5dde8b2b947b977e6cc7cb5d
-U dan
-Z 1aebafb090eb6eb9b114980658da8eef
+P 4d8f03dc7b6529cf8d46fc0fcc82112c0394dc13
+R fbb9a8281a2419b24b4e3dac11d68d49
+U drh
+Z 5691cdc8be217e93d777dc94fd8a29f7
diff --git a/manifest.uuid b/manifest.uuid
index 8adf4ef83c..937c76d6ea 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4d8f03dc7b6529cf8d46fc0fcc82112c0394dc13
\ No newline at end of file
+2b7567229e3ba7e22271bb909ba977b3de19cca5
\ No newline at end of file
diff --git a/test/speedtest1.c b/test/speedtest1.c
index faa44c6c5a..caa2330bbd 100644
--- a/test/speedtest1.c
+++ b/test/speedtest1.c
@@ -15,6 +15,8 @@ static const char zHelp[] =
   "  --journal M         Set the journal_mode to M\n"
   "  --key KEY           Set the encryption key to KEY\n"
   "  --lookaside N SZ    Configure lookaside for N slots of SZ bytes each\n"
+  "  --multithread       Set multithreaded mode\n"
+  "  --nomemstat         Disable memory statistics\n"
   "  --nosync            Set PRAGMA synchronous=OFF\n"
   "  --notnull           Add NOT NULL constraints to table columns\n"
   "  --pagesize N        Set the page size to N\n"
@@ -22,6 +24,8 @@ static const char zHelp[] =
   "  --primarykey        Use PRIMARY KEY instead of UNIQUE where appropriate\n"
   "  --reprepare         Reprepare each statement upon every invocation\n"
   "  --scratch N SZ      Configure scratch memory for N slots of SZ bytes each\n"
+  "  --serialized        Set serialized threading mode\n"
+  "  --singlethread      Set single-threaded mode - disables all mutexing\n"
   "  --sqlonly           No-op.  Only show the SQL that would have been run.\n"
   "  --shrink-memory     Invoke sqlite3_db_release_memory() frequently.\n"
   "  --size N            Relative test size.  Default=100\n"
@@ -1227,6 +1231,10 @@ int main(int argc, char **argv){
         nLook = integerValue(argv[i+1]);
         szLook = integerValue(argv[i+2]);
         i += 2;
+      }else if( strcmp(z,"multithread")==0 ){
+        sqlite3_config(SQLITE_CONFIG_MULTITHREAD);
+      }else if( strcmp(z,"nomemstat")==0 ){
+        sqlite3_config(SQLITE_CONFIG_MEMSTATUS, 0);
       }else if( strcmp(z,"nosync")==0 ){
         noSync = 1;
       }else if( strcmp(z,"notnull")==0 ){
@@ -1253,6 +1261,10 @@ int main(int argc, char **argv){
         nScratch = integerValue(argv[i+1]);
         szScratch = integerValue(argv[i+2]);
         i += 2;
+      }else if( strcmp(z,"serialized")==0 ){
+        sqlite3_config(SQLITE_CONFIG_SERIALIZED);
+      }else if( strcmp(z,"singlethread")==0 ){
+        sqlite3_config(SQLITE_CONFIG_SINGLETHREAD);
       }else if( strcmp(z,"sqlonly")==0 ){
         g.bSqlOnly = 1;
       }else if( strcmp(z,"shrink-memory")==0 ){

From ec7c1de354eaf2de3f1378c5e9e98fe11fb151ef Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 3 Jul 2015 17:14:18 +0000
Subject: [PATCH 06/40] Avoid freeing the array of instance matches each time
 the fts5 xNext() method is called.

FossilOrigin-Name: b9b0c1e50d77f5d6e02f43fbb100c722cb692cc5
---
 ext/fts5/fts5_main.c       | 22 +++++++++++++---------
 ext/fts5/tool/loadfts5.tcl |  1 +
 ext/fts5/tool/mkfts5c.tcl  |  2 +-
 ext/fts5/tool/showfts5.tcl |  1 +
 manifest                   | 20 ++++++++++----------
 manifest.uuid              |  2 +-
 6 files changed, 27 insertions(+), 21 deletions(-)

diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c
index 775c027756..aead532733 100644
--- a/ext/fts5/fts5_main.c
+++ b/ext/fts5/fts5_main.c
@@ -219,9 +219,10 @@ struct Fts5Cursor {
 */
 #define FTS5CSR_REQUIRE_CONTENT   0x01
 #define FTS5CSR_REQUIRE_DOCSIZE   0x02
-#define FTS5CSR_EOF               0x04
-#define FTS5CSR_FREE_ZRANK        0x08
-#define FTS5CSR_REQUIRE_RESEEK    0x10
+#define FTS5CSR_REQUIRE_INST      0x04
+#define FTS5CSR_EOF               0x08
+#define FTS5CSR_FREE_ZRANK        0x10
+#define FTS5CSR_REQUIRE_RESEEK    0x20
 
 #define BitFlagAllTest(x,y) (((x) & (y))==(y))
 #define BitFlagTest(x,y)    (((x) & (y))!=0)
@@ -611,10 +612,11 @@ static int fts5StmtType(Fts5Cursor *pCsr){
 ** specific to the previous row stored by the cursor object.
 */
 static void fts5CsrNewrow(Fts5Cursor *pCsr){
-  CsrFlagSet(pCsr, FTS5CSR_REQUIRE_CONTENT | FTS5CSR_REQUIRE_DOCSIZE );
-  sqlite3_free(pCsr->aInst);
-  pCsr->aInst = 0;
-  pCsr->nInstCount = 0;
+  CsrFlagSet(pCsr, 
+      FTS5CSR_REQUIRE_CONTENT 
+    | FTS5CSR_REQUIRE_DOCSIZE 
+    | FTS5CSR_REQUIRE_INST 
+  );
 }
 
 /*
@@ -629,7 +631,7 @@ static int fts5CloseMethod(sqlite3_vtab_cursor *pCursor){
     Fts5Auxdata *pData;
     Fts5Auxdata *pNext;
 
-    fts5CsrNewrow(pCsr);
+    sqlite3_free(pCsr->aInst);
     if( pCsr->pStmt ){
       int eStmt = fts5StmtType(pCsr);
       sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt);
@@ -1522,7 +1524,7 @@ static int fts5CsrPoslist(Fts5Cursor *pCsr, int iPhrase, const u8 **pa){
 */
 static int fts5CacheInstArray(Fts5Cursor *pCsr){
   int rc = SQLITE_OK;
-  if( pCsr->aInst==0 ){
+  if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST) ){
     Fts5PoslistReader *aIter;     /* One iterator for each phrase */
     int nIter;                    /* Number of iterators/phrases */
     int nByte;
@@ -1564,9 +1566,11 @@ static int fts5CacheInstArray(Fts5Cursor *pCsr){
         sqlite3Fts5PoslistReaderNext(&aIter[iBest]);
       }
 
+      sqlite3_free(pCsr->aInst);
       pCsr->aInst = (int*)buf.p;
       pCsr->nInstCount = nInst;
       sqlite3_free(aIter);
+      CsrFlagClear(pCsr, FTS5CSR_REQUIRE_INST);
     }
   }
   return rc;
diff --git a/ext/fts5/tool/loadfts5.tcl b/ext/fts5/tool/loadfts5.tcl
index b61b567c60..048de3ccd9 100644
--- a/ext/fts5/tool/loadfts5.tcl
+++ b/ext/fts5/tool/loadfts5.tcl
@@ -102,6 +102,7 @@ for {set i 0} {$i < $nOpt} {incr i} {
 set dbfile [lindex $argv end-1]
 if {$O(delete)} { file delete -force $dbfile }
 sqlite3 db $dbfile
+catch { load_static_extension db fts5 }
 db func loadfile loadfile
 
 db transaction {
diff --git a/ext/fts5/tool/mkfts5c.tcl b/ext/fts5/tool/mkfts5c.tcl
index 5d292e95d1..f5cf5197e2 100644
--- a/ext/fts5/tool/mkfts5c.tcl
+++ b/ext/fts5/tool/mkfts5c.tcl
@@ -78,7 +78,7 @@ proc fts5c_printfile {zIn} {
   global G
   set data [readfile $zIn]
   set zTail [file tail $zIn]
-  puts $G(fd) "#line 1 \"$zTail\""
+  puts $G(fd) "#line 2 \"$zTail\""
 
   set sub_map [list --FTS5-SOURCE-ID-- [fts5_source_id $::srcdir]]
   if {$zTail=="fts5parse.c"} {
diff --git a/ext/fts5/tool/showfts5.tcl b/ext/fts5/tool/showfts5.tcl
index 3ed5680182..57ed863b4e 100644
--- a/ext/fts5/tool/showfts5.tcl
+++ b/ext/fts5/tool/showfts5.tcl
@@ -18,6 +18,7 @@ set database [lindex $argv 0]
 set tbl [lindex $argv 1]
 
 sqlite3 db $database
+catch { load_static_extension db fts5 }
 
 db eval "SELECT fts5_decode(rowid, block) AS d FROM ${tbl}_data WHERE id=10" {
   foreach lvl [lrange $d 1 end] {
diff --git a/manifest b/manifest
index d282824079..e80a3ed7cf 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C New\scommand-line\soptions\sfor\sspeedtest1:\s--multithread,\s--nomemstat,\n--serialized,\sand\s--singlethread
-D 2015-07-03T14:34:25.319
+C Avoid\sfreeing\sthe\sarray\sof\sinstance\smatches\seach\stime\sthe\sfts5\sxNext()\smethod\sis\scalled.
+D 2015-07-03T17:14:18.530
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 78db7e3b643002849258892ab2a9df10c24ee63d
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -113,7 +113,7 @@ F ext/fts5/fts5_config.c b2456e9625bca41c51d54c363e369c6356895c90
 F ext/fts5/fts5_expr.c d2e148345639c5a5583e0daa39a639bf298ae6a7
 F ext/fts5/fts5_hash.c 219f4edd72e5cf95b19c33f1058809a18fad5229
 F ext/fts5/fts5_index.c fb1f0de6b4cd02a212c0c9c5580daa64a5634035
-F ext/fts5/fts5_main.c 3390a455109b3d2f1341d45bda0556d23348a339
+F ext/fts5/fts5_main.c f35f445dfe7578b79243a696f30ad8154a1cd313
 F ext/fts5/fts5_storage.c 4cae85b5287b159d9d98174a4e70adf872b0930a
 F ext/fts5/fts5_tcl.c 85eb4e0d0fefa9420b78151496ad4599a1783e20
 F ext/fts5/fts5_tokenize.c 30f97a8c74683797b4cd233790444fbefb3b0708
@@ -177,9 +177,9 @@ F ext/fts5/test/fts5unicode3.test 35c3d02aa7acf7d43d8de3bfe32c15ba96e8928e
 F ext/fts5/test/fts5unindexed.test e9539d5b78c677315e7ed8ea911d4fd25437c680
 F ext/fts5/test/fts5version.test bed59038e937c40d3c0056d08076db7874c6cd4a
 F ext/fts5/test/fts5vocab.test cdf97b9678484e9bad5062edf9c9106e5c3b0c5c
-F ext/fts5/tool/loadfts5.tcl 7ef3e62131f0434a78e4f5c5b056b09d221710a8
-F ext/fts5/tool/mkfts5c.tcl 6efea6d3fad978a18e2514971f7b16e00a495275
-F ext/fts5/tool/showfts5.tcl 921f33b30c3189deefd2b2cc81f951638544aaf1
+F ext/fts5/tool/loadfts5.tcl 95edf0b6b92a09f9ed85595038b1108127987556
+F ext/fts5/tool/mkfts5c.tcl 5745072c7de346e18c7f491e4c3281fe8a1cfe51
+F ext/fts5/tool/showfts5.tcl 8478fbe6cb93a8ff37335c9b48a38c4197dcfe5e
 F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43
 F ext/icu/icu.c b2732aef0b076e4276d9b39b5a33cec7a05e1413
 F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37
@@ -1364,7 +1364,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 4d8f03dc7b6529cf8d46fc0fcc82112c0394dc13
-R fbb9a8281a2419b24b4e3dac11d68d49
-U drh
-Z 5691cdc8be217e93d777dc94fd8a29f7
+P 2b7567229e3ba7e22271bb909ba977b3de19cca5
+R c554dae646bd27cf8152fbda5fd30d51
+U dan
+Z 1785aa0d92187fcd270234eec6cbdc20
diff --git a/manifest.uuid b/manifest.uuid
index 937c76d6ea..600bd8e664 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-2b7567229e3ba7e22271bb909ba977b3de19cca5
\ No newline at end of file
+b9b0c1e50d77f5d6e02f43fbb100c722cb692cc5
\ No newline at end of file

From f156c9b6844692344ed2c6566518d694c8ff7770 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Fri, 3 Jul 2015 17:54:49 +0000
Subject: [PATCH 07/40] Enable use of the __builtin_bswap32() only with GCC 4.3
 and higher.

FossilOrigin-Name: 030f60a7ba171650ce8c0ac32dc166eab80aca32
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/util.c    |  4 ++--
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index e80a3ed7cf..7b2614fac4 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Avoid\sfreeing\sthe\sarray\sof\sinstance\smatches\seach\stime\sthe\sfts5\sxNext()\smethod\sis\scalled.
-D 2015-07-03T17:14:18.530
+C Enable\suse\sof\sthe\s__builtin_bswap32()\sonly\swith\sGCC\s4.3\sand\shigher.
+D 2015-07-03T17:54:49.505
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 78db7e3b643002849258892ab2a9df10c24ee63d
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -388,7 +388,7 @@ F src/treeview.c c84b1a8ebc7f1d00cd76ce4958eeb3ae1021beed
 F src/trigger.c 322f23aad694e8f31d384dcfa386d52a48d3c52f
 F src/update.c 487747b328b7216bb7f6af0695d6937d5c9e605f
 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c
-F src/util.c 075c2878fb698bd387164047ecdf76f6cbacf402
+F src/util.c 46358a204b35971a839341cf64599d65b151ba88
 F src/vacuum.c 2ddd5cad2a7b9cef7f9e431b8c7771634c6b1701
 F src/vdbe.c 5ee4a2bf871418f61d06dc256b9b3a0084b5ec46
 F src/vdbe.h 7a75045d879118b9d3af7e8b3c108f2f27c51473
@@ -1364,7 +1364,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 2b7567229e3ba7e22271bb909ba977b3de19cca5
-R c554dae646bd27cf8152fbda5fd30d51
-U dan
-Z 1785aa0d92187fcd270234eec6cbdc20
+P b9b0c1e50d77f5d6e02f43fbb100c722cb692cc5
+R d6a0b54da592b5928ec9d859952e5495
+U mistachkin
+Z 35231e9548761f3a117648fdf36fee13
diff --git a/manifest.uuid b/manifest.uuid
index 600bd8e664..e92b5d1a94 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b9b0c1e50d77f5d6e02f43fbb100c722cb692cc5
\ No newline at end of file
+030f60a7ba171650ce8c0ac32dc166eab80aca32
\ No newline at end of file
diff --git a/src/util.c b/src/util.c
index bafa3cefa9..091481d921 100644
--- a/src/util.c
+++ b/src/util.c
@@ -1082,7 +1082,7 @@ u32 sqlite3Get4byte(const u8 *p){
   u32 x;
   memcpy(&x,p,4);
   return x;
-#elif SQLITE_BYTEORDER==1234 && defined(__GNUC__)
+#elif SQLITE_BYTEORDER==1234 && defined(__GNUC__) && GCC_VERSION>=4003000
   u32 x;
   memcpy(&x,p,4);
   return __builtin_bswap32(x);
@@ -1098,7 +1098,7 @@ u32 sqlite3Get4byte(const u8 *p){
 void sqlite3Put4byte(unsigned char *p, u32 v){
 #if SQLITE_BYTEORDER==4321
   memcpy(p,&v,4);
-#elif SQLITE_BYTEORDER==1234 && defined(__GNUC__)
+#elif SQLITE_BYTEORDER==1234 && defined(__GNUC__) && GCC_VERSION>=4003000
   u32 x = __builtin_bswap32(v);
   memcpy(p,&x,4);
 #elif SQLITE_BYTEORDER==1234 && defined(_MSC_VER) && _MSC_VER>=1300

From 00a1351e554b65b2bd51ba322ba260e33910af9b Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 3 Jul 2015 19:13:56 +0000
Subject: [PATCH 08/40] Speed up eof checks on fts5 cursors.

FossilOrigin-Name: 3df4af5d8c28863783b0bc867abfbe31cc96f1b9
---
 ext/fts5/fts5_index.c | 36 ++++++++++++++++++++---
 ext/fts5/fts5_main.c  | 67 ++++++++++++++++++++++---------------------
 manifest              | 16 +++++------
 manifest.uuid         |  2 +-
 4 files changed, 75 insertions(+), 46 deletions(-)

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 1cde21f437..2db148c215 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -440,6 +440,7 @@ struct Fts5MultiSegIter {
   int nSeg;                       /* Size of aSeg[] array */
   int bRev;                       /* True to iterate in reverse order */
   int bSkipEmpty;                 /* True to skip deleted entries */
+  int bEof;                       /* True at EOF */
   Fts5SegIter *aSeg;              /* Array of segment iterators */
   Fts5CResult *aFirst;            /* Current merge state (see above) */
 };
@@ -2222,6 +2223,9 @@ static void fts5AssertComparisonResult(
 static void fts5AssertMultiIterSetup(Fts5Index *p, Fts5MultiSegIter *pIter){
   if( p->rc==SQLITE_OK ){
     int i;
+
+    assert( (pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf==0)==pIter->bEof );
+
     for(i=0; i<pIter->nSeg; i+=2){
       Fts5SegIter *p1 = &pIter->aSeg[i];
       Fts5SegIter *p2 = &pIter->aSeg[i+1];
@@ -2230,10 +2234,9 @@ static void fts5AssertMultiIterSetup(Fts5Index *p, Fts5MultiSegIter *pIter){
     }
 
     for(i=1; i<(pIter->nSeg / 2); i+=2){
-      Fts5CResult *pRes = &pIter->aFirst[i];
       Fts5SegIter *p1 = &pIter->aSeg[ pIter->aFirst[i*2].iFirst ];
       Fts5SegIter *p2 = &pIter->aSeg[ pIter->aFirst[i*2+1].iFirst ];
-
+      Fts5CResult *pRes = &pIter->aFirst[i];
       fts5AssertComparisonResult(pIter, p1, p2, pRes);
     }
   }
@@ -2422,6 +2425,16 @@ static void fts5MultiIterAdvanced(
   }
 }
 
+/*
+** Sub-iterator iChanged of iterator pIter has just been advanced. It still
+** points to the same term though - just a different rowid. This function
+** attempts to update the contents of the pIter->aFirst[] accordingly.
+** If it does so successfully, 0 is returned. Otherwise 1.
+**
+** If non-zero is returned, the caller should call fts5MultiIterAdvanced()
+** on the iterator instead. That function does the same as this one, except
+** that it deals with more complicated cases as well.
+*/ 
 static int fts5MultiIterAdvanceRowid(
   Fts5Index *p,                   /* FTS5 backend to iterate within */
   Fts5MultiSegIter *pIter,        /* Iterator to update aFirst[] array for */
@@ -2453,6 +2466,13 @@ static int fts5MultiIterAdvanceRowid(
   return 0;
 }
 
+/*
+** Set the pIter->bEof variable based on the state of the sub-iterators.
+*/
+static void fts5MultiIterSetEof(Fts5MultiSegIter *pIter){
+  pIter->bEof = pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf==0;
+}
+
 /*
 ** Move the iterator to the next entry. 
 **
@@ -2483,6 +2503,7 @@ static void fts5MultiIterNext(
        || fts5MultiIterAdvanceRowid(p, pIter, iFirst)
       ){
         fts5MultiIterAdvanced(p, pIter, iFirst, 1);
+        fts5MultiIterSetEof(pIter);
       }
       fts5AssertMultiIterSetup(p, pIter);
 
@@ -2595,6 +2616,7 @@ static void fts5MultiIterNew(
         fts5MultiIterAdvanced(p, pNew, iEq, iIter);
       }
     }
+    fts5MultiIterSetEof(pNew);
     fts5AssertMultiIterSetup(p, pNew);
 
     if( pNew->bSkipEmpty && fts5MultiIterIsEmpty(p, pNew) ){
@@ -2634,6 +2656,8 @@ static void fts5MultiIterNew2(
         fts5SegIterLoadNPos(p, pIter);
       }
       pData = 0;
+    }else{
+      pNew->bEof = 1;
     }
 
     *ppOut = pNew;
@@ -2647,7 +2671,10 @@ static void fts5MultiIterNew2(
 ** False otherwise.
 */
 static int fts5MultiIterEof(Fts5Index *p, Fts5MultiSegIter *pIter){
-  return (p->rc || pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf==0);
+  assert( p->rc 
+      || (pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf==0)==pIter->bEof 
+  );
+  return (p->rc || pIter->bEof);
 }
 
 /*
@@ -4401,7 +4428,7 @@ int sqlite3Fts5IndexQuery(
 */
 int sqlite3Fts5IterEof(Fts5IndexIter *pIter){
   assert( pIter->pIndex->rc==SQLITE_OK );
-  return fts5MultiIterEof(pIter->pIndex, pIter->pMulti);
+  return pIter->pMulti->bEof;
 }
 
 /*
@@ -4429,6 +4456,7 @@ int sqlite3Fts5IterNextScan(Fts5IndexIter *pIter){
     if( pSeg->pLeaf && pSeg->term.p[0]!=FTS5_MAIN_PREFIX ){
       fts5DataRelease(pSeg->pLeaf);
       pSeg->pLeaf = 0;
+      pMulti->bEof = 1;
     }
   }
 
diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c
index aead532733..05ae344479 100644
--- a/ext/fts5/fts5_main.c
+++ b/ext/fts5/fts5_main.c
@@ -438,12 +438,12 @@ static int fts5CreateMethod(
 /*
 ** The different query plans.
 */
-#define FTS5_PLAN_SCAN           1       /* No usable constraint */
-#define FTS5_PLAN_MATCH          2       /* (<tbl> MATCH ?) */
+#define FTS5_PLAN_MATCH          0       /* (<tbl> MATCH ?) */
+#define FTS5_PLAN_SOURCE         1       /* A source cursor for SORTED_MATCH */
+#define FTS5_PLAN_SPECIAL        2       /* An internal query */
 #define FTS5_PLAN_SORTED_MATCH   3       /* (<tbl> MATCH ? ORDER BY rank) */
-#define FTS5_PLAN_ROWID          4       /* (rowid = ?) */
-#define FTS5_PLAN_SOURCE         5       /* A source cursor for SORTED_MATCH */
-#define FTS5_PLAN_SPECIAL        6       /* An internal query */
+#define FTS5_PLAN_SCAN           4       /* No usable constraint */
+#define FTS5_PLAN_ROWID          5       /* (rowid = ?) */
 
 /*
 ** Implementation of the xBestIndex method for FTS5 tables. Within the 
@@ -764,41 +764,42 @@ static int fts5CursorReseek(Fts5Cursor *pCsr, int *pbSkip){
 */
 static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){
   Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
-  int ePlan = pCsr->ePlan;
-  int bSkip = 0;
   int rc;
 
-  if( (rc = fts5CursorReseek(pCsr, &bSkip)) || bSkip ) return rc;
+  assert( (pCsr->ePlan<2)==
+          (pCsr->ePlan==FTS5_PLAN_MATCH || pCsr->ePlan==FTS5_PLAN_SOURCE) 
+  );
 
-  switch( ePlan ){
-    case FTS5_PLAN_MATCH:
-    case FTS5_PLAN_SOURCE:
-      rc = sqlite3Fts5ExprNext(pCsr->pExpr, pCsr->iLastRowid);
-      if( sqlite3Fts5ExprEof(pCsr->pExpr) ){
-        CsrFlagSet(pCsr, FTS5CSR_EOF);
-      }
-      fts5CsrNewrow(pCsr);
-      break;
-
-    case FTS5_PLAN_SPECIAL: {
+  if( pCsr->ePlan<2 ){
+    int bSkip = 0;
+    if( (rc = fts5CursorReseek(pCsr, &bSkip)) || bSkip ) return rc;
+    rc = sqlite3Fts5ExprNext(pCsr->pExpr, pCsr->iLastRowid);
+    if( sqlite3Fts5ExprEof(pCsr->pExpr) ){
       CsrFlagSet(pCsr, FTS5CSR_EOF);
-      break;
     }
-
-    case FTS5_PLAN_SORTED_MATCH: {
-      rc = fts5SorterNext(pCsr);
-      break;
-    }
-
-    default:
-      rc = sqlite3_step(pCsr->pStmt);
-      if( rc!=SQLITE_ROW ){
+    fts5CsrNewrow(pCsr);
+  }else{
+    switch( pCsr->ePlan ){
+      case FTS5_PLAN_SPECIAL: {
         CsrFlagSet(pCsr, FTS5CSR_EOF);
-        rc = sqlite3_reset(pCsr->pStmt);
-      }else{
-        rc = SQLITE_OK;
+        break;
       }
-      break;
+  
+      case FTS5_PLAN_SORTED_MATCH: {
+        rc = fts5SorterNext(pCsr);
+        break;
+      }
+  
+      default:
+        rc = sqlite3_step(pCsr->pStmt);
+        if( rc!=SQLITE_ROW ){
+          CsrFlagSet(pCsr, FTS5CSR_EOF);
+          rc = sqlite3_reset(pCsr->pStmt);
+        }else{
+          rc = SQLITE_OK;
+        }
+        break;
+    }
   }
   
   return rc;
diff --git a/manifest b/manifest
index 7b2614fac4..e3ab480915 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Enable\suse\sof\sthe\s__builtin_bswap32()\sonly\swith\sGCC\s4.3\sand\shigher.
-D 2015-07-03T17:54:49.505
+C Speed\sup\seof\schecks\son\sfts5\scursors.
+D 2015-07-03T19:13:56.700
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 78db7e3b643002849258892ab2a9df10c24ee63d
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -112,8 +112,8 @@ F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015
 F ext/fts5/fts5_config.c b2456e9625bca41c51d54c363e369c6356895c90
 F ext/fts5/fts5_expr.c d2e148345639c5a5583e0daa39a639bf298ae6a7
 F ext/fts5/fts5_hash.c 219f4edd72e5cf95b19c33f1058809a18fad5229
-F ext/fts5/fts5_index.c fb1f0de6b4cd02a212c0c9c5580daa64a5634035
-F ext/fts5/fts5_main.c f35f445dfe7578b79243a696f30ad8154a1cd313
+F ext/fts5/fts5_index.c 84c8aa1c226898b781f2cbe90040a57ddc4a4259
+F ext/fts5/fts5_main.c 37b0055cb4036c4b4bb4eb36e30ebd1c21c63939
 F ext/fts5/fts5_storage.c 4cae85b5287b159d9d98174a4e70adf872b0930a
 F ext/fts5/fts5_tcl.c 85eb4e0d0fefa9420b78151496ad4599a1783e20
 F ext/fts5/fts5_tokenize.c 30f97a8c74683797b4cd233790444fbefb3b0708
@@ -1364,7 +1364,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P b9b0c1e50d77f5d6e02f43fbb100c722cb692cc5
-R d6a0b54da592b5928ec9d859952e5495
-U mistachkin
-Z 35231e9548761f3a117648fdf36fee13
+P 030f60a7ba171650ce8c0ac32dc166eab80aca32
+R 2a024a904d5b19f97eab1afbe90fa7a1
+U dan
+Z 8ac5a178ca95670b120c3cf4b40c3d86
diff --git a/manifest.uuid b/manifest.uuid
index e92b5d1a94..ee55a3b26f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-030f60a7ba171650ce8c0ac32dc166eab80aca32
\ No newline at end of file
+3df4af5d8c28863783b0bc867abfbe31cc96f1b9
\ No newline at end of file

From 28a10c3c1f28cd6d349762ce9a19759baf9cd1c3 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 3 Jul 2015 20:47:18 +0000
Subject: [PATCH 09/40] Rework the Fts5MultiSegIter structure a bit to make it
 more efficient.

FossilOrigin-Name: 0778825d0ec9315c70659fae8d0640b209049dd8
---
 ext/fts5/fts5_index.c | 24 ++++++++++++------------
 manifest              | 12 ++++++------
 manifest.uuid         |  2 +-
 3 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 2db148c215..6b3dbe0641 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -436,15 +436,6 @@ struct Fts5CResult {
   u8 bTermEq;                     /* True if the terms are equal */
 };
 
-struct Fts5MultiSegIter {
-  int nSeg;                       /* Size of aSeg[] array */
-  int bRev;                       /* True to iterate in reverse order */
-  int bSkipEmpty;                 /* True to skip deleted entries */
-  int bEof;                       /* True at EOF */
-  Fts5SegIter *aSeg;              /* Array of segment iterators */
-  Fts5CResult *aFirst;            /* Current merge state (see above) */
-};
-
 /*
 ** Object for iterating through a single segment, visiting each term/docid
 ** pair in the segment.
@@ -519,6 +510,16 @@ struct Fts5SegIter {
 #define FTS5_SEGITER_REVERSE 0x02
 
 
+struct Fts5MultiSegIter {
+  int nSeg;                       /* Size of aSeg[] array */
+  int bRev;                       /* True to iterate in reverse order */
+  int bSkipEmpty;                 /* True to skip deleted entries */
+  int bEof;                       /* True at EOF */
+  Fts5CResult *aFirst;            /* Current merge state (see above) */
+  Fts5SegIter aSeg[1];            /* Array of segment iterators */
+};
+
+
 /*
 ** Object for iterating through the conents of a single internal node in 
 ** memory.
@@ -2444,7 +2445,7 @@ static int fts5MultiIterAdvanceRowid(
   Fts5SegIter *pNew = &pIter->aSeg[iChanged];
   Fts5SegIter *pOther = &pIter->aSeg[iChanged ^ 0x0001];
 
-  for(i=(pIter->nSeg+iChanged)/2; p->rc==SQLITE_OK; i=i/2){
+  for(i=(pIter->nSeg+iChanged)/2; 1; i=i/2){
     Fts5CResult *pRes = &pIter->aFirst[i];
 
     assert( pNew->pLeaf );
@@ -2522,12 +2523,11 @@ static Fts5MultiSegIter *fts5MultiIterAlloc(
   for(nSlot=2; nSlot<nSeg; nSlot=nSlot*2);
   pNew = fts5IdxMalloc(p, 
       sizeof(Fts5MultiSegIter) +          /* pNew */
-      sizeof(Fts5SegIter) * nSlot +       /* pNew->aSeg[] */
+      sizeof(Fts5SegIter) * (nSlot-1) +   /* pNew->aSeg[] */
       sizeof(Fts5CResult) * nSlot         /* pNew->aFirst[] */
   );
   if( pNew ){
     pNew->nSeg = nSlot;
-    pNew->aSeg = (Fts5SegIter*)&pNew[1];
     pNew->aFirst = (Fts5CResult*)&pNew->aSeg[nSlot];
   }
   return pNew;
diff --git a/manifest b/manifest
index e3ab480915..bfee4bc697 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Speed\sup\seof\schecks\son\sfts5\scursors.
-D 2015-07-03T19:13:56.700
+C Rework\sthe\sFts5MultiSegIter\sstructure\sa\sbit\sto\smake\sit\smore\sefficient.
+D 2015-07-03T20:47:18.417
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 78db7e3b643002849258892ab2a9df10c24ee63d
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -112,7 +112,7 @@ F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015
 F ext/fts5/fts5_config.c b2456e9625bca41c51d54c363e369c6356895c90
 F ext/fts5/fts5_expr.c d2e148345639c5a5583e0daa39a639bf298ae6a7
 F ext/fts5/fts5_hash.c 219f4edd72e5cf95b19c33f1058809a18fad5229
-F ext/fts5/fts5_index.c 84c8aa1c226898b781f2cbe90040a57ddc4a4259
+F ext/fts5/fts5_index.c 50f137bc97ae8c28ea555e156a01a155601cf0ff
 F ext/fts5/fts5_main.c 37b0055cb4036c4b4bb4eb36e30ebd1c21c63939
 F ext/fts5/fts5_storage.c 4cae85b5287b159d9d98174a4e70adf872b0930a
 F ext/fts5/fts5_tcl.c 85eb4e0d0fefa9420b78151496ad4599a1783e20
@@ -1364,7 +1364,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 030f60a7ba171650ce8c0ac32dc166eab80aca32
-R 2a024a904d5b19f97eab1afbe90fa7a1
+P 3df4af5d8c28863783b0bc867abfbe31cc96f1b9
+R bd4f50649100a2ce39bd3db1b7c2671b
 U dan
-Z 8ac5a178ca95670b120c3cf4b40c3d86
+Z 0da0ca80bc4a2dfa97d128394464cd4d
diff --git a/manifest.uuid b/manifest.uuid
index ee55a3b26f..8c9d3a3e9c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-3df4af5d8c28863783b0bc867abfbe31cc96f1b9
\ No newline at end of file
+0778825d0ec9315c70659fae8d0640b209049dd8
\ No newline at end of file

From 93de6538444cd5065ee7a768ff9240043ef15135 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Fri, 3 Jul 2015 21:38:09 +0000
Subject: [PATCH 10/40] Add static mutexes for use by the built-in /
 third-party VFSs and use the built-in VFS mutex where appropriate.

FossilOrigin-Name: b202e2a1d73d104d795d2252b1c6f61d65bfb295
---
 manifest         | 24 ++++++++++++------------
 manifest.uuid    |  2 +-
 src/mutex_noop.c |  2 +-
 src/mutex_unix.c |  6 ++++++
 src/mutex_w32.c  |  6 ++++++
 src/os_unix.c    |  6 +++---
 src/os_win.c     |  6 +++---
 src/sqlite.h.in  |  3 +++
 8 files changed, 35 insertions(+), 20 deletions(-)

diff --git a/manifest b/manifest
index bfee4bc697..76c4625bca 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Rework\sthe\sFts5MultiSegIter\sstructure\sa\sbit\sto\smake\sit\smore\sefficient.
-D 2015-07-03T20:47:18.417
+C Add\sstatic\smutexes\sfor\suse\sby\sthe\sbuilt-in\s/\sthird-party\sVFSs\sand\suse\sthe\sbuilt-in\sVFS\smutex\swhere\sappropriate.
+D 2015-07-03T21:38:09.670
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 78db7e3b643002849258892ab2a9df10c24ee63d
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -303,16 +303,16 @@ F src/memjournal.c 3eb2c0b51adbd869cb6a44780323f05fa904dc85
 F src/msvc.h d9ba56c6851227ab44b3f228a35f3f5772296495
 F src/mutex.c 529e95739f815300a33c73fd8a7d6bdf0c24bd18
 F src/mutex.h 779d588e3b7756ec3ecf7d78cde1d84aba414f85
-F src/mutex_noop.c 529bab0743c3321c940f32c3464de494fd38cfa9
-F src/mutex_unix.c 5cf676464bd19e0a866297515d146e8bf1669dfb
-F src/mutex_w32.c 61660ada28d8308ad190f444c2170c4f2a590c2f
+F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4
+F src/mutex_unix.c b0d280089df0f49545f1318f45d61d07d2f674a8
+F src/mutex_w32.c b601f9e3073f7bd2c1f42a8c0ce59e42d6a08f85
 F src/notify.c 9711a7575036f0d3040ba61bc6e217f13a9888e7
 F src/os.c 8fd25588eeba74068d41102d26810e216999b6c8
 F src/os.h 3e57a24e2794a94d3cf2342c6d9a884888cd96bf
 F src/os_common.h abdb9a191a367793268fe553d25bab894e986a0e
 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa
-F src/os_unix.c 23eb5f56fac54d8fe0cb204291f3b3b2d94f23fc
-F src/os_win.c 27cc135e2d0b8b1e2e4944db1e2669a6a18fa0f8
+F src/os_unix.c 388c023582b17890f10c980b30ec1922b471753b
+F src/os_win.c 40b3af7a47eb1107d0d69e592bec345a3b7b798a
 F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca
 F src/pager.c aa916ca28606ccf4b6877dfc2b643ccbca86589f
 F src/pager.h 6d435f563b3f7fcae4b84433b76a6ac2730036e2
@@ -329,7 +329,7 @@ F src/resolve.c 2d47554370de8de6dd5be060cef9559eec315005
 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e
 F src/select.c 7003fe663bc0636b656874440845a85dcbad4ba7
 F src/shell.c 8af3cced094aebb5f57a8ad739b9dafc7867eed7
-F src/sqlite.h.in 76d2f5637eb795b6300d9dd3c3ec3632ffafd721
+F src/sqlite.h.in 3d951bf985839de7fcf4d3f69568bb4df2641abe
 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad
 F src/sqlite3ext.h be1a718b7d2ce40ceba725ae92c8eb5f18003066
 F src/sqliteInt.h bc2cb1755150fc0b490d1af23ac2911b03f2f92d
@@ -1364,7 +1364,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 3df4af5d8c28863783b0bc867abfbe31cc96f1b9
-R bd4f50649100a2ce39bd3db1b7c2671b
-U dan
-Z 0da0ca80bc4a2dfa97d128394464cd4d
+P 0778825d0ec9315c70659fae8d0640b209049dd8
+R c3e93636a97f33c44303b10895e73ee1
+U mistachkin
+Z 0231ff1cbe7c5e5e0d327907d22772e6
diff --git a/manifest.uuid b/manifest.uuid
index 8c9d3a3e9c..3632f6c9ae 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0778825d0ec9315c70659fae8d0640b209049dd8
\ No newline at end of file
+b202e2a1d73d104d795d2252b1c6f61d65bfb295
\ No newline at end of file
diff --git a/src/mutex_noop.c b/src/mutex_noop.c
index 7f68aea6c1..ecc84b4a94 100644
--- a/src/mutex_noop.c
+++ b/src/mutex_noop.c
@@ -107,7 +107,7 @@ static int debugMutexEnd(void){ return SQLITE_OK; }
 ** that means that a mutex could not be allocated. 
 */
 static sqlite3_mutex *debugMutexAlloc(int id){
-  static sqlite3_debug_mutex aStatic[SQLITE_MUTEX_STATIC_APP3 - 1];
+  static sqlite3_debug_mutex aStatic[SQLITE_MUTEX_STATIC_VFS3 - 1];
   sqlite3_debug_mutex *pNew = 0;
   switch( id ){
     case SQLITE_MUTEX_FAST:
diff --git a/src/mutex_unix.c b/src/mutex_unix.c
index e08448e022..0a493fa6a7 100644
--- a/src/mutex_unix.c
+++ b/src/mutex_unix.c
@@ -105,6 +105,9 @@ static int pthreadMutexEnd(void){ return SQLITE_OK; }
 ** <li>  SQLITE_MUTEX_STATIC_APP1
 ** <li>  SQLITE_MUTEX_STATIC_APP2
 ** <li>  SQLITE_MUTEX_STATIC_APP3
+** <li>  SQLITE_MUTEX_STATIC_VFS1
+** <li>  SQLITE_MUTEX_STATIC_VFS2
+** <li>  SQLITE_MUTEX_STATIC_VFS3
 ** </ul>
 **
 ** The first two constants cause sqlite3_mutex_alloc() to create
@@ -141,6 +144,9 @@ static sqlite3_mutex *pthreadMutexAlloc(int iType){
     SQLITE3_MUTEX_INITIALIZER,
     SQLITE3_MUTEX_INITIALIZER,
     SQLITE3_MUTEX_INITIALIZER,
+    SQLITE3_MUTEX_INITIALIZER,
+    SQLITE3_MUTEX_INITIALIZER,
+    SQLITE3_MUTEX_INITIALIZER,
     SQLITE3_MUTEX_INITIALIZER
   };
   sqlite3_mutex *p;
diff --git a/src/mutex_w32.c b/src/mutex_w32.c
index 6786614d8e..fc943acaa0 100644
--- a/src/mutex_w32.c
+++ b/src/mutex_w32.c
@@ -89,6 +89,9 @@ static sqlite3_mutex winMutex_staticMutexes[] = {
   SQLITE3_MUTEX_INITIALIZER,
   SQLITE3_MUTEX_INITIALIZER,
   SQLITE3_MUTEX_INITIALIZER,
+  SQLITE3_MUTEX_INITIALIZER,
+  SQLITE3_MUTEX_INITIALIZER,
+  SQLITE3_MUTEX_INITIALIZER,
   SQLITE3_MUTEX_INITIALIZER
 };
 
@@ -160,6 +163,9 @@ static int winMutexEnd(void){
 ** <li>  SQLITE_MUTEX_STATIC_APP1
 ** <li>  SQLITE_MUTEX_STATIC_APP2
 ** <li>  SQLITE_MUTEX_STATIC_APP3
+** <li>  SQLITE_MUTEX_STATIC_VFS1
+** <li>  SQLITE_MUTEX_STATIC_VFS2
+** <li>  SQLITE_MUTEX_STATIC_VFS3
 ** </ul>
 **
 ** The first two constants cause sqlite3_mutex_alloc() to create
diff --git a/src/os_unix.c b/src/os_unix.c
index 9ec100323c..d7a94ab096 100644
--- a/src/os_unix.c
+++ b/src/os_unix.c
@@ -629,14 +629,14 @@ static int robust_open(const char *z, int f, mode_t m){
 **   unixEnterLeave()
 */
 static void unixEnterMutex(void){
-  sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER));
+  sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1));
 }
 static void unixLeaveMutex(void){
-  sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER));
+  sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1));
 }
 #ifdef SQLITE_DEBUG
 static int unixMutexHeld(void) {
-  return sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER));
+  return sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1));
 }
 #endif
 
diff --git a/src/os_win.c b/src/os_win.c
index d84bda4ef5..41bd94098c 100644
--- a/src/os_win.c
+++ b/src/os_win.c
@@ -3390,14 +3390,14 @@ static SYSTEM_INFO winSysInfo;
 **   winShmLeaveMutex()
 */
 static void winShmEnterMutex(void){
-  sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER));
+  sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1));
 }
 static void winShmLeaveMutex(void){
-  sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER));
+  sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1));
 }
 #ifndef NDEBUG
 static int winShmMutexHeld(void) {
-  return sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER));
+  return sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1));
 }
 #endif
 
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index c6e77c38cc..34849814fb 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -6291,6 +6291,9 @@ int sqlite3_mutex_notheld(sqlite3_mutex*);
 #define SQLITE_MUTEX_STATIC_APP1      8  /* For use by application */
 #define SQLITE_MUTEX_STATIC_APP2      9  /* For use by application */
 #define SQLITE_MUTEX_STATIC_APP3     10  /* For use by application */
+#define SQLITE_MUTEX_STATIC_VFS1     11  /* For use by built-in VFS */
+#define SQLITE_MUTEX_STATIC_VFS2     12  /* For use by extension VFS */
+#define SQLITE_MUTEX_STATIC_VFS3     13  /* For use by application VFS */
 
 /*
 ** CAPI3REF: Retrieve the mutex for a database connection

From 28ae577a3d6d48302678929c92123b065a913bb3 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Fri, 3 Jul 2015 23:11:36 +0000
Subject: [PATCH 11/40] Enhance mutex testing to include APP and VFS static
 mutexes.

FossilOrigin-Name: 1025873fdfd9e7e53094c48af1a79c60ae50ae97
---
 manifest         | 14 +++----
 manifest.uuid    |  2 +-
 src/test_mutex.c | 96 ++++++++++++++++++++++++++++++++++++++++--------
 test/mutex1.test | 39 ++++++++++++++++----
 4 files changed, 120 insertions(+), 31 deletions(-)

diff --git a/manifest b/manifest
index 76c4625bca..29325cfe72 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sstatic\smutexes\sfor\suse\sby\sthe\sbuilt-in\s/\sthird-party\sVFSs\sand\suse\sthe\sbuilt-in\sVFS\smutex\swhere\sappropriate.
-D 2015-07-03T21:38:09.670
+C Enhance\smutex\stesting\sto\sinclude\sAPP\sand\sVFS\sstatic\smutexes.
+D 2015-07-03T23:11:36.336
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 78db7e3b643002849258892ab2a9df10c24ee63d
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -365,7 +365,7 @@ F src/test_loadext.c a5251f956ab6af21e138dc1f9c0399394a510cb4
 F src/test_malloc.c 208f09a4e21defa496bc1094fcfadea19385a112
 F src/test_multiplex.c 9fefd23f6cc3fa9bf0748a5e453167e7b9f193ce
 F src/test_multiplex.h c08e4e8f8651f0c5e0509b138ff4d5b43ed1f5d3
-F src/test_mutex.c 293042d623ebba969160f471a82aa1551626454f
+F src/test_mutex.c 486bea424c66005d587e8272b2a742a25d251c73
 F src/test_onefile.c 38f7cbe79d5bafe95bde683cc3a53b8ca16daf10
 F src/test_osinst.c 5423dc1d355f594371f27dd292ca54bd320b8196
 F src/test_pcache.c a5cd24730cb43c5b18629043314548c9169abb00
@@ -868,7 +868,7 @@ F test/multiplex.test efd015ca0b5b4a57dc9535b8feb1273eebeadb60
 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a
 F test/multiplex3.test d228f59eac91839a977eac19f21d053f03e4d101
 F test/multiplex4.test e8ae4c4bd70606a5727743241f13b5701990abe4
-F test/mutex1.test 78b2b9bb320e51d156c4efdb71b99b051e7a4b41
+F test/mutex1.test e0a44072d98189003deae4b091106f085d94bea8
 F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660
 F test/nan.test e9648b9d007c7045242af35e11a984d4b169443a
 F test/nolock.test 0540dd96f39b8876e3ffdd8814fad0ea425efeee
@@ -1364,7 +1364,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 0778825d0ec9315c70659fae8d0640b209049dd8
-R c3e93636a97f33c44303b10895e73ee1
+P b202e2a1d73d104d795d2252b1c6f61d65bfb295
+R 1dd264571c6a612a20dc2b3433c44096
 U mistachkin
-Z 0231ff1cbe7c5e5e0d327907d22772e6
+Z 47fb3c7897d245bf0611496fd190514f
diff --git a/manifest.uuid b/manifest.uuid
index 3632f6c9ae..6d7bed7cd8 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b202e2a1d73d104d795d2252b1c6f61d65bfb295
\ No newline at end of file
+1025873fdfd9e7e53094c48af1a79c60ae50ae97
\ No newline at end of file
diff --git a/src/test_mutex.c b/src/test_mutex.c
index c9b4a29ab7..65d853f775 100644
--- a/src/test_mutex.c
+++ b/src/test_mutex.c
@@ -19,9 +19,18 @@
 #include <assert.h>
 #include <string.h>
 
+#define MAX_MUTEXES (SQLITE_MUTEX_STATIC_VFS3+1)
+
 /* defined in main.c */
 extern const char *sqlite3ErrName(int);
 
+static const char *aName[MAX_MUTEXES+1] = {
+  "fast",        "recursive",   "static_master", "static_mem",
+  "static_open", "static_prng", "static_lru",    "static_pmem",
+  "static_app1", "static_app2", "static_app3",   "static_vfs1",
+  "static_vfs2", "static_vfs3", 0
+};
+
 /* A countable mutex */
 struct sqlite3_mutex {
   sqlite3_mutex *pReal;
@@ -30,13 +39,13 @@ struct sqlite3_mutex {
 
 /* State variables */
 static struct test_mutex_globals {
-  int isInstalled;              /* True if installed */
-  int disableInit;              /* True to cause sqlite3_initalize() to fail */
-  int disableTry;               /* True to force sqlite3_mutex_try() to fail */
-  int isInit;                   /* True if initialized */
-  sqlite3_mutex_methods m;      /* Interface to "real" mutex system */
-  int aCounter[8];              /* Number of grabs of each type of mutex */
-  sqlite3_mutex aStatic[6];     /* The six static mutexes */
+  int isInstalled;           /* True if installed */
+  int disableInit;           /* True to cause sqlite3_initalize() to fail */
+  int disableTry;            /* True to force sqlite3_mutex_try() to fail */
+  int isInit;                /* True if initialized */
+  sqlite3_mutex_methods m;   /* Interface to "real" mutex system */
+  int aCounter[MAX_MUTEXES]; /* Number of grabs of each type of mutex */
+  sqlite3_mutex aStatic[MAX_MUTEXES]; /* The static mutexes */
 } g = {0};
 
 /* Return true if the countable mutex is currently held */
@@ -78,7 +87,8 @@ static sqlite3_mutex *counterMutexAlloc(int eType){
   sqlite3_mutex *pRet = 0;
 
   assert( g.isInit );
-  assert(eType<8 && eType>=0);
+  assert( eType>=SQLITE_MUTEX_FAST );
+  assert( eType<=SQLITE_MUTEX_STATIC_VFS3 );
 
   pReal = g.m.xMutexAlloc(eType);
   if( !pReal ) return 0;
@@ -86,7 +96,10 @@ static sqlite3_mutex *counterMutexAlloc(int eType){
   if( eType==SQLITE_MUTEX_FAST || eType==SQLITE_MUTEX_RECURSIVE ){
     pRet = (sqlite3_mutex *)malloc(sizeof(sqlite3_mutex));
   }else{
-    pRet = &g.aStatic[eType-2];
+    int eStaticType = eType - (SQLITE_MUTEX_RECURSIVE + 1);
+    assert( eStaticType>=0 );
+    assert( eStaticType<MAX_MUTEXES );
+    pRet = &g.aStatic[eStaticType];
   }
 
   pRet->eType = eType;
@@ -110,6 +123,8 @@ static void counterMutexFree(sqlite3_mutex *p){
 */
 static void counterMutexEnter(sqlite3_mutex *p){
   assert( g.isInit );
+  assert( p->eType>=0 );
+  assert( p->eType<MAX_MUTEXES );
   g.aCounter[p->eType]++;
   g.m.xMutexEnter(p->pReal);
 }
@@ -119,6 +134,8 @@ static void counterMutexEnter(sqlite3_mutex *p){
 */
 static int counterMutexTry(sqlite3_mutex *p){
   assert( g.isInit );
+  assert( p->eType>=0 );
+  assert( p->eType<MAX_MUTEXES );
   g.aCounter[p->eType]++;
   if( g.disableTry ) return SQLITE_BUSY;
   return g.m.xMutexTry(p->pReal);
@@ -245,10 +262,6 @@ static int test_read_mutex_counters(
 ){
   Tcl_Obj *pRet;
   int ii;
-  char *aName[8] = {
-    "fast",        "recursive",   "static_master", "static_mem", 
-    "static_open", "static_prng", "static_lru",    "static_pmem"
-  };
 
   if( objc!=1 ){
     Tcl_WrongNumArgs(interp, 1, objv, "");
@@ -257,7 +270,7 @@ static int test_read_mutex_counters(
 
   pRet = Tcl_NewObj();
   Tcl_IncrRefCount(pRet);
-  for(ii=0; ii<8; ii++){
+  for(ii=0; ii<MAX_MUTEXES; ii++){
     Tcl_ListObjAppendElement(interp, pRet, Tcl_NewStringObj(aName[ii], -1));
     Tcl_ListObjAppendElement(interp, pRet, Tcl_NewIntObj(g.aCounter[ii]));
   }
@@ -283,7 +296,7 @@ static int test_clear_mutex_counters(
     return TCL_ERROR;
   }
 
-  for(ii=0; ii<8; ii++){
+  for(ii=0; ii<MAX_MUTEXES; ii++){
     g.aCounter[ii] = 0;
   }
   return TCL_OK;
@@ -371,6 +384,56 @@ static sqlite3 *getDbPointer(Tcl_Interp *pInterp, Tcl_Obj *pObj){
   return db;
 }
 
+static sqlite3_mutex *getStaticMutexPointer(
+  Tcl_Interp *pInterp,
+  Tcl_Obj *pObj
+){
+  int iMutex;
+  if( Tcl_GetIndexFromObj(pInterp, pObj, aName, "mutex name", 0, &iMutex) ){
+    return 0;
+  }
+  assert( iMutex!=SQLITE_MUTEX_FAST && iMutex!=SQLITE_MUTEX_RECURSIVE );
+  return counterMutexAlloc(iMutex);
+}
+
+static int test_enter_static_mutex(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  sqlite3_mutex *pMutex;
+  if( objc!=2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "NAME");
+    return TCL_ERROR;
+  }
+  pMutex = getStaticMutexPointer(interp, objv[1]);
+  if( !pMutex ){
+    return TCL_ERROR;
+  }
+  sqlite3_mutex_enter(pMutex);
+  return TCL_OK;
+}
+
+static int test_leave_static_mutex(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  sqlite3_mutex *pMutex;
+  if( objc!=2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "NAME");
+    return TCL_ERROR;
+  }
+  pMutex = getStaticMutexPointer(interp, objv[1]);
+  if( !pMutex ){
+    return TCL_ERROR;
+  }
+  sqlite3_mutex_leave(pMutex);
+  return TCL_OK;
+}
+
 static int test_enter_db_mutex(
   void * clientData,
   Tcl_Interp *interp,
@@ -418,6 +481,9 @@ int Sqlitetest_mutex_Init(Tcl_Interp *interp){
     { "sqlite3_initialize",      (Tcl_ObjCmdProc*)test_initialize },
     { "sqlite3_config",          (Tcl_ObjCmdProc*)test_config },
 
+    { "enter_static_mutex",      (Tcl_ObjCmdProc*)test_enter_static_mutex },
+    { "leave_static_mutex",      (Tcl_ObjCmdProc*)test_leave_static_mutex },
+
     { "enter_db_mutex",          (Tcl_ObjCmdProc*)test_enter_db_mutex },
     { "leave_db_mutex",          (Tcl_ObjCmdProc*)test_leave_db_mutex },
 
diff --git a/test/mutex1.test b/test/mutex1.test
index 4bdf769ad3..340e271175 100644
--- a/test/mutex1.test
+++ b/test/mutex1.test
@@ -37,9 +37,9 @@ proc mutex_counters {varname} {
 
 #-------------------------------------------------------------------------
 # Tests mutex1-1.* test that sqlite3_config() returns SQLITE_MISUSE if
-# is called at the wrong time. And that the first time sqlite3_initialize 
+# is called at the wrong time. And that the first time sqlite3_initialize
 # is called it obtains the 'static_master' mutex 3 times and a recursive
-# mutex (sqlite3Config.pInitMutex) twice. Subsequent calls are no-ops 
+# mutex (sqlite3Config.pInitMutex) twice. Subsequent calls are no-ops
 # that do not require any mutexes.
 #
 do_test mutex1-1.0 {
@@ -102,12 +102,16 @@ ifcapable threadsafe&&shared_cache {
   foreach {mode mutexes} {
     singlethread {}
     multithread  {
-      fast static_lru static_master static_mem static_open static_prng 
-      static_pmem
+      fast static_app1 static_app2 static_app3
+      static_lru static_master static_mem static_open
+      static_prng static_pmem static_vfs1 static_vfs2
+      static_vfs3
     }
     serialized  {
-      fast recursive static_lru static_master static_mem static_open 
-      static_prng static_pmem
+      fast recursive static_app1 static_app2
+      static_app3 static_lru static_master static_mem
+      static_open static_prng static_pmem static_vfs1
+      static_vfs2 static_vfs3
     }
   } {
 
@@ -129,9 +133,28 @@ ifcapable threadsafe&&shared_cache {
     ifcapable !memorymanage {
       regsub { static_lru} $mutexes {} mutexes
     }
-    do_test mutex1.2.$mode.3 {
+    if {$mode ne "singlethread"} {
+      do_test mutex1.2.$mode.3 {
+        #
+        # NOTE: Make sure all the app and vfs mutexes get used.
+        #
+        enter_static_mutex static_app1
+        leave_static_mutex static_app1
+        enter_static_mutex static_app2
+        leave_static_mutex static_app2
+        enter_static_mutex static_app3
+        leave_static_mutex static_app3
+        enter_static_mutex static_vfs1
+        leave_static_mutex static_vfs1
+        enter_static_mutex static_vfs2
+        leave_static_mutex static_vfs2
+        enter_static_mutex static_vfs3
+        leave_static_mutex static_vfs3
+      } {}
+    }
+    do_test mutex1.2.$mode.4 {
       mutex_counters counters
-  
+
       set res [list]
       foreach {key value} [array get counters] {
         if {$key ne "total" && $value > 0} {

From 82194aa6e2285225514c43b34199fdb4d2767815 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Fri, 3 Jul 2015 23:12:33 +0000
Subject: [PATCH 12/40] Update clean target in MSVC makefile.

FossilOrigin-Name: e6c03e72010e919dbac2055b6c3f3b13b1663b15
---
 Makefile.msc  |  2 +-
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/Makefile.msc b/Makefile.msc
index ba00856180..02a3330a24 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -1833,7 +1833,7 @@ clean:
 	del /Q sqlite3_analyzer.exe sqlite3_analyzer.c 2>NUL
 	del /Q sqlite-*-output.vsix 2>NUL
 	del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe 2>NUL
-	del /Q fts5.c fts5parse.* 2>NUL
+	del /Q fts5.c fts5.h fts5parse.* 2>NUL
 
 # Dynamic link library section.
 #
diff --git a/manifest b/manifest
index 29325cfe72..89ed18ab55 100644
--- a/manifest
+++ b/manifest
@@ -1,9 +1,9 @@
-C Enhance\smutex\stesting\sto\sinclude\sAPP\sand\sVFS\sstatic\smutexes.
-D 2015-07-03T23:11:36.336
+C Update\sclean\starget\sin\sMSVC\smakefile.
+D 2015-07-03T23:12:33.008
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 78db7e3b643002849258892ab2a9df10c24ee63d
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
-F Makefile.msc 8c5911bcee5867bd594f99cae130b2796520b6f4
+F Makefile.msc 86926e2330ad4b7ae6d4674762b53b293873b3fb
 F Makefile.vxworks e1b65dea203f054e71653415bd8f96dcaed47858
 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7
 F VERSION ce0ae95abd7121c534f6917c1c8f2b70d9acd4db
@@ -1364,7 +1364,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P b202e2a1d73d104d795d2252b1c6f61d65bfb295
-R 1dd264571c6a612a20dc2b3433c44096
+P 1025873fdfd9e7e53094c48af1a79c60ae50ae97
+R b3a13182f647db180395c6651e571755
 U mistachkin
-Z 47fb3c7897d245bf0611496fd190514f
+Z e84454cfa4f76a73cfc440339f3d3545
diff --git a/manifest.uuid b/manifest.uuid
index 6d7bed7cd8..6a1a2a6337 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-1025873fdfd9e7e53094c48af1a79c60ae50ae97
\ No newline at end of file
+e6c03e72010e919dbac2055b6c3f3b13b1663b15
\ No newline at end of file

From 74db35672644830089c0e73675746c7dff71aa40 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Fri, 3 Jul 2015 23:29:55 +0000
Subject: [PATCH 13/40] Correction to check-in [1025873fdf], tighten up the
 number of static test mutexes.

FossilOrigin-Name: 4e515897af97cb3a4158bcc34318992e8dcee77a
---
 manifest         | 12 ++++++------
 manifest.uuid    |  2 +-
 src/test_mutex.c |  9 +++++----
 3 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index 89ed18ab55..8eeb2e122f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Update\sclean\starget\sin\sMSVC\smakefile.
-D 2015-07-03T23:12:33.008
+C Correction\sto\scheck-in\s[1025873fdf],\stighten\sup\sthe\snumber\sof\sstatic\stest\smutexes.
+D 2015-07-03T23:29:55.094
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 78db7e3b643002849258892ab2a9df10c24ee63d
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -365,7 +365,7 @@ F src/test_loadext.c a5251f956ab6af21e138dc1f9c0399394a510cb4
 F src/test_malloc.c 208f09a4e21defa496bc1094fcfadea19385a112
 F src/test_multiplex.c 9fefd23f6cc3fa9bf0748a5e453167e7b9f193ce
 F src/test_multiplex.h c08e4e8f8651f0c5e0509b138ff4d5b43ed1f5d3
-F src/test_mutex.c 486bea424c66005d587e8272b2a742a25d251c73
+F src/test_mutex.c dbdfaff8580071f2212a0deae3325a93a737819c
 F src/test_onefile.c 38f7cbe79d5bafe95bde683cc3a53b8ca16daf10
 F src/test_osinst.c 5423dc1d355f594371f27dd292ca54bd320b8196
 F src/test_pcache.c a5cd24730cb43c5b18629043314548c9169abb00
@@ -1364,7 +1364,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 1025873fdfd9e7e53094c48af1a79c60ae50ae97
-R b3a13182f647db180395c6651e571755
+P e6c03e72010e919dbac2055b6c3f3b13b1663b15
+R 238f2504523ab15b65a60c28dd676b51
 U mistachkin
-Z e84454cfa4f76a73cfc440339f3d3545
+Z 50c8bc64878c693af322bb7e054ce81e
diff --git a/manifest.uuid b/manifest.uuid
index 6a1a2a6337..b73c320071 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e6c03e72010e919dbac2055b6c3f3b13b1663b15
\ No newline at end of file
+4e515897af97cb3a4158bcc34318992e8dcee77a
\ No newline at end of file
diff --git a/src/test_mutex.c b/src/test_mutex.c
index 65d853f775..995b89a4c6 100644
--- a/src/test_mutex.c
+++ b/src/test_mutex.c
@@ -19,7 +19,8 @@
 #include <assert.h>
 #include <string.h>
 
-#define MAX_MUTEXES (SQLITE_MUTEX_STATIC_VFS3+1)
+#define MAX_MUTEXES        (SQLITE_MUTEX_STATIC_VFS3+1)
+#define STATIC_MUTEXES     (MAX_MUTEXES-(SQLITE_MUTEX_RECURSIVE+1))
 
 /* defined in main.c */
 extern const char *sqlite3ErrName(int);
@@ -45,7 +46,7 @@ static struct test_mutex_globals {
   int isInit;                /* True if initialized */
   sqlite3_mutex_methods m;   /* Interface to "real" mutex system */
   int aCounter[MAX_MUTEXES]; /* Number of grabs of each type of mutex */
-  sqlite3_mutex aStatic[MAX_MUTEXES]; /* The static mutexes */
+  sqlite3_mutex aStatic[STATIC_MUTEXES]; /* The static mutexes */
 } g = {0};
 
 /* Return true if the countable mutex is currently held */
@@ -96,9 +97,9 @@ static sqlite3_mutex *counterMutexAlloc(int eType){
   if( eType==SQLITE_MUTEX_FAST || eType==SQLITE_MUTEX_RECURSIVE ){
     pRet = (sqlite3_mutex *)malloc(sizeof(sqlite3_mutex));
   }else{
-    int eStaticType = eType - (SQLITE_MUTEX_RECURSIVE + 1);
+    int eStaticType = eType - (MAX_MUTEXES - STATIC_MUTEXES);
     assert( eStaticType>=0 );
-    assert( eStaticType<MAX_MUTEXES );
+    assert( eStaticType<STATIC_MUTEXES );
     pRet = &g.aStatic[eStaticType];
   }
 

From 68b134c73a135fd425996bad0aec89ba2defea6c Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Sat, 4 Jul 2015 00:04:37 +0000
Subject: [PATCH 14/40] Sync up Makefile clean targets for FTS5.

FossilOrigin-Name: 43ead44ef6416d69602e123c3be7a258772837f1
---
 Makefile.in   |  1 +
 Makefile.msc  |  2 +-
 main.mk       |  1 +
 manifest      | 16 ++++++++--------
 manifest.uuid |  2 +-
 5 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/Makefile.in b/Makefile.in
index 63d311d375..3aaa4b5827 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -1189,6 +1189,7 @@ clean:
 	rm -f fuzzershell fuzzershell.exe
 	rm -f fuzzcheck fuzzcheck.exe
 	rm -f sqldiff sqldiff.exe
+	rm -f fts5.c fts5.h fts5parse.c fts5parse.h
 
 distclean:	clean
 	rm -f config.log config.status libtool Makefile sqlite3.pc
diff --git a/Makefile.msc b/Makefile.msc
index 02a3330a24..7a57489870 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -1833,7 +1833,7 @@ clean:
 	del /Q sqlite3_analyzer.exe sqlite3_analyzer.c 2>NUL
 	del /Q sqlite-*-output.vsix 2>NUL
 	del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe 2>NUL
-	del /Q fts5.c fts5.h fts5parse.* 2>NUL
+	del /Q fts5.c fts5.h fts5parse.c fts5parse.h 2>NUL
 
 # Dynamic link library section.
 #
diff --git a/main.mk b/main.mk
index b3a240f4ec..3c248996df 100644
--- a/main.mk
+++ b/main.mk
@@ -869,3 +869,4 @@ clean:
 	rm -f fuzzershell fuzzershell.exe
 	rm -f fuzzcheck fuzzcheck.exe
 	rm -f sqldiff sqldiff.exe
+	rm -f fts5.c fts5.h fts5parse.c fts5parse.h
diff --git a/manifest b/manifest
index 8eeb2e122f..e4c861b73e 100644
--- a/manifest
+++ b/manifest
@@ -1,9 +1,9 @@
-C Correction\sto\scheck-in\s[1025873fdf],\stighten\sup\sthe\snumber\sof\sstatic\stest\smutexes.
-D 2015-07-03T23:29:55.094
+C Sync\sup\sMakefile\sclean\stargets\sfor\sFTS5.
+D 2015-07-04T00:04:37.262
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
-F Makefile.in 78db7e3b643002849258892ab2a9df10c24ee63d
+F Makefile.in 017bf0511d1b2dd1db5e16488fbf75a17b526cbc
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
-F Makefile.msc 86926e2330ad4b7ae6d4674762b53b293873b3fb
+F Makefile.msc e97727cdfab1d4b890eb1f7e9d3f11a6717d329d
 F Makefile.vxworks e1b65dea203f054e71653415bd8f96dcaed47858
 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7
 F VERSION ce0ae95abd7121c534f6917c1c8f2b70d9acd4db
@@ -248,7 +248,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk a5b0c90e724860fdab98dd54d3a73d9e32cb0702
+F main.mk 2fda07d2e73021aaa98aeaf98e4b39b2ef2d5ccb
 F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea
 F mkopcodeh.awk 0e7f04a8eb90f92259e47d80110e4e98d7ce337a
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
@@ -1364,7 +1364,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P e6c03e72010e919dbac2055b6c3f3b13b1663b15
-R 238f2504523ab15b65a60c28dd676b51
+P 4e515897af97cb3a4158bcc34318992e8dcee77a
+R 0ed175228476a5c7601d367e38925a77
 U mistachkin
-Z 50c8bc64878c693af322bb7e054ce81e
+Z 452e29ca7fd5db2edd7c137285535715
diff --git a/manifest.uuid b/manifest.uuid
index b73c320071..1ab45e2737 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4e515897af97cb3a4158bcc34318992e8dcee77a
\ No newline at end of file
+43ead44ef6416d69602e123c3be7a258772837f1
\ No newline at end of file

From 72e191ed71d2183d284e7759c812f6dcec2ef990 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 4 Jul 2015 11:14:20 +0000
Subject: [PATCH 15/40] Do not preserve the btreeInitPage() results in PRAGMA
 integrity_check if there were cell sizing errors because PRAGMA
 cell_size_check was turned off during the btreeInitPage() call.

FossilOrigin-Name: 743bf1f184be316971f2e9c651bcf573bde8654a
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/btree.c   |  3 +++
 3 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index e4c861b73e..786e403a09 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Sync\sup\sMakefile\sclean\stargets\sfor\sFTS5.
-D 2015-07-04T00:04:37.262
+C Do\snot\spreserve\sthe\sbtreeInitPage()\sresults\sin\sPRAGMA\sintegrity_check\sif\nthere\swere\scell\ssizing\serrors\sbecause\sPRAGMA\scell_size_check\swas\sturned\soff\nduring\sthe\sbtreeInitPage()\scall.
+D 2015-07-04T11:14:20.284
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 017bf0511d1b2dd1db5e16488fbf75a17b526cbc
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -269,7 +269,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240
 F src/backup.c 4d9134dc988a87838c06056c89c0e8c4700a0452
 F src/bitvec.c d1f21d7d91690747881f03940584f4cc548c9d3d
 F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79
-F src/btree.c 7705ce3e17c2b0cb8d4c3d935b269270d3c2a548
+F src/btree.c 740ab30df845f4a2c2bdb731ed191b4b63b2b2eb
 F src/btree.h 969adc948e89e449220ff0ff724c94bb2a52e9f1
 F src/btreeInt.h 2ad754dd4528baa8d0946a593cc373b890bf859e
 F src/build.c b3f15255d5b16e42dafeaa638fd4f8a47c94ed70
@@ -1364,7 +1364,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 4e515897af97cb3a4158bcc34318992e8dcee77a
-R 0ed175228476a5c7601d367e38925a77
-U mistachkin
-Z 452e29ca7fd5db2edd7c137285535715
+P 43ead44ef6416d69602e123c3be7a258772837f1
+R a66ab7da98093b47fcf696f0e3baef91
+U drh
+Z 85e64a90850a0b5ecb5f23fb7b4db837
diff --git a/manifest.uuid b/manifest.uuid
index 1ab45e2737..6c31177c16 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-43ead44ef6416d69602e123c3be7a258772837f1
\ No newline at end of file
+743bf1f184be316971f2e9c651bcf573bde8654a
\ No newline at end of file
diff --git a/src/btree.c b/src/btree.c
index 30f24e9129..f353e8df1f 100644
--- a/src/btree.c
+++ b/src/btree.c
@@ -8959,6 +8959,7 @@ static int checkTreePage(
   const char *saved_zPfx = pCheck->zPfx;
   int saved_v1 = pCheck->v1;
   int saved_v2 = pCheck->v2;
+  u8 savedIsInit;
 
   /* Check that the page exists
   */
@@ -8976,6 +8977,7 @@ static int checkTreePage(
 
   /* Clear MemPage.isInit to make sure the corruption detection code in
   ** btreeInitPage() is executed.  */
+  savedIsInit = pPage->isInit;
   pPage->isInit = 0;
   if( (rc = btreeInitPage(pPage))!=0 ){
     assert( rc==SQLITE_CORRUPT );  /* The only possible error from InitPage */
@@ -9168,6 +9170,7 @@ static int checkTreePage(
   }
 
 end_of_check:
+  if( !doCoverageCheck ) pPage->isInit = savedIsInit;
   releasePage(pPage);
   pCheck->zPfx = saved_zPfx;
   pCheck->v1 = saved_v1;

From 70c8885417dfaba63df287ffa4ff23ec0170a026 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sat, 4 Jul 2015 15:35:02 +0000
Subject: [PATCH 16/40] Fix a problem in the fts5 integrity-check.

FossilOrigin-Name: 4e473559c9bdcf4d0b823e995a3642cd50ebd337
---
 ext/fts5/fts5_index.c |  7 +++----
 manifest              | 14 +++++++-------
 manifest.uuid         |  2 +-
 3 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 6b3dbe0641..d29ee04a15 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -4828,7 +4828,6 @@ static void fts5TestTerm(
     int nTerm = pPrev->n-1;            /* Size of zTerm in bytes */
     int iIdx = (pPrev->p[0] - FTS5_MAIN_PREFIX);
     int flags = (iIdx==0 ? 0 : FTS5INDEX_QUERY_PREFIX);
-    int rc;
     u64 ck1 = 0;
     u64 ck2 = 0;
 
@@ -5054,6 +5053,9 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){
     i64 iRowid = fts5MultiIterRowid(pIter);
     char *z = (char*)fts5MultiIterTerm(pIter, &n);
 
+    /* If this is a new term, query for it. Update cksum3 with the results. */
+    fts5TestTerm(p, &term, z, n, cksum2, &cksum3);
+
     poslist.n = 0;
     fts5MultiIterPoslist(p, pIter, 0, &poslist);
     while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){
@@ -5061,9 +5063,6 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){
       int iTokOff = FTS5_POS2OFFSET(iPos);
       cksum2 ^= fts5IndexEntryCksum(iRowid, iCol, iTokOff, -1, z, n);
     }
-
-    /* If this is a new term, query for it. Update cksum3 with the results. */
-    fts5TestTerm(p, &term, z, n, cksum2, &cksum3);
   }
   fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3);
 
diff --git a/manifest b/manifest
index 786e403a09..52014a9ab7 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Do\snot\spreserve\sthe\sbtreeInitPage()\sresults\sin\sPRAGMA\sintegrity_check\sif\nthere\swere\scell\ssizing\serrors\sbecause\sPRAGMA\scell_size_check\swas\sturned\soff\nduring\sthe\sbtreeInitPage()\scall.
-D 2015-07-04T11:14:20.284
+C Fix\sa\sproblem\sin\sthe\sfts5\sintegrity-check.
+D 2015-07-04T15:35:02.267
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 017bf0511d1b2dd1db5e16488fbf75a17b526cbc
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -112,7 +112,7 @@ F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015
 F ext/fts5/fts5_config.c b2456e9625bca41c51d54c363e369c6356895c90
 F ext/fts5/fts5_expr.c d2e148345639c5a5583e0daa39a639bf298ae6a7
 F ext/fts5/fts5_hash.c 219f4edd72e5cf95b19c33f1058809a18fad5229
-F ext/fts5/fts5_index.c 50f137bc97ae8c28ea555e156a01a155601cf0ff
+F ext/fts5/fts5_index.c 6564c103148e8b59d6b816f34d0b31f5f4025054
 F ext/fts5/fts5_main.c 37b0055cb4036c4b4bb4eb36e30ebd1c21c63939
 F ext/fts5/fts5_storage.c 4cae85b5287b159d9d98174a4e70adf872b0930a
 F ext/fts5/fts5_tcl.c 85eb4e0d0fefa9420b78151496ad4599a1783e20
@@ -1364,7 +1364,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 43ead44ef6416d69602e123c3be7a258772837f1
-R a66ab7da98093b47fcf696f0e3baef91
-U drh
-Z 85e64a90850a0b5ecb5f23fb7b4db837
+P 743bf1f184be316971f2e9c651bcf573bde8654a
+R 9bf7551534e5c2a541b49f913ee30aa3
+U dan
+Z 59b496d519385223221d6df6daef9f3a
diff --git a/manifest.uuid b/manifest.uuid
index 6c31177c16..0ee45a678c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-743bf1f184be316971f2e9c651bcf573bde8654a
\ No newline at end of file
+4e473559c9bdcf4d0b823e995a3642cd50ebd337
\ No newline at end of file

From 01c5c00c1299dd4c7933cc3df231c04183aedc47 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 4 Jul 2015 18:15:04 +0000
Subject: [PATCH 17/40] Preserve the number of requested PAGECACHE pages even
 if the memory pointer or size is zero.  Enhance the pcache1.c header comment
 to explain the memory layout of a page cache line.

FossilOrigin-Name: dacb2a615ce1c0573baf4518000454038745cf2a
---
 manifest      | 16 ++++++++--------
 manifest.uuid |  2 +-
 src/malloc.c  |  3 +--
 src/pcache1.c | 35 +++++++++++++++++++++++++++++++++++
 4 files changed, 45 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index 52014a9ab7..380f99bcb1 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sproblem\sin\sthe\sfts5\sintegrity-check.
-D 2015-07-04T15:35:02.267
+C Preserve\sthe\snumber\sof\srequested\sPAGECACHE\spages\seven\sif\sthe\smemory\spointer\nor\ssize\sis\szero.\s\sEnhance\sthe\spcache1.c\sheader\scomment\sto\sexplain\sthe\smemory\nlayout\sof\sa\spage\scache\sline.
+D 2015-07-04T18:15:04.326
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 017bf0511d1b2dd1db5e16488fbf75a17b526cbc
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -293,7 +293,7 @@ F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e
 F src/lempar.c 92bafa308607dd985ca389a788cd9e0a2b608712
 F src/loadext.c e722f4b832f923744788365df5fb8515c0bc8a47
 F src/main.c 92d79bfa1a36c7c554700bb58eb8327abff1ac5c
-F src/malloc.c 9be4e645f2fb411e5a04cf97e91f68b4faa6dc81
+F src/malloc.c 19461e159bccf0e2cf06a50e867963d0a7b124a8
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c abe6ee469b6c5a35c7f22bfeb9c9bac664a1c987
 F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3
@@ -319,7 +319,7 @@ F src/pager.h 6d435f563b3f7fcae4b84433b76a6ac2730036e2
 F src/parse.y 6d60dda8f8d418b6dc034f1fbccd816c459983a8
 F src/pcache.c cde06aa50962595e412d497e22fd2e07878ba1f0
 F src/pcache.h 9968603796240cdf83da7e7bef76edf90619cea9
-F src/pcache1.c 9ec20f98f50ed7415019303ae9bd3745d4b7bd9b
+F src/pcache1.c b800e90896640bb0d1b0f6baa34fb059ca3b8b2e
 F src/pragma.c c1f4d012ea9f6b1ce52d341b2cd0ad72d560afd7
 F src/pragma.h b8632d7cdda7b25323fa580e3e558a4f0d4502cc
 F src/prepare.c 82e5db1013846a819f198336fed72c44c974e7b1
@@ -1364,7 +1364,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 743bf1f184be316971f2e9c651bcf573bde8654a
-R 9bf7551534e5c2a541b49f913ee30aa3
-U dan
-Z 59b496d519385223221d6df6daef9f3a
+P 4e473559c9bdcf4d0b823e995a3642cd50ebd337
+R 3bd4765a55a11bf2fbafbabfd8169c43
+U drh
+Z 0199ebd97104f84a289dac04b3e8dc79
diff --git a/manifest.uuid b/manifest.uuid
index 0ee45a678c..67c01f1a49 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4e473559c9bdcf4d0b823e995a3642cd50ebd337
\ No newline at end of file
+dacb2a615ce1c0573baf4518000454038745cf2a
\ No newline at end of file
diff --git a/src/malloc.c b/src/malloc.c
index 97b9cd5778..1e77734ecb 100644
--- a/src/malloc.c
+++ b/src/malloc.c
@@ -193,10 +193,9 @@ int sqlite3MallocInit(void){
     sqlite3GlobalConfig.nScratch = 0;
   }
   if( sqlite3GlobalConfig.pPage==0 || sqlite3GlobalConfig.szPage<512
-      || sqlite3GlobalConfig.nPage<1 ){
+      || sqlite3GlobalConfig.nPage<=0 ){
     sqlite3GlobalConfig.pPage = 0;
     sqlite3GlobalConfig.szPage = 0;
-    sqlite3GlobalConfig.nPage = 0;
   }
   rc = sqlite3GlobalConfig.m.xInit(sqlite3GlobalConfig.m.pAppData);
   if( rc!=SQLITE_OK ) memset(&mem0, 0, sizeof(mem0));
diff --git a/src/pcache1.c b/src/pcache1.c
index 7185ab441b..797354bd75 100644
--- a/src/pcache1.c
+++ b/src/pcache1.c
@@ -15,6 +15,41 @@
 ** of the SQLITE_CONFIG_PAGECACHE and sqlite3_release_memory() features.
 ** If the default page cache implementation is overridden, then neither of
 ** these two features are available.
+**
+** A Page cache line looks like this:
+**
+**  -------------------------------------------------------------
+**  |  database page content   |  PgHdr1  |  MemPage  |  PgHdr  |
+**  -------------------------------------------------------------
+**
+** The database page content is up front (so that buffer overreads tend to
+** flow harmlessly into the PgHdr1, MemPage, and PgHdr extensions).   MemPage
+** is the extension added by the btree.c module containing information such
+** as the database page number and how that database page is used.  PgHdr
+** is added by the pcache.c layer and contains information used to keep track
+** of which pages are "dirty".  PgHdr1 is an extension added by this
+** module (pcache1.c).  The PgHdr1 header is a subclass of sqlite3_pcache_page.
+** PgHdr1 contains information needed to look up a page by its page number.
+** The superclass sqlite3_pcache_page.pBuf points to the start of the
+** database page content and sqlite3_pcache_page.pExtra points to PgHdr.
+**
+** The size of the extension (MemPage+PgHdr+PgHdr1) can be determined at
+** runtime using sqlite3_config(SQLITE_CONFIG_PCACHE_HDRSZ, &size).  The
+** sizes of the extensions sum to 272 bytes on x64 for 3.8.10, but this
+** size can vary according to architecture, compile-time options, and
+** SQLite library version number.
+**
+** If SQLITE_PCACHE_SEPARATE_HEADER is defined, then the extension is obtained
+** using a separate memory allocation from the database page content.  This
+** seeks to overcome the "clownfoot" problem of allocating a few bytes more
+** than a power of two with the memory allocator rounding up to the next
+** power of two, and leaving the rounded-up space unused.
+**
+** This module tracks pointers to PgHdr1 objects.  Only pcache.c communicates
+** with this module.  Information is passed back and forth as PgHdr1 pointers.
+**
+** The pcache.c and pager.c modules deal pointers to PgHdr objects.
+** The btree.c module deals with pointers to MemPage objects.
 */
 
 #include "sqliteInt.h"

From 30ae503167e29743919f6df61a4d14edf087ca3d Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sat, 4 Jul 2015 18:44:07 +0000
Subject: [PATCH 18/40] Optimize seek operations on fts5 b-trees.

FossilOrigin-Name: 8cf02090ce53ec150492d77d9e5e5f27665bd34f
---
 ext/fts5/fts5_index.c | 127 +++++++++++++++++++++++++++++++++++++-----
 manifest              |  14 ++---
 manifest.uuid         |   2 +-
 3 files changed, 121 insertions(+), 22 deletions(-)

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index d29ee04a15..ae3ff35dbb 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -2012,6 +2012,118 @@ static void fts5SegIterLoadDlidx(Fts5Index *p, Fts5SegIter *pIter){
   pIter->pDlidx = fts5DlidxIterInit(p, bRev, iSeg, pIter->iTermLeafPgno);
 }
 
+#ifdef SQLITE_DEBUG
+static void fts5AssertNodeSeekOk(
+  Fts5Data *pNode,
+  const u8 *pTerm, int nTerm,     /* Term to search for */
+  int iExpectPg,
+  int bExpectDlidx
+){
+  int bDlidx;
+  int iPg;
+  int rc = SQLITE_OK;
+  Fts5NodeIter node;
+
+  fts5NodeIterInit(pNode->p, pNode->n, &node);
+  assert( node.term.n==0 );
+  iPg = node.iChild;
+  bDlidx = node.bDlidx;
+  for(fts5NodeIterNext(&rc, &node);
+      node.aData && fts5BufferCompareBlob(&node.term, pTerm, nTerm)<=0;
+      fts5NodeIterNext(&rc, &node)
+  ){
+    iPg = node.iChild;
+    bDlidx = node.bDlidx;
+  }
+  fts5NodeIterFree(&node);
+
+  assert( rc!=SQLITE_OK || iPg==iExpectPg );
+  assert( rc!=SQLITE_OK || bDlidx==bExpectDlidx );
+}
+#else
+#define fts5AssertNodeSeekOk(v,w,x,y,z)
+#endif
+
+/*
+** Argument pNode is an internal b-tree node. This function searches
+** within the node for the largest term that is smaller than or equal
+** to (pTerm/nTerm).
+**
+** It returns the associated page number. Or, if (pTerm/nTerm) is smaller
+** than all terms within the node, the leftmost child page number. 
+**
+** Before returning, (*pbDlidx) is set to true if the last term on the
+** returned child page number has a doclist-index. Or left as is otherwise.
+*/
+static int fts5NodeSeek(
+  Fts5Data *pNode,                /* Node to search */
+  const u8 *pTerm, int nTerm,     /* Term to search for */
+  int *pbDlidx                    /* OUT: True if dlidx flag is set */
+){
+  int iPg;
+  u8 *pPtr = pNode->p;
+  u8 *pEnd = &pPtr[pNode->n];
+  int nMatch = 0;                 /* Number of bytes of pTerm already matched */
+  
+  assert( *pbDlidx==0 );
+
+  pPtr += fts5GetVarint32(pPtr, iPg);
+  while( pPtr<pEnd ){
+    int nEmpty = 0;
+    int nKeep;
+    int nNew;
+
+    /* If there is a "no terms" record at pPtr, read it now. Store the
+    ** number of termless pages in nEmpty. If it indicates a doclist-index, 
+    ** set (*pbDlidx) to true.*/
+    if( *pPtr<2 ){
+      *pbDlidx = (*pPtr==0x01);
+      pPtr++;
+      pPtr += fts5GetVarint32(pPtr, nEmpty);
+    }
+
+    /* Read the next "term" pointer. Set nKeep to the number of bytes to
+    ** keep from the previous term, and nNew to the number of bytes of
+    ** new data that will be appended to it. */
+    nKeep = (int)*pPtr++;
+    nNew = (int)*pPtr++;
+    if( (nKeep | nNew) & 0x0080 ){
+      pPtr -= 2;
+      pPtr += fts5GetVarint32(pPtr, nKeep);
+      pPtr += fts5GetVarint32(pPtr, nNew);
+    }
+    nKeep -= 2;
+
+    /* Compare (pTerm/nTerm) to the current term on the node (the one described
+    ** by nKeep/nNew). If the node term is larger, break out of the while()
+    ** loop. 
+    **
+    ** Otherwise, if (pTerm/nTerm) is larger or the two terms are equal, 
+    ** leave variable nMatch set to the size of the largest prefix common to
+    ** both terms in bytes.  */
+    if( nKeep==nMatch ){
+      int nTst = MIN(nNew, nTerm-nMatch);
+      int i;
+      for(i=0; i<nTst; i++){
+        if( pTerm[nKeep+i]!=pPtr[i] ) break;
+      }
+      nMatch += i;
+      assert( nMatch<=nTerm );
+
+      if( i<nNew && (nMatch==nTerm || pPtr[i] > pTerm[nMatch]) ) break;
+    }else if( nKeep<nMatch ){
+      break;
+    }
+
+    iPg += 1 + nEmpty;
+    *pbDlidx = 0;
+    pPtr += nNew;
+  }
+
+  fts5AssertNodeSeekOk(pNode, pTerm, nTerm, iPg, *pbDlidx);
+  return iPg;
+}
+
 /*
 ** Initialize the object pIter to point to term pTerm/nTerm within segment
 ** pSeg. If there is no such term in the index, the iterator is set to EOF.
@@ -2040,24 +2152,11 @@ static void fts5SegIterSeekInit(
   /* This block sets stack variable iPg to the leaf page number that may
   ** contain term (pTerm/nTerm), if it is present in the segment. */
   for(h=pSeg->nHeight-1; h>0; h--){
-    Fts5NodeIter node;              /* For iterating through internal nodes */
     i64 iRowid = FTS5_SEGMENT_ROWID(pSeg->iSegid, h, iPg);
     Fts5Data *pNode = fts5DataRead(p, iRowid);
     if( pNode==0 ) break;
 
-    fts5NodeIterInit(pNode->p, pNode->n, &node);
-    assert( node.term.n==0 );
-
-    iPg = node.iChild;
-    bDlidx = node.bDlidx;
-    for(fts5NodeIterNext(&p->rc, &node);
-        node.aData && fts5BufferCompareBlob(&node.term, pTerm, nTerm)<=0;
-        fts5NodeIterNext(&p->rc, &node)
-    ){
-      iPg = node.iChild;
-      bDlidx = node.bDlidx;
-    }
-    fts5NodeIterFree(&node);
+    iPg = fts5NodeSeek(pNode, pTerm, nTerm, &bDlidx);
     fts5DataRelease(pNode);
   }
 
diff --git a/manifest b/manifest
index 380f99bcb1..23e00d3b33 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Preserve\sthe\snumber\sof\srequested\sPAGECACHE\spages\seven\sif\sthe\smemory\spointer\nor\ssize\sis\szero.\s\sEnhance\sthe\spcache1.c\sheader\scomment\sto\sexplain\sthe\smemory\nlayout\sof\sa\spage\scache\sline.
-D 2015-07-04T18:15:04.326
+C Optimize\sseek\soperations\son\sfts5\sb-trees.
+D 2015-07-04T18:44:07.139
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 017bf0511d1b2dd1db5e16488fbf75a17b526cbc
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -112,7 +112,7 @@ F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015
 F ext/fts5/fts5_config.c b2456e9625bca41c51d54c363e369c6356895c90
 F ext/fts5/fts5_expr.c d2e148345639c5a5583e0daa39a639bf298ae6a7
 F ext/fts5/fts5_hash.c 219f4edd72e5cf95b19c33f1058809a18fad5229
-F ext/fts5/fts5_index.c 6564c103148e8b59d6b816f34d0b31f5f4025054
+F ext/fts5/fts5_index.c 6224454702f852392cae8755c920f93b06b09283
 F ext/fts5/fts5_main.c 37b0055cb4036c4b4bb4eb36e30ebd1c21c63939
 F ext/fts5/fts5_storage.c 4cae85b5287b159d9d98174a4e70adf872b0930a
 F ext/fts5/fts5_tcl.c 85eb4e0d0fefa9420b78151496ad4599a1783e20
@@ -1364,7 +1364,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 4e473559c9bdcf4d0b823e995a3642cd50ebd337
-R 3bd4765a55a11bf2fbafbabfd8169c43
-U drh
-Z 0199ebd97104f84a289dac04b3e8dc79
+P dacb2a615ce1c0573baf4518000454038745cf2a
+R a74787a76aedf64f2fa9d52272be4a28
+U dan
+Z 70c5b32ed488e093070d3138a449d5f1
diff --git a/manifest.uuid b/manifest.uuid
index 67c01f1a49..85d4c60780 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-dacb2a615ce1c0573baf4518000454038745cf2a
\ No newline at end of file
+8cf02090ce53ec150492d77d9e5e5f27665bd34f
\ No newline at end of file

From b63ce02f570beff45f7fe899541569a63997f464 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sun, 5 Jul 2015 22:15:10 +0000
Subject: [PATCH 19/40] Do not allow recursive CTEs that use aggregate queries
 in the recursive part.

FossilOrigin-Name: 6d2999afbc25b9c238e4028f637c10eaaf0ec75e
---
 manifest        | 16 ++++++++--------
 manifest.uuid   |  2 +-
 src/select.c    | 12 ++++++++----
 test/with1.test |  7 +++++++
 4 files changed, 24 insertions(+), 13 deletions(-)

diff --git a/manifest b/manifest
index 23e00d3b33..1954e6998c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Optimize\sseek\soperations\son\sfts5\sb-trees.
-D 2015-07-04T18:44:07.139
+C Do\snot\sallow\srecursive\sCTEs\sthat\suse\saggregate\squeries\sin\sthe\srecursive\spart.
+D 2015-07-05T22:15:10.026
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 017bf0511d1b2dd1db5e16488fbf75a17b526cbc
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -327,7 +327,7 @@ F src/printf.c db11b5960105ee661dcac690f2ae6276e49bf251
 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696
 F src/resolve.c 2d47554370de8de6dd5be060cef9559eec315005
 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e
-F src/select.c 7003fe663bc0636b656874440845a85dcbad4ba7
+F src/select.c d3c04f01549317afbe02455c4ca9465100e9c5fe
 F src/shell.c 8af3cced094aebb5f57a8ad739b9dafc7867eed7
 F src/sqlite.h.in 3d951bf985839de7fcf4d3f69568bb4df2641abe
 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad
@@ -1296,7 +1296,7 @@ F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c
 F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c
 F test/win32lock.test fbf107c91d8f5512be5a5b87c4c42ab9fdd54972
 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d
-F test/with1.test e99845d4f4bf7863b61f104de554c61739d65764
+F test/with1.test a1e8660be88e2eb4648f8860f831d1e38b5b5443
 F test/with2.test ee227a663586aa09771cafd4fa269c5217eaf775
 F test/withM.test e97f2a8c506ab3ea9eab94e6f6072f6cc924c991
 F test/without_rowid1.test 1a7b9bd51b899928d327052df9741d2fe8dbe701
@@ -1364,7 +1364,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P dacb2a615ce1c0573baf4518000454038745cf2a
-R a74787a76aedf64f2fa9d52272be4a28
-U dan
-Z 70c5b32ed488e093070d3138a449d5f1
+P 8cf02090ce53ec150492d77d9e5e5f27665bd34f
+R 2ced95f329e3743e78bec1f4f82122ca
+U drh
+Z 74748fde014fce363e747a5eecd57396
diff --git a/manifest.uuid b/manifest.uuid
index 85d4c60780..0975b167fb 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-8cf02090ce53ec150492d77d9e5e5f27665bd34f
\ No newline at end of file
+6d2999afbc25b9c238e4028f637c10eaaf0ec75e
\ No newline at end of file
diff --git a/src/select.c b/src/select.c
index ed5d2c6bf4..13de029420 100644
--- a/src/select.c
+++ b/src/select.c
@@ -2070,10 +2070,14 @@ static void generateWithRecursiveQuery(
   /* Execute the recursive SELECT taking the single row in Current as
   ** the value for the recursive-table. Store the results in the Queue.
   */
-  p->pPrior = 0;
-  sqlite3Select(pParse, p, &destQueue);
-  assert( p->pPrior==0 );
-  p->pPrior = pSetup;
+  if( p->selFlags & SF_Aggregate ){
+    sqlite3ErrorMsg(pParse, "recursive aggregate queries not supported");
+  }else{
+    p->pPrior = 0;
+    sqlite3Select(pParse, p, &destQueue);
+    assert( p->pPrior==0 );
+    p->pPrior = pSetup;
+  }
 
   /* Keep running the loop until the Queue is empty */
   sqlite3VdbeAddOp2(v, OP_Goto, 0, addrTop);
diff --git a/test/with1.test b/test/with1.test
index 8d8b6f75b5..71eec61e7d 100644
--- a/test/with1.test
+++ b/test/with1.test
@@ -857,5 +857,12 @@ do_catchsql_test 15.1 {
   SELECT x FROM d;
 } {1 {no such column: rowid}}
 
+# 2015-07-05:  Do not allow aggregate recursive queries
+#
+do_catchsql_test 16.1 {
+  WITH RECURSIVE
+    i(x) AS (VALUES(1) UNION SELECT count(*) FROM i)
+  SELECT * FROM i;
+} {1 {recursive aggregate queries not supported}}
 
 finish_test

From ee70a84ea7d709979dc09fa42bf4aa9afae21667 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 6 Jul 2015 18:54:52 +0000
Subject: [PATCH 20/40] Enhance separate pcache1 to allocate a block of pages
 from heap on startup, if possible, for a 5.2% performance improvement.

FossilOrigin-Name: aa7341c8736732a0a59f6688cc34e78be02a7bfc
---
 manifest          |  19 +++---
 manifest.uuid     |   2 +-
 src/global.c      |   2 +-
 src/pcache1.c     | 148 ++++++++++++++++++++++++++++++++--------------
 test/speedtest1.c |  12 ++--
 5 files changed, 126 insertions(+), 57 deletions(-)

diff --git a/manifest b/manifest
index 1954e6998c..c2e77a579b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Do\snot\sallow\srecursive\sCTEs\sthat\suse\saggregate\squeries\sin\sthe\srecursive\spart.
-D 2015-07-05T22:15:10.026
+C Enhance\sseparate\spcache1\sto\sallocate\sa\sblock\sof\spages\sfrom\sheap\son\sstartup,\nif\spossible,\sfor\sa\s5.2%\sperformance\simprovement.
+D 2015-07-06T18:54:52.424
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 017bf0511d1b2dd1db5e16488fbf75a17b526cbc
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -283,7 +283,7 @@ F src/expr.c c5c58e4d01c7ceb2266791d8d877f1b23a88e316
 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb
 F src/fkey.c c9b63a217d86582c22121699a47f22f524608869
 F src/func.c a98ea5880dc50e9ca6dd6f57079a37b9cfcdecf1
-F src/global.c 4f77cadbc5427d00139ba43d0f3979804cbb700e
+F src/global.c 32c0ebfdca59db55c761f1405abf7bdfd24ecae0
 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5
 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094
 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08
@@ -319,7 +319,7 @@ F src/pager.h 6d435f563b3f7fcae4b84433b76a6ac2730036e2
 F src/parse.y 6d60dda8f8d418b6dc034f1fbccd816c459983a8
 F src/pcache.c cde06aa50962595e412d497e22fd2e07878ba1f0
 F src/pcache.h 9968603796240cdf83da7e7bef76edf90619cea9
-F src/pcache1.c b800e90896640bb0d1b0f6baa34fb059ca3b8b2e
+F src/pcache1.c 28f3434788cac24bc95f8c1af4a61cbde179ef18
 F src/pragma.c c1f4d012ea9f6b1ce52d341b2cd0ad72d560afd7
 F src/pragma.h b8632d7cdda7b25323fa580e3e558a4f0d4502cc
 F src/prepare.c 82e5db1013846a819f198336fed72c44c974e7b1
@@ -1005,7 +1005,7 @@ F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523
 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715
 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa
 F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b
-F test/speedtest1.c 2e8f367c5fe5e0c3df24a157e17cb2f75c1b88c7
+F test/speedtest1.c 54f211994e2fb5b3f7c5c82137378f46ca89aae8
 F test/spellfix.test 0597065ff57042df1f138e6a2611ae19c2698135
 F test/sqldiff1.test 8f6bc7c6a5b3585d350d779c6078869ba402f8f5
 F test/sqllimits1.test e05786eaed7950ff6a2d00031d001d8a26131e68
@@ -1364,7 +1364,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 8cf02090ce53ec150492d77d9e5e5f27665bd34f
-R 2ced95f329e3743e78bec1f4f82122ca
+P 6d2999afbc25b9c238e4028f637c10eaaf0ec75e
+R 773f110b2d424984f83126d1917fdcbb
+T *branch * pcache-bulk-local
+T *sym-pcache-bulk-local *
+T -sym-trunk *
 U drh
-Z 74748fde014fce363e747a5eecd57396
+Z a5a615d5b6a74570375077a92838726e
diff --git a/manifest.uuid b/manifest.uuid
index 0975b167fb..29a53816d9 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6d2999afbc25b9c238e4028f637c10eaaf0ec75e
\ No newline at end of file
+aa7341c8736732a0a59f6688cc34e78be02a7bfc
\ No newline at end of file
diff --git a/src/global.c b/src/global.c
index 61450b3d35..3b1e804bb8 100644
--- a/src/global.c
+++ b/src/global.c
@@ -186,7 +186,7 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = {
    0,                         /* nScratch */
    (void*)0,                  /* pPage */
    0,                         /* szPage */
-   0,                         /* nPage */
+   100,                       /* nPage */
    0,                         /* mxParserStack */
    0,                         /* sharedCacheEnabled */
    SQLITE_SORTER_PMASZ,       /* szPma */
diff --git a/src/pcache1.c b/src/pcache1.c
index 797354bd75..30ea7d0d8b 100644
--- a/src/pcache1.c
+++ b/src/pcache1.c
@@ -50,8 +50,35 @@
 **
 ** The pcache.c and pager.c modules deal pointers to PgHdr objects.
 ** The btree.c module deals with pointers to MemPage objects.
+**
+** SOURCE OF PAGE CACHE MEMORY:
+**
+** Memory for a page might come from any of three sources:
+**
+**    (1)  The general-purpose memory allocator - sqlite3Malloc()
+**    (2)  Global page-cache memory provided using sqlite3_config() with
+**         SQLITE_CONFIG_PAGECACHE.
+**    (3)  PCache-local bulk allocation.
+**
+** The third case is a chunk of heap memory (defaulting to 100 pages worth)
+** that is allocated when the page cache is created.  The size of the local
+** bulk allocation can be adjusted using 
+**
+**     sqlite3_config(SQLITE_CONFIG_PCACHE, 0, 0, N).
+**
+** If N is positive, then N pages worth of memory are allocated using a single
+** sqlite3Malloc() call and that memory is used for the first N pages allocated.
+** Or if N is negative, then -1024*N bytes of memory are allocated and used
+** for as many pages as can be accomodated.
+**
+** Only one of (2) or (3) can be used.  Once the memory available to (2) or
+** (3) is exhausted, subsequent allocations fail over to the general-purpose
+** memory allocator (1).
+**
+** Earlier versions of SQLite used only methods (1) and (2).  But experiments
+** show that method (3) with N==100 provides about a 5% performance boost for
+** common workloads.
 */
-
 #include "sqliteInt.h"
 
 typedef struct PCache1 PCache1;
@@ -105,8 +132,9 @@ struct PCache1 {
   ** The PGroup mutex must be held when accessing nMax.
   */
   PGroup *pGroup;                     /* PGroup this cache belongs to */
-  int szPage;                         /* Size of allocated pages in bytes */
-  int szExtra;                        /* Size of extra space in bytes */
+  int szPage;                         /* Size of database content section */
+  int szExtra;                        /* sizeof(MemPage)+sizeof(PgHdr) */
+  int szAlloc;                        /* Total size of one pcache line */
   int bPurgeable;                     /* True if cache is purgeable */
   unsigned int nMin;                  /* Minimum number of pages reserved */
   unsigned int nMax;                  /* Configured "cache_size" value */
@@ -120,6 +148,8 @@ struct PCache1 {
   unsigned int nPage;                 /* Total number of pages in apHash */
   unsigned int nHash;                 /* Number of slots in apHash[] */
   PgHdr1 **apHash;                    /* Hash table for fast lookup by key */
+  PgHdr1 *pFree;                      /* List of unused pcache-local pages */
+  void *pBulk;                        /* Bulk memory used by pcache-local */
 };
 
 /*
@@ -132,6 +162,7 @@ struct PgHdr1 {
   sqlite3_pcache_page page;
   unsigned int iKey;             /* Key value (page number) */
   u8 isPinned;                   /* Page in use, not on the LRU list */
+  u8 isBulkLocal;                /* This page from bulk local storage */
   PgHdr1 *pNext;                 /* Next in hash table chain */
   PCache1 *pCache;               /* Cache that currently owns this page */
   PgHdr1 *pLruNext;              /* Next in LRU list of unpinned pages */
@@ -139,8 +170,8 @@ struct PgHdr1 {
 };
 
 /*
-** Free slots in the allocator used to divide up the buffer provided using
-** the SQLITE_CONFIG_PAGECACHE mechanism.
+** Free slots in the allocator used to divide up the global page cache
+** buffer provided using the SQLITE_CONFIG_PAGECACHE mechanism.
 */
 struct PgFreeslot {
   PgFreeslot *pNext;  /* Next free slot */
@@ -161,7 +192,7 @@ static SQLITE_WSD struct PCacheGlobal {
   int szSlot;                    /* Size of each free slot */
   int nSlot;                     /* The number of pcache slots */
   int nReserve;                  /* Try to keep nFreeSlot above this */
-  void *pStart, *pEnd;           /* Bounds of pagecache malloc range */
+  void *pStart, *pEnd;           /* Bounds of global page cache memory */
   /* Above requires no mutex.  Use mutex below for variable that follow. */
   sqlite3_mutex *mutex;          /* Mutex for accessing the following: */
   PgFreeslot *pFree;             /* Free page blocks */
@@ -208,6 +239,7 @@ static SQLITE_WSD struct PCacheGlobal {
 void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){
   if( pcache1.isInit ){
     PgFreeslot *p;
+    if( pBuf==0 ) sz = n = 0;
     sz = ROUNDDOWN8(sz);
     pcache1.szSlot = sz;
     pcache1.nSlot = pcache1.nFreeSlot = n;
@@ -272,9 +304,9 @@ static void *pcache1Alloc(int nByte){
 /*
 ** Free an allocated buffer obtained from pcache1Alloc().
 */
-static int pcache1Free(void *p){
+static void pcache1Free(void *p){
   int nFreed = 0;
-  if( p==0 ) return 0;
+  if( p==0 ) return;
   if( p>=pcache1.pStart && p<pcache1.pEnd ){
     PgFreeslot *pSlot;
     sqlite3_mutex_enter(pcache1.mutex);
@@ -289,15 +321,14 @@ static int pcache1Free(void *p){
   }else{
     assert( sqlite3MemdebugHasType(p, MEMTYPE_PCACHE) );
     sqlite3MemdebugSetType(p, MEMTYPE_HEAP);
-    nFreed = sqlite3MallocSize(p);
 #ifndef SQLITE_DISABLE_PAGECACHE_OVERFLOW_STATS
+    nFreed = sqlite3MallocSize(p);
     sqlite3_mutex_enter(pcache1.mutex);
     sqlite3StatusDown(SQLITE_STATUS_PAGECACHE_OVERFLOW, nFreed);
     sqlite3_mutex_leave(pcache1.mutex);
 #endif
     sqlite3_free(p);
   }
-  return nFreed;
 }
 
 #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
@@ -325,34 +356,38 @@ static PgHdr1 *pcache1AllocPage(PCache1 *pCache){
   PgHdr1 *p = 0;
   void *pPg;
 
-  /* The group mutex must be released before pcache1Alloc() is called. This
-  ** is because it may call sqlite3_release_memory(), which assumes that 
-  ** this mutex is not held. */
   assert( sqlite3_mutex_held(pCache->pGroup->mutex) );
-  pcache1LeaveMutex(pCache->pGroup);
+  if( pCache->pFree ){
+    p = pCache->pFree;
+    pCache->pFree = p->pNext;
+    p->pNext = 0;
+  }else{
+    /* The group mutex must be released before pcache1Alloc() is called. This
+    ** is because it may call sqlite3_release_memory(), which assumes that 
+    ** this mutex is not held. */
+    pcache1LeaveMutex(pCache->pGroup);
 #ifdef SQLITE_PCACHE_SEPARATE_HEADER
-  pPg = pcache1Alloc(pCache->szPage);
-  p = sqlite3Malloc(sizeof(PgHdr1) + pCache->szExtra);
-  if( !pPg || !p ){
-    pcache1Free(pPg);
-    sqlite3_free(p);
-    pPg = 0;
-  }
+    pPg = pcache1Alloc(pCache->szPage);
+    p = sqlite3Malloc(sizeof(PgHdr1) + pCache->szExtra);
+    if( !pPg || !p ){
+      pcache1Free(pPg);
+      sqlite3_free(p);
+      pPg = 0;
+    }
 #else
-  pPg = pcache1Alloc(ROUND8(sizeof(PgHdr1)) + pCache->szPage + pCache->szExtra);
-  p = (PgHdr1 *)&((u8 *)pPg)[pCache->szPage];
+    pPg = pcache1Alloc(pCache->szAlloc);
+    p = (PgHdr1 *)&((u8 *)pPg)[pCache->szPage];
 #endif
-  pcache1EnterMutex(pCache->pGroup);
-
-  if( pPg ){
+    pcache1EnterMutex(pCache->pGroup);
+    if( pPg==0 ) return 0;
     p->page.pBuf = pPg;
     p->page.pExtra = &p[1];
-    if( pCache->bPurgeable ){
-      pCache->pGroup->nCurrentPage++;
-    }
-    return p;
+    p->isBulkLocal = 0;
   }
-  return 0;
+  if( pCache->bPurgeable ){
+    pCache->pGroup->nCurrentPage++;
+  }
+  return p;
 }
 
 /*
@@ -366,10 +401,15 @@ static void pcache1FreePage(PgHdr1 *p){
   if( ALWAYS(p) ){
     PCache1 *pCache = p->pCache;
     assert( sqlite3_mutex_held(p->pCache->pGroup->mutex) );
-    pcache1Free(p->page.pBuf);
+    if( p->isBulkLocal ){
+      p->pNext = pCache->pFree;
+      pCache->pFree = p;
+    }else{
+      pcache1Free(p->page.pBuf);
 #ifdef SQLITE_PCACHE_SEPARATE_HEADER
-    sqlite3_free(p);
+      sqlite3_free(p);
 #endif
+    }
     if( pCache->bPurgeable ){
       pCache->pGroup->nCurrentPage--;
     }
@@ -640,6 +680,7 @@ static sqlite3_pcache *pcache1Create(int szPage, int szExtra, int bPurgeable){
     pCache->pGroup = pGroup;
     pCache->szPage = szPage;
     pCache->szExtra = szExtra;
+    pCache->szAlloc = szPage + szExtra + ROUND8(sizeof(PgHdr1));
     pCache->bPurgeable = (bPurgeable ? 1 : 0);
     pcache1EnterMutex(pGroup);
     pcache1ResizeHash(pCache);
@@ -649,6 +690,34 @@ static sqlite3_pcache *pcache1Create(int szPage, int szExtra, int bPurgeable){
       pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage;
     }
     pcache1LeaveMutex(pGroup);
+    if( separateCache
+     && sqlite3GlobalConfig.szPage==0
+     && sqlite3GlobalConfig.nPage!=0
+    ){
+      int szBulk;
+      char *zBulk;
+      sqlite3BeginBenignMalloc();
+      if( sqlite3GlobalConfig.nPage>0 ){
+        szBulk = pCache->szAlloc * sqlite3GlobalConfig.nPage;
+      }else{
+        szBulk = -1024*sqlite3GlobalConfig.nPage;
+      }
+      zBulk = pCache->pBulk = sqlite3Malloc( szBulk );
+      sqlite3EndBenignMalloc();
+      if( zBulk ){
+        int nBulk = sqlite3MallocSize(zBulk)/pCache->szAlloc;
+        int i;
+        for(i=0; i<nBulk; i++){
+          PgHdr1 *pX = (PgHdr1*)&zBulk[szPage];
+          pX->page.pBuf = zBulk;
+          pX->page.pExtra = &pX[1];
+          pX->isBulkLocal = 1;
+          pX->pNext = pCache->pFree;
+          pCache->pFree = pX;
+          zBulk += pCache->szAlloc;
+        }
+      }
+    }
     if( pCache->nHash==0 ){
       pcache1Destroy((sqlite3_pcache*)pCache);
       pCache = 0;
@@ -753,15 +822,7 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2(
     pcache1RemoveFromHash(pPage, 0);
     pcache1PinPage(pPage);
     pOther = pPage->pCache;
-
-    /* We want to verify that szPage and szExtra are the same for pOther
-    ** and pCache.  Assert that we can verify this by comparing sums. */
-    assert( (pCache->szPage & (pCache->szPage-1))==0 && pCache->szPage>=512 );
-    assert( pCache->szExtra<512 );
-    assert( (pOther->szPage & (pOther->szPage-1))==0 && pOther->szPage>=512 );
-    assert( pOther->szExtra<512 );
-
-    if( pOther->szPage+pOther->szExtra != pCache->szPage+pCache->szExtra ){
+    if( pOther->szAlloc != pCache->szAlloc ){
       pcache1FreePage(pPage);
       pPage = 0;
     }else{
@@ -1037,6 +1098,7 @@ static void pcache1Destroy(sqlite3_pcache *p){
   pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage;
   pcache1EnforceMaxPage(pGroup);
   pcache1LeaveMutex(pGroup);
+  sqlite3_free(pCache->pBulk);
   sqlite3_free(pCache->apHash);
   sqlite3_free(pCache);
 }
@@ -1092,7 +1154,7 @@ int sqlite3PcacheReleaseMemory(int nReq){
   int nFree = 0;
   assert( sqlite3_mutex_notheld(pcache1.grp.mutex) );
   assert( sqlite3_mutex_notheld(pcache1.mutex) );
-  if( pcache1.pStart==0 ){
+  if( sqlite3GlobalConfig.nPage==0 ){
     PgHdr1 *p;
     pcache1EnterMutex(&pcache1.grp);
     while( (nReq<0 || nFree<nReq) && ((p=pcache1.grp.pLruTail)!=0) ){
diff --git a/test/speedtest1.c b/test/speedtest1.c
index caa2330bbd..9ac9fbb962 100644
--- a/test/speedtest1.c
+++ b/test/speedtest1.c
@@ -1177,6 +1177,7 @@ int main(int argc, char **argv){
   int noSync = 0;               /* True for --nosync */
   int pageSize = 0;             /* Desired page size.  0 means default */
   int nPCache = 0, szPCache = 0;/* --pcache configuration */
+  int doPCache = 0;             /* True if --pcache is seen */
   int nScratch = 0, szScratch=0;/* --scratch configuration */
   int showStats = 0;            /* True for --stats */
   int nThread = 0;              /* --threads value */
@@ -1251,6 +1252,7 @@ int main(int argc, char **argv){
         if( i>=argc-2 ) fatal_error("missing arguments on %s\n", argv[i]);
         nPCache = integerValue(argv[i+1]);
         szPCache = integerValue(argv[i+2]);
+        doPCache = 1;
         i += 2;
       }else if( strcmp(z,"primarykey")==0 ){
         g.zPK = "PRIMARY KEY";
@@ -1317,10 +1319,12 @@ int main(int argc, char **argv){
     rc = sqlite3_config(SQLITE_CONFIG_HEAP, pHeap, nHeap, mnHeap);
     if( rc ) fatal_error("heap configuration failed: %d\n", rc);
   }
-  if( nPCache>0 && szPCache>0 ){
-    pPCache = malloc( nPCache*(sqlite3_int64)szPCache );
-    if( pPCache==0 ) fatal_error("cannot allocate %lld-byte pcache\n",
-                                 nPCache*(sqlite3_int64)szPCache);
+  if( doPCache ){
+    if( nPCache>0 && szPCache>0 ){
+      pPCache = malloc( nPCache*(sqlite3_int64)szPCache );
+      if( pPCache==0 ) fatal_error("cannot allocate %lld-byte pcache\n",
+                                   nPCache*(sqlite3_int64)szPCache);
+    }
     rc = sqlite3_config(SQLITE_CONFIG_PAGECACHE, pPCache, szPCache, nPCache);
     if( rc ) fatal_error("pcache configuration failed: %d\n", rc);
   }

From 656773b6751d29e7d240cdc4f3e2345c4f101102 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 6 Jul 2015 20:27:19 +0000
Subject: [PATCH 21/40] Speed up seek operations on fts5 b-tree structures.

FossilOrigin-Name: 7b7da1eb435d321fc4283f6aa2161fa1e16f2cf3
---
 ext/fts5/fts5_index.c        | 174 +++++++++++++++++++++++++++++------
 ext/fts5/test/fts5aa.test    |  14 ++-
 ext/fts5/test/fts5ac.test    |   1 +
 ext/fts5/test/fts5ad.test    |   3 +-
 ext/fts5/test/fts5alter.test |  17 ++++
 manifest                     |  22 ++---
 manifest.uuid                |   2 +-
 7 files changed, 187 insertions(+), 46 deletions(-)

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index ae3ff35dbb..3ba3046e36 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -1580,6 +1580,23 @@ static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){
   }
 }
 
+static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){
+  u8 *a = pIter->pLeaf->p;        /* Buffer to read data from */
+  int iOff = pIter->iLeafOffset;
+
+  if( iOff>=pIter->pLeaf->n ){
+    fts5SegIterNextPage(p, pIter);
+    if( pIter->pLeaf==0 ){
+      if( p->rc==SQLITE_OK ) p->rc = FTS5_CORRUPT;
+      return;
+    }
+    iOff = 4;
+    a = pIter->pLeaf->p;
+  }
+  iOff += sqlite3Fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid);
+  pIter->iLeafOffset = iOff;
+}
+
 /*
 ** Fts5SegIter.iLeafOffset currently points to the first byte of the 
 ** "nSuffix" field of a term. Function parameter nKeep contains the value
@@ -1606,17 +1623,9 @@ static void fts5SegIterLoadTerm(Fts5Index *p, Fts5SegIter *pIter, int nKeep){
   iOff += nNew;
   pIter->iTermLeafOffset = iOff;
   pIter->iTermLeafPgno = pIter->iLeafPgno;
-  if( iOff>=pIter->pLeaf->n ){
-    fts5SegIterNextPage(p, pIter);
-    if( pIter->pLeaf==0 ){
-      if( p->rc==SQLITE_OK ) p->rc = FTS5_CORRUPT;
-      return;
-    }
-    iOff = 4;
-    a = pIter->pLeaf->p;
-  }
-  iOff += sqlite3Fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid);
   pIter->iLeafOffset = iOff;
+
+  fts5SegIterLoadRowid(p, pIter);
 }
 
 /*
@@ -2124,6 +2133,129 @@ static int fts5NodeSeek(
   return iPg;
 }
 
+/*
+** The iterator object passed as the second argument currently contains
+** no valid values except for the Fts5SegIter.pLeaf member variable. This
+** function searches the leaf page for a term matching (pTerm/nTerm).
+**
+*/
+static void fts5LeafSeek(
+  Fts5Index *p,                   /* Leave any error code here */
+  int bGe,                        /* True for a >= search */
+  Fts5SegIter *pIter,             /* Iterator to seek */
+  const u8 *pTerm, int nTerm      /* Term to search for */
+){
+  int iOff;
+  const u8 *a = pIter->pLeaf->p;
+  int n = pIter->pLeaf->n;
+
+  int nMatch = 0;
+  int nKeep = 0;
+  int nNew = 0;
+
+  assert( p->rc==SQLITE_OK );
+  assert( pIter->pLeaf );
+
+  iOff = fts5GetU16(&a[2]);
+  if( iOff<4 || iOff>=n ){
+    p->rc = FTS5_CORRUPT;
+    return;
+  }
+
+  while( 1 ){
+    int i;
+    int nCmp;
+    i64 rowid;
+
+    /* Figure out how many new bytes are in this term */
+
+    nNew = a[iOff++];
+    if( nNew & 0x80 ){ 
+      iOff--;
+      iOff += fts5GetVarint32(&a[iOff], nNew);
+    }
+
+    if( nKeep<nMatch ){
+      goto search_failed;
+    }
+
+    assert( nKeep>=nMatch );
+    if( nKeep==nMatch ){
+      nCmp = MIN(nNew, nTerm-nMatch);
+      for(i=0; i<nCmp; i++){
+        if( a[iOff+i]!=pTerm[nMatch+i] ) break;
+      }
+      nMatch += i;
+
+      if( nTerm==nMatch ){
+        if( i==nNew ){
+          goto search_success;
+        }else{
+          goto search_failed;
+        }
+      }else if( i<nNew && a[iOff+i]>pTerm[nMatch] ){
+        goto search_failed;
+      }
+    }
+    iOff += nNew;
+
+    /* Skip past the doclist. If the end of the page is reached, bail out. */
+    iOff += fts5GetVarint(&a[iOff], &rowid);
+    while( iOff<n ){
+      int nPos;
+
+      iOff += fts5GetVarint32(&a[iOff], nPos);
+      iOff += (nPos / 2);
+
+      /* Skip past docid delta */
+      iOff += fts5GetVarint(&a[iOff], &rowid);
+      if( rowid==0 ) break;
+    };
+    if( iOff>=n ) goto search_failed;
+
+    /* Read the nKeep field of the next term. */
+    nKeep = a[iOff++];
+    if( nKeep & 0x80 ){
+      iOff--;
+      iOff += fts5GetVarint32(&a[iOff], nKeep);
+    }
+  }
+
+ search_failed:
+  if( bGe==0 ){
+    fts5DataRelease(pIter->pLeaf);
+    pIter->pLeaf = 0;
+    return;
+  }else if( iOff>=n ){
+    do {
+      fts5SegIterNextPage(p, pIter);
+      if( pIter->pLeaf==0 ) return;
+      a = pIter->pLeaf->p;
+      iOff = fts5GetU16(&a[2]);
+      if( iOff ){
+        if( iOff<4 || iOff>=n ){
+          p->rc = FTS5_CORRUPT;
+        }else{
+          nKeep = 0;
+          iOff += fts5GetVarint32(&a[iOff], nNew);
+          break;
+        }
+      }
+    }while( 1 );
+  }
+
+ search_success:
+  pIter->iLeafOffset = iOff + nNew;
+  pIter->iTermLeafOffset = pIter->iLeafOffset;
+  pIter->iTermLeafPgno = pIter->iLeafPgno;
+
+  fts5BufferSet(&p->rc, &pIter->term, nKeep, pTerm);
+  fts5BufferAppendBlob(&p->rc, &pIter->term, nNew, &a[iOff]);
+
+  fts5SegIterLoadRowid(p, pIter);
+  fts5SegIterLoadNPos(p, pIter);
+}
+
 /*
 ** Initialize the object pIter to point to term pTerm/nTerm within segment
 ** pSeg. If there is no such term in the index, the iterator is set to EOF.
@@ -2168,26 +2300,8 @@ static void fts5SegIterSeekInit(
   pIter->iLeafPgno = iPg - 1;
   fts5SegIterNextPage(p, pIter);
 
-  if( (pLeaf = pIter->pLeaf) ){
-    int res;
-    pIter->iLeafOffset = fts5GetU16(&pLeaf->p[2]);
-    if( pIter->iLeafOffset<4 || pIter->iLeafOffset>=pLeaf->n ){
-      p->rc = FTS5_CORRUPT;
-    }else{
-      fts5SegIterLoadTerm(p, pIter, 0);
-      fts5SegIterLoadNPos(p, pIter);
-      do {
-        res = fts5BufferCompareBlob(&pIter->term, pTerm, nTerm);
-        if( res>=0 ) break;
-        fts5SegIterNext(p, pIter, 0);
-      }while( pIter->pLeaf && p->rc==SQLITE_OK );
-
-      if( bGe==0 && res ){
-        /* Set iterator to point to EOF */
-        fts5DataRelease(pIter->pLeaf);
-        pIter->pLeaf = 0;
-      }
-    }
+  if( pIter->pLeaf ){
+    fts5LeafSeek(p, bGe, pIter, pTerm, nTerm);
   }
 
   if( p->rc==SQLITE_OK && bGe==0 ){
diff --git a/ext/fts5/test/fts5aa.test b/ext/fts5/test/fts5aa.test
index 77ef19dd67..a18005cfdb 100644
--- a/ext/fts5/test/fts5aa.test
+++ b/ext/fts5/test/fts5aa.test
@@ -50,7 +50,17 @@ do_execsql_test 2.1 {
 do_test 2.2 {
   execsql { SELECT fts5_decode(id, block) FROM t1_data WHERE id==10 }
 } {/{\(structure\) {lvl=0 nMerge=0 {id=[0123456789]* h=1 leaves=1..1}}}/}
-do_execsql_test 2.3 {
+
+foreach w {a b c d e f} {
+  do_execsql_test 2.3.$w.asc {
+    SELECT rowid FROM t1 WHERE t1 MATCH $w;
+  } {1}
+  do_execsql_test 2.3.$w.desc {
+    SELECT rowid FROM t1 WHERE t1 MATCH $w ORDER BY rowid DESC;
+  } {1}
+}
+
+do_execsql_test 2.4 {
   INSERT INTO t1(t1) VALUES('integrity-check');
 }
 
@@ -190,8 +200,6 @@ for {set i 1} {$i <= 10} {incr i} {
     execsql { INSERT INTO t1(t1) VALUES('integrity-check'); }
   } {}
 }
-#db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r}
-#exit
 
 #-------------------------------------------------------------------------
 #
diff --git a/ext/fts5/test/fts5ac.test b/ext/fts5/test/fts5ac.test
index 8bc8accb9b..0de4848145 100644
--- a/ext/fts5/test/fts5ac.test
+++ b/ext/fts5/test/fts5ac.test
@@ -205,6 +205,7 @@ foreach {tn2 sql} {
     execsql { INSERT INTO xx(xx) VALUES('integrity-check') }
   } {}
 
+
   #-------------------------------------------------------------------------
   # Test phrase queries.
   #
diff --git a/ext/fts5/test/fts5ad.test b/ext/fts5/test/fts5ad.test
index 66ca1f1640..b998db05ab 100644
--- a/ext/fts5/test/fts5ad.test
+++ b/ext/fts5/test/fts5ad.test
@@ -204,10 +204,11 @@ foreach {T create} {
     }
     return $ret
   }
+
   
   foreach {bAsc sql} {
-    0 {SELECT rowid FROM t1 WHERE t1 MATCH $prefix ORDER BY rowid DESC}
     1 {SELECT rowid FROM t1 WHERE t1 MATCH $prefix}
+    0 {SELECT rowid FROM t1 WHERE t1 MATCH $prefix ORDER BY rowid DESC}
   } {
     foreach {tn prefix} {
       1  {a*} 2 {ab*} 3 {abc*} 4 {abcd*} 5 {abcde*} 
diff --git a/ext/fts5/test/fts5alter.test b/ext/fts5/test/fts5alter.test
index 0ed788b8a9..eae01b7386 100644
--- a/ext/fts5/test/fts5alter.test
+++ b/ext/fts5/test/fts5alter.test
@@ -81,6 +81,23 @@ do_execsql_test 2.3 {
   SELECT rowid FROM yy WHERE yy MATCH 'a + b + c';
 } {-56 -22}
 
+#-------------------------------------------------------------------------
+
+do_execsql_test 3.1 {
+  CREATE VIRTUAL TABLE abc USING fts5(a);
+  INSERT INTO abc(rowid, a) VALUES(1, 'a');
+  BEGIN;
+    INSERT INTO abc(rowid, a) VALUES(2, 'a');
+}
+breakpoint
+do_execsql_test 3.2 {
+    SELECT rowid FROM abc WHERE abc MATCH 'a';
+} {1 2}
+
+do_execsql_test 3.3 {
+  COMMIT;
+  SELECT rowid FROM abc WHERE abc MATCH 'a';
+} {1 2}
 
 finish_test
 
diff --git a/manifest b/manifest
index 1954e6998c..6b86af9d85 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Do\snot\sallow\srecursive\sCTEs\sthat\suse\saggregate\squeries\sin\sthe\srecursive\spart.
-D 2015-07-05T22:15:10.026
+C Speed\sup\sseek\soperations\son\sfts5\sb-tree\sstructures.
+D 2015-07-06T20:27:19.997
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 017bf0511d1b2dd1db5e16488fbf75a17b526cbc
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -112,7 +112,7 @@ F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015
 F ext/fts5/fts5_config.c b2456e9625bca41c51d54c363e369c6356895c90
 F ext/fts5/fts5_expr.c d2e148345639c5a5583e0daa39a639bf298ae6a7
 F ext/fts5/fts5_hash.c 219f4edd72e5cf95b19c33f1058809a18fad5229
-F ext/fts5/fts5_index.c 6224454702f852392cae8755c920f93b06b09283
+F ext/fts5/fts5_index.c d81cada8367eae5e5482860ccb6ae574eee3d74a
 F ext/fts5/fts5_main.c 37b0055cb4036c4b4bb4eb36e30ebd1c21c63939
 F ext/fts5/fts5_storage.c 4cae85b5287b159d9d98174a4e70adf872b0930a
 F ext/fts5/fts5_tcl.c 85eb4e0d0fefa9420b78151496ad4599a1783e20
@@ -123,10 +123,10 @@ F ext/fts5/fts5_vocab.c 4e268a3fcbc099e50e335a1135be985a41ff6f7f
 F ext/fts5/fts5parse.y 833db1101b78c0c47686ab1b84918e38c36e9452
 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
 F ext/fts5/test/fts5_common.tcl 9553cce0757092d194307c2168d4edd100eab578
-F ext/fts5/test/fts5aa.test 0be21c89fd66b588db355a6398911fd875bdcc6c
+F ext/fts5/test/fts5aa.test f7f057811eb0113d3259e059218bc85cef444280
 F ext/fts5/test/fts5ab.test 6fe3a56731d15978afbb74ae51b355fc9310f2ad
-F ext/fts5/test/fts5ac.test 0990ae7497ebaea2ab5f7fd5caedd93a71a905fc
-F ext/fts5/test/fts5ad.test 312f3c8ed9592533499c5b94d2059ae6382913a0
+F ext/fts5/test/fts5ac.test 9737992d08c56bfd4803e933744d2d764e23795c
+F ext/fts5/test/fts5ad.test b2edee8b7de0c21d2c88f8a18c195034aad6952d
 F ext/fts5/test/fts5ae.test ddc558e3e3b52db0101f7541b2e3849b77052c92
 F ext/fts5/test/fts5af.test c2501ec2b61d6b179c305f5d2b8782ab3d4f832a
 F ext/fts5/test/fts5ag.test ec3e119b728196620a31507ef503c455a7a73505
@@ -135,7 +135,7 @@ F ext/fts5/test/fts5ai.test f20e53bbf0c55bc596f1fd47f2740dae028b8f37
 F ext/fts5/test/fts5aj.test 05b569f5c16ea3098fb1984eec5cf50dbdaae5d8
 F ext/fts5/test/fts5ak.test 7b8c5df96df599293f920b7e5521ebc79f647592
 F ext/fts5/test/fts5al.test fc60ebeac9d8e366e71309d4c31fa72199d711d7
-F ext/fts5/test/fts5alter.test 78b63e088646dd623cacbdc1899a54d638dcf3d8
+F ext/fts5/test/fts5alter.test 6022c61467a82aa11c70822ccad22b328dcf0d04
 F ext/fts5/test/fts5auto.test caa5bcf917db11944655a2a9bd38c67c520376ca
 F ext/fts5/test/fts5aux.test 8c687c948cc98e9a94be014df7d518acc1b3b74f
 F ext/fts5/test/fts5auxdata.test 141a7cbffcceb1bd2799b4b29c183ff8780d586e
@@ -1364,7 +1364,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 8cf02090ce53ec150492d77d9e5e5f27665bd34f
-R 2ced95f329e3743e78bec1f4f82122ca
-U drh
-Z 74748fde014fce363e747a5eecd57396
+P 6d2999afbc25b9c238e4028f637c10eaaf0ec75e
+R d5f5e61506b7123fb097865134e2b1bc
+U dan
+Z 4783766915d09d11d606d5d129590086
diff --git a/manifest.uuid b/manifest.uuid
index 0975b167fb..94b18735e0 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6d2999afbc25b9c238e4028f637c10eaaf0ec75e
\ No newline at end of file
+7b7da1eb435d321fc4283f6aa2161fa1e16f2cf3
\ No newline at end of file

From db7ae89d5ba3a97cf23a8600d21e6d181eef8210 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 6 Jul 2015 20:57:22 +0000
Subject: [PATCH 22/40] A unified cache is now only used for
 SQLITE_ENABLE_MEMORY_MANAGEMENT, or if SQLITE_CONFIG_PAGECACHE defines a
 shared start-time page cache buffer and the application is single-threaded.

FossilOrigin-Name: d5ff1d69054d51a55eff6e1979971822db7f8f30
---
 manifest      | 15 ++++-----
 manifest.uuid |  2 +-
 src/pcache1.c | 91 +++++++++++++++++++++++++++------------------------
 3 files changed, 56 insertions(+), 52 deletions(-)

diff --git a/manifest b/manifest
index c2e77a579b..4622da25d7 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Enhance\sseparate\spcache1\sto\sallocate\sa\sblock\sof\spages\sfrom\sheap\son\sstartup,\nif\spossible,\sfor\sa\s5.2%\sperformance\simprovement.
-D 2015-07-06T18:54:52.424
+C A\sunified\scache\sis\snow\sonly\sused\sfor\sSQLITE_ENABLE_MEMORY_MANAGEMENT,\sor\sif\nSQLITE_CONFIG_PAGECACHE\sdefines\sa\sshared\sstart-time\spage\scache\sbuffer\sand\sthe\napplication\sis\ssingle-threaded.
+D 2015-07-06T20:57:22.615
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 017bf0511d1b2dd1db5e16488fbf75a17b526cbc
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -319,7 +319,7 @@ F src/pager.h 6d435f563b3f7fcae4b84433b76a6ac2730036e2
 F src/parse.y 6d60dda8f8d418b6dc034f1fbccd816c459983a8
 F src/pcache.c cde06aa50962595e412d497e22fd2e07878ba1f0
 F src/pcache.h 9968603796240cdf83da7e7bef76edf90619cea9
-F src/pcache1.c 28f3434788cac24bc95f8c1af4a61cbde179ef18
+F src/pcache1.c 17c392b4208ea6e1e5d220be18bcfebac07110b1
 F src/pragma.c c1f4d012ea9f6b1ce52d341b2cd0ad72d560afd7
 F src/pragma.h b8632d7cdda7b25323fa580e3e558a4f0d4502cc
 F src/prepare.c 82e5db1013846a819f198336fed72c44c974e7b1
@@ -1364,10 +1364,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 6d2999afbc25b9c238e4028f637c10eaaf0ec75e
-R 773f110b2d424984f83126d1917fdcbb
-T *branch * pcache-bulk-local
-T *sym-pcache-bulk-local *
-T -sym-trunk *
+P aa7341c8736732a0a59f6688cc34e78be02a7bfc
+R ab876e072a5f820e07c8ea2cb7e263ca
 U drh
-Z a5a615d5b6a74570375077a92838726e
+Z d6cf6ae811ec210c263fdd6a92d21a0f
diff --git a/manifest.uuid b/manifest.uuid
index 29a53816d9..d027464ae1 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-aa7341c8736732a0a59f6688cc34e78be02a7bfc
\ No newline at end of file
+d5ff1d69054d51a55eff6e1979971822db7f8f30
\ No newline at end of file
diff --git a/src/pcache1.c b/src/pcache1.c
index 30ea7d0d8b..c34d0bba1c 100644
--- a/src/pcache1.c
+++ b/src/pcache1.c
@@ -189,6 +189,7 @@ static SQLITE_WSD struct PCacheGlobal {
   ** The nFreeSlot and pFree values do require mutex protection.
   */
   int isInit;                    /* True if initialized */
+  int separateCache;             /* Use a new PGroup for each PCache */
   int szSlot;                    /* Size of each free slot */
   int nSlot;                     /* The number of pcache slots */
   int nReserve;                  /* Try to keep nFreeSlot above this */
@@ -362,10 +363,14 @@ static PgHdr1 *pcache1AllocPage(PCache1 *pCache){
     pCache->pFree = p->pNext;
     p->pNext = 0;
   }else{
+#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
     /* The group mutex must be released before pcache1Alloc() is called. This
-    ** is because it may call sqlite3_release_memory(), which assumes that 
+    ** is because it might call sqlite3_release_memory(), which assumes that 
     ** this mutex is not held. */
+    assert( pcache1.separateCache==0 );
+    assert( pCache->pGroup==&pcache1.grp );
     pcache1LeaveMutex(pCache->pGroup);
+#endif
 #ifdef SQLITE_PCACHE_SEPARATE_HEADER
     pPg = pcache1Alloc(pCache->szPage);
     p = sqlite3Malloc(sizeof(PgHdr1) + pCache->szExtra);
@@ -378,7 +383,9 @@ static PgHdr1 *pcache1AllocPage(PCache1 *pCache){
     pPg = pcache1Alloc(pCache->szAlloc);
     p = (PgHdr1 *)&((u8 *)pPg)[pCache->szPage];
 #endif
+#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
     pcache1EnterMutex(pCache->pGroup);
+#endif
     if( pPg==0 ) return 0;
     p->page.pBuf = pPg;
     p->page.pExtra = &p[1];
@@ -392,27 +399,23 @@ static PgHdr1 *pcache1AllocPage(PCache1 *pCache){
 
 /*
 ** Free a page object allocated by pcache1AllocPage().
-**
-** The pointer is allowed to be NULL, which is prudent.  But it turns out
-** that the current implementation happens to never call this routine
-** with a NULL pointer, so we mark the NULL test with ALWAYS().
 */
 static void pcache1FreePage(PgHdr1 *p){
-  if( ALWAYS(p) ){
-    PCache1 *pCache = p->pCache;
-    assert( sqlite3_mutex_held(p->pCache->pGroup->mutex) );
-    if( p->isBulkLocal ){
-      p->pNext = pCache->pFree;
-      pCache->pFree = p;
-    }else{
-      pcache1Free(p->page.pBuf);
+  PCache1 *pCache;
+  assert( p!=0 );
+  pCache = p->pCache;
+  assert( sqlite3_mutex_held(p->pCache->pGroup->mutex) );
+  if( p->isBulkLocal ){
+    p->pNext = pCache->pFree;
+    pCache->pFree = p;
+  }else{
+    pcache1Free(p->page.pBuf);
 #ifdef SQLITE_PCACHE_SEPARATE_HEADER
-      sqlite3_free(p);
+    sqlite3_free(p);
 #endif
-    }
-    if( pCache->bPurgeable ){
-      pCache->pGroup->nCurrentPage--;
-    }
+  }
+  if( pCache->bPurgeable ){
+    pCache->pGroup->nCurrentPage--;
   }
 }
 
@@ -612,6 +615,29 @@ static int pcache1Init(void *NotUsed){
   UNUSED_PARAMETER(NotUsed);
   assert( pcache1.isInit==0 );
   memset(&pcache1, 0, sizeof(pcache1));
+
+
+  /*
+  ** The pcache1.separateCache variable is true if each PCache has its own
+  ** private PGroup (mode-1).  pcache1.separateCache is false if the single
+  ** PGroup in pcache1.grp is used for all page caches (mode-2).
+  **
+  **   *  Always use a unified cache (mode-2) if ENABLE_MEMORY_MANAGEMENT
+  **
+  **   *  Use a unified cache in single-threaded applications that have
+  **      configured a start-time buffer for use as page-cache memory using
+  **      sqlite3_config(SQLITE_CONFIG_PAGECACHE, pBuf, sz, N) with non-NULL 
+  **      pBuf argument.
+  **
+  **   *  Otherwise use separate caches (mode-1)
+  */
+#if defined(SQLITE_ENABLE_MEMORY_MANAGEMENT)
+  pcache1.separateCache = 0;
+#else
+  pcache1.separateCache = sqlite3GlobalConfig.pPage==0
+                          || sqlite3GlobalConfig.bCoreMutex>0;
+#endif
+
 #if SQLITE_THREADSAFE
   if( sqlite3GlobalConfig.bCoreMutex ){
     pcache1.grp.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU);
@@ -647,31 +673,13 @@ static sqlite3_pcache *pcache1Create(int szPage, int szExtra, int bPurgeable){
   PGroup *pGroup;       /* The group the new page cache will belong to */
   int sz;               /* Bytes of memory required to allocate the new cache */
 
-  /*
-  ** The separateCache variable is true if each PCache has its own private
-  ** PGroup.  In other words, separateCache is true for mode (1) where no
-  ** mutexing is required.
-  **
-  **   *  Always use a unified cache (mode-2) if ENABLE_MEMORY_MANAGEMENT
-  **
-  **   *  Always use a unified cache in single-threaded applications
-  **
-  **   *  Otherwise (if multi-threaded and ENABLE_MEMORY_MANAGEMENT is off)
-  **      use separate caches (mode-1)
-  */
-#if defined(SQLITE_ENABLE_MEMORY_MANAGEMENT) || SQLITE_THREADSAFE==0
-  const int separateCache = 0;
-#else
-  int separateCache = sqlite3GlobalConfig.bCoreMutex>0;
-#endif
-
   assert( (szPage & (szPage-1))==0 && szPage>=512 && szPage<=65536 );
   assert( szExtra < 300 );
 
-  sz = sizeof(PCache1) + sizeof(PGroup)*separateCache;
+  sz = sizeof(PCache1) + sizeof(PGroup)*pcache1.separateCache;
   pCache = (PCache1 *)sqlite3MallocZero(sz);
   if( pCache ){
-    if( separateCache ){
+    if( pcache1.separateCache ){
       pGroup = (PGroup*)&pCache[1];
       pGroup->mxPinned = 10;
     }else{
@@ -690,10 +698,9 @@ static sqlite3_pcache *pcache1Create(int szPage, int szExtra, int bPurgeable){
       pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage;
     }
     pcache1LeaveMutex(pGroup);
-    if( separateCache
-     && sqlite3GlobalConfig.szPage==0
-     && sqlite3GlobalConfig.nPage!=0
-    ){
+    /* Try to initialize the local bulk pagecache line allocation if using
+    ** separate caches and if nPage!=0 */
+    if( pcache1.separateCache && sqlite3GlobalConfig.nPage!=0 ){
       int szBulk;
       char *zBulk;
       sqlite3BeginBenignMalloc();

From b2fe306a2e7bce2f50182e5bb8fb52c4bad78b65 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 7 Jul 2015 08:29:32 +0000
Subject: [PATCH 23/40] Further optimizations for fts5 b-tree seeks.

FossilOrigin-Name: f37899686c032145f431f81c1072738536c61c7e
---
 ext/fts5/fts5_index.c | 55 ++++++++++++++++++++++++++++---------------
 manifest              | 12 +++++-----
 manifest.uuid         |  2 +-
 3 files changed, 43 insertions(+), 26 deletions(-)

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 3ba3046e36..861d300c2b 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -2133,11 +2133,32 @@ static int fts5NodeSeek(
   return iPg;
 }
 
+#define fts5IndexGetVarint32(a, iOff, nVal) {     \
+  nVal = a[iOff++];                               \
+  if( nVal & 0x80 ){                              \
+    iOff--;                                       \
+    iOff += fts5GetVarint32(&a[iOff], nVal);      \
+  }                                               \
+}
+
+#define fts5IndexSkipVarint(a, iOff) {            \
+  int iEnd = iOff+9;                              \
+  while( (a[iOff++] & 0x80) && iOff<iEnd );       \
+}
+
 /*
 ** The iterator object passed as the second argument currently contains
 ** no valid values except for the Fts5SegIter.pLeaf member variable. This
 ** function searches the leaf page for a term matching (pTerm/nTerm).
 **
+** If the specified term is found on the page, then the iterator is left
+** pointing to it. If argument bGe is zero and the term is not found,
+** the iterator is left pointing at EOF.
+**
+** If bGe is non-zero and the specified term is not found, then the
+** iterator is left pointing to the smallest term in the segment that
+** is larger than the specified term, even if this term is not on the
+** current page.
 */
 static void fts5LeafSeek(
   Fts5Index *p,                   /* Leave any error code here */
@@ -2168,12 +2189,7 @@ static void fts5LeafSeek(
     i64 rowid;
 
     /* Figure out how many new bytes are in this term */
-
-    nNew = a[iOff++];
-    if( nNew & 0x80 ){ 
-      iOff--;
-      iOff += fts5GetVarint32(&a[iOff], nNew);
-    }
+    fts5IndexGetVarint32(a, iOff, nNew);
 
     if( nKeep<nMatch ){
       goto search_failed;
@@ -2200,25 +2216,26 @@ static void fts5LeafSeek(
     iOff += nNew;
 
     /* Skip past the doclist. If the end of the page is reached, bail out. */
-    iOff += fts5GetVarint(&a[iOff], &rowid);
-    while( iOff<n ){
+    while( 1 ){
       int nPos;
 
-      iOff += fts5GetVarint32(&a[iOff], nPos);
-      iOff += (nPos / 2);
-
       /* Skip past docid delta */
-      iOff += fts5GetVarint(&a[iOff], &rowid);
-      if( rowid==0 ) break;
+      fts5IndexSkipVarint(a, iOff);
+
+      /* Skip past position list */
+      fts5IndexGetVarint32(a, iOff, nPos);
+      iOff += (nPos >> 1);
+      if( iOff>=n ) goto search_failed;
+
+      /* If this is the end of the doclist, break out of the loop */
+      if( a[iOff]==0x00 ){
+        iOff++;
+        break;
+      }
     };
-    if( iOff>=n ) goto search_failed;
 
     /* Read the nKeep field of the next term. */
-    nKeep = a[iOff++];
-    if( nKeep & 0x80 ){
-      iOff--;
-      iOff += fts5GetVarint32(&a[iOff], nKeep);
-    }
+    fts5IndexGetVarint32(a, iOff, nKeep);
   }
 
  search_failed:
diff --git a/manifest b/manifest
index 6b86af9d85..d8e9e48dae 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Speed\sup\sseek\soperations\son\sfts5\sb-tree\sstructures.
-D 2015-07-06T20:27:19.997
+C Further\soptimizations\sfor\sfts5\sb-tree\sseeks.
+D 2015-07-07T08:29:32.749
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 017bf0511d1b2dd1db5e16488fbf75a17b526cbc
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -112,7 +112,7 @@ F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015
 F ext/fts5/fts5_config.c b2456e9625bca41c51d54c363e369c6356895c90
 F ext/fts5/fts5_expr.c d2e148345639c5a5583e0daa39a639bf298ae6a7
 F ext/fts5/fts5_hash.c 219f4edd72e5cf95b19c33f1058809a18fad5229
-F ext/fts5/fts5_index.c d81cada8367eae5e5482860ccb6ae574eee3d74a
+F ext/fts5/fts5_index.c 19f6db2a43c0126267c7fa11e3ed41105eb89e2f
 F ext/fts5/fts5_main.c 37b0055cb4036c4b4bb4eb36e30ebd1c21c63939
 F ext/fts5/fts5_storage.c 4cae85b5287b159d9d98174a4e70adf872b0930a
 F ext/fts5/fts5_tcl.c 85eb4e0d0fefa9420b78151496ad4599a1783e20
@@ -1364,7 +1364,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 6d2999afbc25b9c238e4028f637c10eaaf0ec75e
-R d5f5e61506b7123fb097865134e2b1bc
+P 7b7da1eb435d321fc4283f6aa2161fa1e16f2cf3
+R 24874c7422a967bb097e1a3642b88228
 U dan
-Z 4783766915d09d11d606d5d129590086
+Z f8ef7d272cb32b051f2a236ea6d3cc77
diff --git a/manifest.uuid b/manifest.uuid
index 94b18735e0..c950998394 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-7b7da1eb435d321fc4283f6aa2161fa1e16f2cf3
\ No newline at end of file
+f37899686c032145f431f81c1072738536c61c7e
\ No newline at end of file

From c54357cc2ae6ff5526f3dd38c1f28fb633a36117 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 7 Jul 2015 14:06:18 +0000
Subject: [PATCH 24/40] Simplify the page recycling condition in pcache1.  Fix
 test cases to work with the latest changes.

FossilOrigin-Name: 5b60df5503f61922326b3e10e94aa197b17afe4b
---
 manifest             | 18 +++++++++---------
 manifest.uuid        |  2 +-
 src/pcache1.c        |  9 ++++-----
 test/memdb.test      |  7 +++----
 test/memsubsys1.test |  4 +++-
 test/pcache2.test    |  2 ++
 6 files changed, 22 insertions(+), 20 deletions(-)

diff --git a/manifest b/manifest
index 4622da25d7..bd367c068c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C A\sunified\scache\sis\snow\sonly\sused\sfor\sSQLITE_ENABLE_MEMORY_MANAGEMENT,\sor\sif\nSQLITE_CONFIG_PAGECACHE\sdefines\sa\sshared\sstart-time\spage\scache\sbuffer\sand\sthe\napplication\sis\ssingle-threaded.
-D 2015-07-06T20:57:22.615
+C Simplify\sthe\spage\srecycling\scondition\sin\spcache1.\s\sFix\stest\scases\sto\swork\nwith\sthe\slatest\schanges.
+D 2015-07-07T14:06:18.833
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 017bf0511d1b2dd1db5e16488fbf75a17b526cbc
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -319,7 +319,7 @@ F src/pager.h 6d435f563b3f7fcae4b84433b76a6ac2730036e2
 F src/parse.y 6d60dda8f8d418b6dc034f1fbccd816c459983a8
 F src/pcache.c cde06aa50962595e412d497e22fd2e07878ba1f0
 F src/pcache.h 9968603796240cdf83da7e7bef76edf90619cea9
-F src/pcache1.c 17c392b4208ea6e1e5d220be18bcfebac07110b1
+F src/pcache1.c 1b6bc285d8d6a1a7df41e345cb45914093aecfdf
 F src/pragma.c c1f4d012ea9f6b1ce52d341b2cd0ad72d560afd7
 F src/pragma.h b8632d7cdda7b25323fa580e3e558a4f0d4502cc
 F src/prepare.c 82e5db1013846a819f198336fed72c44c974e7b1
@@ -843,9 +843,9 @@ F test/mallocL.test 252ddc7eb4fbf75364eab17b938816085ff1fc17
 F test/malloc_common.tcl aac62499b76be719fac31e7a3e54a7fd53272e7f
 F test/manydb.test 28385ae2087967aa05c38624cec7d96ec74feb3e
 F test/mem5.test c6460fba403c5703141348cd90de1c294188c68f
-F test/memdb.test fcb5297b321b562084fc79d64d5a12a1cd2b639b
+F test/memdb.test c1f2a343ad14398d5d6debda6ea33e80d0dafcc7
 F test/memleak.test 10b9c6c57e19fc68c32941495e9ba1c50123f6e2
-F test/memsubsys1.test 690d142525a7d31efb638b0d232e25fac3afeb1a
+F test/memsubsys1.test 1733c617e246642db5bf3b9f78c18e2b14fac96c
 F test/memsubsys2.test 3a1c1a9de48e5726faa85108b02459fae8cb9ee9
 F test/minmax.test 42fbad0e81afaa6e0de41c960329f2b2c3526efd
 F test/minmax2.test b44bae787fc7b227597b01b0ca5575c7cb54d3bc
@@ -900,7 +900,7 @@ F test/pagerfault3.test 1003fcda009bf48a8e22a516e193b6ef0dd1bbd8
 F test/pageropt.test 6b8f6a123a5572c195ad4ae40f2987007923bbd6
 F test/pagesize.test 5769fc62d8c890a83a503f67d47508dfdc543305
 F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d
-F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025
+F test/pcache2.test ec3ae192f444ee6a0a80d1fd80d99695d884bfb3
 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff
 F test/permutations.test 6a88fd9ca15b804e9c20990773262ca67494058f
 F test/pragma.test be7195f0aa72bdb8a512133e9640ac40f15b57a2
@@ -1364,7 +1364,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P aa7341c8736732a0a59f6688cc34e78be02a7bfc
-R ab876e072a5f820e07c8ea2cb7e263ca
+P d5ff1d69054d51a55eff6e1979971822db7f8f30
+R 968aaca5a1f6e381bde3c0ae75591e74
 U drh
-Z d6cf6ae811ec210c263fdd6a92d21a0f
+Z 609b7f68c8ef0f31b06857cf6a1355d8
diff --git a/manifest.uuid b/manifest.uuid
index d027464ae1..68d4259bf2 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d5ff1d69054d51a55eff6e1979971822db7f8f30
\ No newline at end of file
+5b60df5503f61922326b3e10e94aa197b17afe4b
\ No newline at end of file
diff --git a/src/pcache1.c b/src/pcache1.c
index c34d0bba1c..71c5971c3a 100644
--- a/src/pcache1.c
+++ b/src/pcache1.c
@@ -818,11 +818,10 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2(
   assert( pCache->nHash>0 && pCache->apHash );
 
   /* Step 4. Try to recycle a page. */
-  if( pCache->bPurgeable && pGroup->pLruTail && (
-         (pCache->nPage+1>=pCache->nMax)
-      || pGroup->nCurrentPage>=pGroup->nMaxPage
-      || pcache1UnderMemoryPressure(pCache)
-  )){
+  if( pCache->bPurgeable
+   && pGroup->pLruTail
+   && ((pCache->nPage+1>=pCache->nMax) || pcache1UnderMemoryPressure(pCache))
+  ){
     PCache1 *pOther;
     pPage = pGroup->pLruTail;
     assert( pPage->isPinned==0 );
diff --git a/test/memdb.test b/test/memdb.test
index a2efc0369b..7bad26f526 100644
--- a/test/memdb.test
+++ b/test/memdb.test
@@ -419,11 +419,10 @@ ifcapable autovacuum {
       INSERT INTO t1 VALUES(randstr(1000,1000));
       INSERT INTO t1 VALUES(randstr(1000,1000));
     }
-    set memused [lindex [sqlite3_status SQLITE_STATUS_MEMORY_USED 0] 1]
-    set pgovfl [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_OVERFLOW 0] 1]
+    set before [db one {PRAGMA page_count}]
     execsql { DELETE FROM t1 }
-    set memused2 [lindex [sqlite3_status SQLITE_STATUS_MEMORY_USED 0] 1]
-    expr {($memused2 + 2048 < $memused) || $pgovfl==0}
+    set after [db one {PRAGMA page_count}]
+    expr {$before>$after}
   } {1}
 }
 
diff --git a/test/memsubsys1.test b/test/memsubsys1.test
index 891558fd52..3eb3cd4a95 100644
--- a/test/memsubsys1.test
+++ b/test/memsubsys1.test
@@ -75,6 +75,7 @@ set xtra_size 290
 db close
 sqlite3_shutdown
 sqlite3_config_lookaside 0 0
+sqlite3_config_pagecache 0 0
 sqlite3_initialize
 reset_highwater_marks
 build_test_db memsubsys1-1 {PRAGMA page_size=1024}
@@ -115,10 +116,10 @@ do_test memsubsys1-2.5 {
 db close
 sqlite3_shutdown
 sqlite3_config_pagecache [expr 512+$xtra_size] 20
+sqlite3_config singlethread
 sqlite3_initialize
 reset_highwater_marks
 build_test_db memsubsys1-3.1 {PRAGMA page_size=1024}
-#show_memstats
 do_test memsubsys1-3.1.3 {
   set pg_used [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0] 2]
 } 0
@@ -312,6 +313,7 @@ sqlite3_config_memstatus 1
 sqlite3_config_pagecache 0 0
 sqlite3_config_scratch 0 0
 sqlite3_config_lookaside 100 500
+sqlite3_config serialized
 sqlite3_initialize
 autoinstall_test_functions
 finish_test
diff --git a/test/pcache2.test b/test/pcache2.test
index 77e7a26132..1c15b8b56a 100644
--- a/test/pcache2.test
+++ b/test/pcache2.test
@@ -24,6 +24,7 @@ do_test pcache2-1.1 {
   sqlite3_reset_auto_extension
   sqlite3_shutdown
   sqlite3_config_pagecache 6000 100
+  sqlite3_config singlethread
   sqlite3_initialize
   autoinstall_test_functions
   sqlite3_status SQLITE_STATUS_PAGECACHE_USED 1
@@ -73,6 +74,7 @@ catch {db2 close}
 sqlite3_reset_auto_extension
 sqlite3_shutdown
 sqlite3_config_pagecache 0 0
+sqlite3_config serialized
 sqlite3_initialize
 autoinstall_test_functions
 

From fd5ae9649c5293d7f05fe8b6b6038663987ee454 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 7 Jul 2015 15:14:16 +0000
Subject: [PATCH 25/40] No need to test sqlite3GlobalConfig.bCoreMutex when
 SQLITE_THREADSAFE==0.

FossilOrigin-Name: 26db0fee06947d75621237a641b9652b312b3b37
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/pcache1.c |  4 +++-
 3 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index bd367c068c..ebdaa318e2 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Simplify\sthe\spage\srecycling\scondition\sin\spcache1.\s\sFix\stest\scases\sto\swork\nwith\sthe\slatest\schanges.
-D 2015-07-07T14:06:18.833
+C No\sneed\sto\stest\ssqlite3GlobalConfig.bCoreMutex\swhen\sSQLITE_THREADSAFE==0.
+D 2015-07-07T15:14:16.002
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 017bf0511d1b2dd1db5e16488fbf75a17b526cbc
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -319,7 +319,7 @@ F src/pager.h 6d435f563b3f7fcae4b84433b76a6ac2730036e2
 F src/parse.y 6d60dda8f8d418b6dc034f1fbccd816c459983a8
 F src/pcache.c cde06aa50962595e412d497e22fd2e07878ba1f0
 F src/pcache.h 9968603796240cdf83da7e7bef76edf90619cea9
-F src/pcache1.c 1b6bc285d8d6a1a7df41e345cb45914093aecfdf
+F src/pcache1.c e3b0de837c9af01335e7e59fa8290dfa9338b2a3
 F src/pragma.c c1f4d012ea9f6b1ce52d341b2cd0ad72d560afd7
 F src/pragma.h b8632d7cdda7b25323fa580e3e558a4f0d4502cc
 F src/prepare.c 82e5db1013846a819f198336fed72c44c974e7b1
@@ -1364,7 +1364,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P d5ff1d69054d51a55eff6e1979971822db7f8f30
-R 968aaca5a1f6e381bde3c0ae75591e74
+P 5b60df5503f61922326b3e10e94aa197b17afe4b
+R 2d7b920ac2e4dfd8e6b212c060ee3e55
 U drh
-Z 609b7f68c8ef0f31b06857cf6a1355d8
+Z 96d25228913b7734acfab488bf16d902
diff --git a/manifest.uuid b/manifest.uuid
index 68d4259bf2..cd9a67d080 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5b60df5503f61922326b3e10e94aa197b17afe4b
\ No newline at end of file
+26db0fee06947d75621237a641b9652b312b3b37
\ No newline at end of file
diff --git a/src/pcache1.c b/src/pcache1.c
index 71c5971c3a..90989bc1db 100644
--- a/src/pcache1.c
+++ b/src/pcache1.c
@@ -633,9 +633,11 @@ static int pcache1Init(void *NotUsed){
   */
 #if defined(SQLITE_ENABLE_MEMORY_MANAGEMENT)
   pcache1.separateCache = 0;
-#else
+#elif SQLITE_THREADSAFE
   pcache1.separateCache = sqlite3GlobalConfig.pPage==0
                           || sqlite3GlobalConfig.bCoreMutex>0;
+#else
+  pcache1.separateCache = sqlite3GlobalConfig.pPage==0;
 #endif
 
 #if SQLITE_THREADSAFE

From 01233d3e5437fe334388cb99268eb58178d3c186 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 7 Jul 2015 19:07:30 +0000
Subject: [PATCH 26/40] Add a test case to verify that "PRAGMA data_version"
 works as expected when an OTA client writes to the database.

FossilOrigin-Name: 6441f2dc9e7f53de64bc15b1d766c885b2635112
---
 ext/ota/ota12.test | 63 ++++++++++++++++++++++++++++++++++++++++++++++
 manifest           | 12 ++++-----
 manifest.uuid      |  2 +-
 3 files changed, 70 insertions(+), 7 deletions(-)

diff --git a/ext/ota/ota12.test b/ext/ota/ota12.test
index 844b541671..b97f10653e 100644
--- a/ext/ota/ota12.test
+++ b/ext/ota/ota12.test
@@ -168,5 +168,68 @@ do_multiclient_test tn {
 
 }
 
+#-------------------------------------------------------------------------
+# Test that "PRAGMA data_version" works when an OTA client writes the
+# database.
+#
+do_multiclient_test tn {
+
+  # Initialize a target (test.db) and ota (ota.db) database.
+  #
+  forcedelete ota.db
+  sql1 $setup_sql
+
+  # Check the initial database contains table "xx" with a single row.
+  # Also save the current values of "PRAGMA data-version" for [db1] 
+  # and [db2].
+  #
+  do_test 2.$tn.1 {
+    list [sql1 { SELECT count(*) FROM xx }] [sql2 { SELECT count(*) FROM xx }]
+  } {1 1}
+  set V1 [sql1 {PRAGMA data_version}] 
+  set V2 [sql2 {PRAGMA data_version}]
+
+  # Check the values of data-version have not magically changed.
+  #
+  do_test 2.$tn.2 {
+    list [sql1 {PRAGMA data_version}] [sql2 {PRAGMA data_version}]
+  } [list $V1 $V2]
+
+  # Start stepping the OTA. From the point of view of [db1] and [db2], the 
+  # data-version values remain unchanged until the database contents are
+  # modified. At which point the values are incremented.
+  #
+  sqlite3ota ota test.db ota.db
+  set x 0
+  while {[db one {SELECT count(*) FROM xx}]==1} {
+    do_test 2.$tn.3.[incr x] {
+      list [sql1 {PRAGMA data_version}] [sql2 {PRAGMA data_version}]
+    } [list $V1 $V2]
+    ota step
+  }
+  do_test 2.$tn.5.1 { expr {$V1 < [sql1 {PRAGMA data_version}]} } 1
+  do_test 2.$tn.5.2 { expr {$V2 < [sql2 {PRAGMA data_version}]} } 1
+
+  # Check the db contents is as expected.
+  #
+  do_test 2.$tn.4 {
+    list [sql1 {SELECT count(*) FROM xx}] [sql2 {SELECT count(*) FROM xx}]
+  } {3 3}
+
+  set V1 [sql1 {PRAGMA data_version}] 
+  set V2 [sql2 {PRAGMA data_version}]
+
+  # Finish applying the OTA (i.e. do the incremental checkpoint). Check that
+  # this does not cause the data-version values to change.
+  #
+  while {[ota step]=="SQLITE_OK"} { }
+  ota close
+
+  do_test 2.$tn.6 {
+    list [sql1 {PRAGMA data_version}] [sql2 {PRAGMA data_version}]
+  } [list $V1 $V2]
+
+}
+
 finish_test
 
diff --git a/manifest b/manifest
index d8e9e48dae..f4db9d2282 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Further\soptimizations\sfor\sfts5\sb-tree\sseeks.
-D 2015-07-07T08:29:32.749
+C Add\sa\stest\scase\sto\sverify\sthat\s"PRAGMA\sdata_version"\sworks\sas\sexpected\swhen\san\sOTA\sclient\swrites\sto\sthe\sdatabase.
+D 2015-07-07T19:07:30.099
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 017bf0511d1b2dd1db5e16488fbf75a17b526cbc
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -204,7 +204,7 @@ F ext/ota/ota.c 3a849c3b0a4ad6e63125668be9f67be03621216e
 F ext/ota/ota1.test abdcbe746db4c7f7b51e842b576cacb33eef28f5
 F ext/ota/ota10.test 85e0f6e7964db5007590c1b299e75211ed4240d4
 F ext/ota/ota11.test 2f606cd2b4af260a86b549e91b9f395450fc75cb
-F ext/ota/ota12.test 0dff44474de448fb4b0b28c20da63273a4149abb
+F ext/ota/ota12.test e4c0b9a14255ffbe04d241fc15da2c65b3c06846
 F ext/ota/ota13.test f7a3d73fa5d3fabf2755b569f125fce7390a874c
 F ext/ota/ota3.test 3fe3521fbdce32d0e4e116a60999c3cba47712c5
 F ext/ota/ota5.test ad0799daf8923ddebffe75ae8c5504ca90b7fadb
@@ -1364,7 +1364,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 7b7da1eb435d321fc4283f6aa2161fa1e16f2cf3
-R 24874c7422a967bb097e1a3642b88228
+P f37899686c032145f431f81c1072738536c61c7e
+R f128084ba3d886029accf1024856382b
 U dan
-Z f8ef7d272cb32b051f2a236ea6d3cc77
+Z ea7265b2d9c52b49db7f82cb72c0c2d5
diff --git a/manifest.uuid b/manifest.uuid
index c950998394..ea37ca3c4e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f37899686c032145f431f81c1072738536c61c7e
\ No newline at end of file
+6441f2dc9e7f53de64bc15b1d766c885b2635112
\ No newline at end of file

From bcf2163771f37698c082ab399451af5f54a0714f Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 7 Jul 2015 19:35:11 +0000
Subject: [PATCH 27/40] Merge the Fts5MultiSegIter and Fts5IndexIter objects.

FossilOrigin-Name: 6ace73541ae48aeb19e0c2d9f6772d032d0a7eae
---
 ext/fts5/fts5_index.c          | 175 +++++++++++++++++----------------
 ext/fts5/test/fts5version.test |   1 -
 manifest                       |  14 +--
 manifest.uuid                  |   2 +-
 4 files changed, 96 insertions(+), 96 deletions(-)

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 861d300c2b..966ea0abd1 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -294,7 +294,6 @@ typedef struct Fts5Data Fts5Data;
 typedef struct Fts5DlidxIter Fts5DlidxIter;
 typedef struct Fts5DlidxLvl Fts5DlidxLvl;
 typedef struct Fts5DlidxWriter Fts5DlidxWriter;
-typedef struct Fts5MultiSegIter Fts5MultiSegIter;
 typedef struct Fts5NodeIter Fts5NodeIter;
 typedef struct Fts5PageWriter Fts5PageWriter;
 typedef struct Fts5SegIter Fts5SegIter;
@@ -348,16 +347,6 @@ struct Fts5DoclistIter {
   int nPoslist;
 };
 
-/*
-** Each iterator used by external modules is an instance of this type.
-*/
-struct Fts5IndexIter {
-  Fts5Index *pIndex;
-  Fts5Structure *pStruct;
-  Fts5MultiSegIter *pMulti;
-  Fts5Buffer poslist;             /* Buffer containing current poslist */
-};
-
 /*
 ** The contents of the "structure" record for each index are represented
 ** using an Fts5Structure record in memory. Which uses instances of the 
@@ -375,6 +364,7 @@ struct Fts5StructureLevel {
   Fts5StructureSegment *aSeg;     /* Array of segments. aSeg[0] is oldest. */
 };
 struct Fts5Structure {
+  int nRef;                       /* Object reference count */
   u64 nWriteCounter;              /* Total leaves written to level 0 */
   int nSegment;                   /* Total segments in this structure */
   int nLevel;                     /* Number of levels in this index */
@@ -510,7 +500,16 @@ struct Fts5SegIter {
 #define FTS5_SEGITER_REVERSE 0x02
 
 
-struct Fts5MultiSegIter {
+/*
+** poslist:
+**   Used by sqlite3Fts5IterPoslist() when the poslist needs to be buffered.
+**   There is no way to tell if this is populated or not.
+*/
+struct Fts5IndexIter {
+  Fts5Index *pIndex;              /* Index that owns this iterator */
+  Fts5Structure *pStruct;         /* Database structure for this iterator */
+  Fts5Buffer poslist;             /* Buffer containing current poslist */
+
   int nSeg;                       /* Size of aSeg[] array */
   int bRev;                       /* True to iterate in reverse order */
   int bSkipEmpty;                 /* True to skip deleted entries */
@@ -859,8 +858,9 @@ static void fts5DataRemoveSegment(Fts5Index *p, int iSegid){
 ** call to fts5StructureRead() or fts5StructureDecode().
 */
 static void fts5StructureRelease(Fts5Structure *pStruct){
-  if( pStruct ){
+  if( pStruct && 0>=(--pStruct->nRef) ){
     int i;
+    assert( pStruct->nRef==0 );
     for(i=0; i<pStruct->nLevel; i++){
       sqlite3_free(pStruct->aLevel[i].aSeg);
     }
@@ -868,6 +868,10 @@ static void fts5StructureRelease(Fts5Structure *pStruct){
   }
 }
 
+static void fts5StructureRef(Fts5Structure *pStruct){
+  pStruct->nRef++;
+}
+
 /*
 ** Deserialize and return the structure record currently stored in serialized
 ** form within buffer pData/nData.
@@ -909,6 +913,7 @@ static int fts5StructureDecode(
   pRet = (Fts5Structure*)sqlite3Fts5MallocZero(&rc, nByte);
 
   if( pRet ){
+    pRet->nRef = 1;
     pRet->nLevel = nLevel;
     pRet->nSegment = nSegment;
     i += sqlite3Fts5GetVarint(&pData[i], &pRet->nWriteCounter);
@@ -1767,7 +1772,7 @@ static void fts5SegIterReverseNewPage(Fts5Index *p, Fts5SegIter *pIter){
 ** points to a delete marker. A delete marker is an entry with a 0 byte
 ** position-list.
 */
-static int fts5MultiIterIsEmpty(Fts5Index *p, Fts5MultiSegIter *pIter){
+static int fts5MultiIterIsEmpty(Fts5Index *p, Fts5IndexIter *pIter){
   Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst];
   return (p->rc==SQLITE_OK && pSeg->pLeaf && pSeg->nPos==0);
 }
@@ -2410,7 +2415,7 @@ static void fts5SegIterClear(Fts5SegIter *pIter){
 ** two iterators.
 */
 static void fts5AssertComparisonResult(
-  Fts5MultiSegIter *pIter, 
+  Fts5IndexIter *pIter, 
   Fts5SegIter *p1,
   Fts5SegIter *p2,
   Fts5CResult *pRes
@@ -2451,7 +2456,7 @@ static void fts5AssertComparisonResult(
 ** statement used to verify that the contents of the pIter->aFirst[] array
 ** are correct.
 */
-static void fts5AssertMultiIterSetup(Fts5Index *p, Fts5MultiSegIter *pIter){
+static void fts5AssertMultiIterSetup(Fts5Index *p, Fts5IndexIter *pIter){
   if( p->rc==SQLITE_OK ){
     int i;
 
@@ -2484,7 +2489,7 @@ static void fts5AssertMultiIterSetup(Fts5Index *p, Fts5MultiSegIter *pIter){
 ** to a key that is a duplicate of another, higher priority, 
 ** segment-iterator in the pSeg->aSeg[] array.
 */
-static int fts5MultiIterDoCompare(Fts5MultiSegIter *pIter, int iOut){
+static int fts5MultiIterDoCompare(Fts5IndexIter *pIter, int iOut){
   int i1;                         /* Index of left-hand Fts5SegIter */
   int i2;                         /* Index of right-hand Fts5SegIter */
   int iRes;
@@ -2630,19 +2635,21 @@ static void fts5SegIterNextFrom(
 /*
 ** Free the iterator object passed as the second argument.
 */
-static void fts5MultiIterFree(Fts5Index *p, Fts5MultiSegIter *pIter){
+static void fts5MultiIterFree(Fts5Index *p, Fts5IndexIter *pIter){
   if( pIter ){
     int i;
     for(i=0; i<pIter->nSeg; i++){
       fts5SegIterClear(&pIter->aSeg[i]);
     }
+    fts5StructureRelease(pIter->pStruct);
+    fts5BufferFree(&pIter->poslist);
     sqlite3_free(pIter);
   }
 }
 
 static void fts5MultiIterAdvanced(
   Fts5Index *p,                   /* FTS5 backend to iterate within */
-  Fts5MultiSegIter *pIter,        /* Iterator to update aFirst[] array for */
+  Fts5IndexIter *pIter,           /* Iterator to update aFirst[] array for */
   int iChanged,                   /* Index of sub-iterator just advanced */
   int iMinset                     /* Minimum entry in aFirst[] to set */
 ){
@@ -2668,7 +2675,7 @@ static void fts5MultiIterAdvanced(
 */ 
 static int fts5MultiIterAdvanceRowid(
   Fts5Index *p,                   /* FTS5 backend to iterate within */
-  Fts5MultiSegIter *pIter,        /* Iterator to update aFirst[] array for */
+  Fts5IndexIter *pIter,           /* Iterator to update aFirst[] array for */
   int iChanged                    /* Index of sub-iterator just advanced */
 ){
   int i;
@@ -2700,7 +2707,7 @@ static int fts5MultiIterAdvanceRowid(
 /*
 ** Set the pIter->bEof variable based on the state of the sub-iterators.
 */
-static void fts5MultiIterSetEof(Fts5MultiSegIter *pIter){
+static void fts5MultiIterSetEof(Fts5IndexIter *pIter){
   pIter->bEof = pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf==0;
 }
 
@@ -2713,7 +2720,7 @@ static void fts5MultiIterSetEof(Fts5MultiSegIter *pIter){
 */
 static void fts5MultiIterNext(
   Fts5Index *p, 
-  Fts5MultiSegIter *pIter,
+  Fts5IndexIter *pIter,
   int bFrom,                      /* True if argument iFrom is valid */
   i64 iFrom                       /* Advance at least as far as this */
 ){
@@ -2743,28 +2750,29 @@ static void fts5MultiIterNext(
   }
 }
 
-static Fts5MultiSegIter *fts5MultiIterAlloc(
+static Fts5IndexIter *fts5MultiIterAlloc(
   Fts5Index *p,                   /* FTS5 backend to iterate within */
   int nSeg
 ){
-  Fts5MultiSegIter *pNew;
+  Fts5IndexIter *pNew;
   int nSlot;                      /* Power of two >= nSeg */
 
   for(nSlot=2; nSlot<nSeg; nSlot=nSlot*2);
   pNew = fts5IdxMalloc(p, 
-      sizeof(Fts5MultiSegIter) +          /* pNew */
+      sizeof(Fts5IndexIter) +             /* pNew */
       sizeof(Fts5SegIter) * (nSlot-1) +   /* pNew->aSeg[] */
       sizeof(Fts5CResult) * nSlot         /* pNew->aFirst[] */
   );
   if( pNew ){
     pNew->nSeg = nSlot;
     pNew->aFirst = (Fts5CResult*)&pNew->aSeg[nSlot];
+    pNew->pIndex = p;
   }
   return pNew;
 }
 
 /*
-** Allocate a new Fts5MultiSegIter object.
+** Allocate a new Fts5IndexIter object.
 **
 ** The new object will be used to iterate through data in structure pStruct.
 ** If iLevel is -ve, then all data in all segments is merged. Or, if iLevel
@@ -2782,13 +2790,13 @@ static void fts5MultiIterNew(
   const u8 *pTerm, int nTerm,     /* Term to seek to (or NULL/0) */
   int iLevel,                     /* Level to iterate (-1 for all) */
   int nSegment,                   /* Number of segments to merge (iLevel>=0) */
-  Fts5MultiSegIter **ppOut        /* New object */
+  Fts5IndexIter **ppOut           /* New object */
 ){
   int nSeg = 0;                   /* Number of segment-iters in use */
   int iIter = 0;                  /* */
   int iSeg;                       /* Used to iterate through segments */
   Fts5StructureLevel *pLvl;
-  Fts5MultiSegIter *pNew;
+  Fts5IndexIter *pNew;
 
   assert( (pTerm==0 && nTerm==0) || iLevel<0 );
 
@@ -2806,6 +2814,8 @@ static void fts5MultiIterNew(
   if( pNew==0 ) return;
   pNew->bRev = (0!=(flags & FTS5INDEX_QUERY_DESC));
   pNew->bSkipEmpty = bSkipEmpty;
+  pNew->pStruct = pStruct;
+  fts5StructureRef(pStruct);
 
   /* Initialize each of the component segment iterators. */
   if( iLevel<0 ){
@@ -2859,16 +2869,16 @@ static void fts5MultiIterNew(
 }
 
 /*
-** Create an Fts5MultiSegIter that iterates through the doclist provided
+** Create an Fts5IndexIter that iterates through the doclist provided
 ** as the second argument.
 */
 static void fts5MultiIterNew2(
   Fts5Index *p,                   /* FTS5 backend to iterate within */
   Fts5Data *pData,                /* Doclist to iterate through */
   int bDesc,                      /* True for descending rowid order */
-  Fts5MultiSegIter **ppOut        /* New object */
+  Fts5IndexIter **ppOut           /* New object */
 ){
-  Fts5MultiSegIter *pNew;
+  Fts5IndexIter *pNew;
   pNew = fts5MultiIterAlloc(p, 2);
   if( pNew ){
     Fts5SegIter *pIter = &pNew->aSeg[1];
@@ -2900,7 +2910,7 @@ static void fts5MultiIterNew2(
 ** Return true if the iterator is at EOF or if an error has occurred. 
 ** False otherwise.
 */
-static int fts5MultiIterEof(Fts5Index *p, Fts5MultiSegIter *pIter){
+static int fts5MultiIterEof(Fts5Index *p, Fts5IndexIter *pIter){
   assert( p->rc 
       || (pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf==0)==pIter->bEof 
   );
@@ -2912,7 +2922,7 @@ static int fts5MultiIterEof(Fts5Index *p, Fts5MultiSegIter *pIter){
 ** to. If the iterator points to EOF when this function is called the
 ** results are undefined.
 */
-static i64 fts5MultiIterRowid(Fts5MultiSegIter *pIter){
+static i64 fts5MultiIterRowid(Fts5IndexIter *pIter){
   assert( pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf );
   return pIter->aSeg[ pIter->aFirst[1].iFirst ].iRowid;
 }
@@ -2922,7 +2932,7 @@ static i64 fts5MultiIterRowid(Fts5MultiSegIter *pIter){
 */
 static void fts5MultiIterNextFrom(
   Fts5Index *p, 
-  Fts5MultiSegIter *pIter, 
+  Fts5IndexIter *pIter, 
   i64 iMatch
 ){
   while( 1 ){
@@ -2939,7 +2949,7 @@ static void fts5MultiIterNextFrom(
 ** Return a pointer to a buffer containing the term associated with the 
 ** entry that the iterator currently points to.
 */
-static const u8 *fts5MultiIterTerm(Fts5MultiSegIter *pIter, int *pn){
+static const u8 *fts5MultiIterTerm(Fts5IndexIter *pIter, int *pn){
   Fts5SegIter *p = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
   *pn = p->term.n;
   return p->term.p;
@@ -3567,7 +3577,7 @@ static void fts5WriteInitForAppend(
 ** incremental merge operation. This function is called if the incremental
 ** merge step has finished but the input has not been completely exhausted.
 */
-static void fts5TrimSegments(Fts5Index *p, Fts5MultiSegIter *pIter){
+static void fts5TrimSegments(Fts5Index *p, Fts5IndexIter *pIter){
   int i;
   Fts5Buffer buf;
   memset(&buf, 0, sizeof(Fts5Buffer));
@@ -3627,7 +3637,7 @@ static void fts5IndexMergeLevel(
   Fts5Structure *pStruct = *ppStruct;
   Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl];
   Fts5StructureLevel *pLvlOut;
-  Fts5MultiSegIter *pIter = 0;    /* Iterator to read input data */
+  Fts5IndexIter *pIter = 0;       /* Iterator to read input data */
   int nRem = pnRem ? *pnRem : 0;  /* Output leaf pages left to write */
   int nInput;                     /* Number of input segments */
   Fts5SegWriter writer;           /* Writer object */
@@ -4101,6 +4111,7 @@ int sqlite3Fts5IndexOptimize(Fts5Index *p){
     Fts5StructureLevel *pLvl;
     int nByte = nSeg * sizeof(Fts5StructureSegment);
     pNew->nLevel = pStruct->nLevel+1;
+    pNew->nRef = 1;
     pNew->nWriteCounter = pStruct->nWriteCounter;
     pLvl = &pNew->aLevel[pStruct->nLevel];
     pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&p->rc, nByte);
@@ -4180,7 +4191,7 @@ static void fts5SegiterPoslist(
 */
 static void fts5MultiIterPoslist(
   Fts5Index *p,
-  Fts5MultiSegIter *pMulti,
+  Fts5IndexIter *pMulti,
   int bSz,                        /* Append a size field before the data */
   Fts5Buffer *pBuf
 ){
@@ -4334,7 +4345,7 @@ static void fts5SetupPrefixIter(
   int bDesc,                      /* True for "ORDER BY rowid DESC" */
   const u8 *pToken,               /* Buffer containing prefix to match */
   int nToken,                     /* Size of buffer pToken in bytes */
-  Fts5IndexIter *pIter            /* Populate this object */
+  Fts5IndexIter **ppIter       /* OUT: New iterator */
 ){
   Fts5Structure *pStruct;
   Fts5Buffer *aBuf;
@@ -4347,7 +4358,7 @@ static void fts5SetupPrefixIter(
     const int flags = FTS5INDEX_QUERY_SCAN;
     int i;
     i64 iLastRowid = 0;
-    Fts5MultiSegIter *p1 = 0;     /* Iterator used to gather data from index */
+    Fts5IndexIter *p1 = 0;     /* Iterator used to gather data from index */
     Fts5Data *pData;
     Fts5Buffer doclist;
 
@@ -4390,7 +4401,7 @@ static void fts5SetupPrefixIter(
       pData->p = (u8*)&pData[1];
       pData->n = doclist.n;
       memcpy(pData->p, doclist.p, doclist.n);
-      fts5MultiIterNew2(p, pData, bDesc, &pIter->pMulti);
+      fts5MultiIterNew2(p, pData, bDesc, ppIter);
     }
     fts5BufferFree(&doclist);
   }
@@ -4600,7 +4611,7 @@ int sqlite3Fts5IndexQuery(
   Fts5IndexIter **ppIter          /* OUT: New iterator object */
 ){
   Fts5Config *pConfig = p->pConfig;
-  Fts5IndexIter *pRet;
+  Fts5IndexIter *pRet = 0;
   int iIdx = 0;
   Fts5Buffer buf = {0, 0, 0};
 
@@ -4611,45 +4622,41 @@ int sqlite3Fts5IndexQuery(
 
   if( sqlite3Fts5BufferGrow(&p->rc, &buf, nToken+1)==0 ){
     memcpy(&buf.p[1], pToken, nToken);
-  }
 
 #ifdef SQLITE_DEBUG
-  if( flags & FTS5INDEX_QUERY_TEST_NOIDX ){
-    assert( flags & FTS5INDEX_QUERY_PREFIX );
-    iIdx = 1+pConfig->nPrefix;
-  }else
+    if( flags & FTS5INDEX_QUERY_TEST_NOIDX ){
+      assert( flags & FTS5INDEX_QUERY_PREFIX );
+      iIdx = 1+pConfig->nPrefix;
+    }else
 #endif
-  if( flags & FTS5INDEX_QUERY_PREFIX ){
-    int nChar = fts5IndexCharlen(pToken, nToken);
-    for(iIdx=1; iIdx<=pConfig->nPrefix; iIdx++){
-      if( pConfig->aPrefix[iIdx-1]==nChar ) break;
+    if( flags & FTS5INDEX_QUERY_PREFIX ){
+      int nChar = fts5IndexCharlen(pToken, nToken);
+      for(iIdx=1; iIdx<=pConfig->nPrefix; iIdx++){
+        if( pConfig->aPrefix[iIdx-1]==nChar ) break;
+      }
     }
-  }
 
-  pRet = (Fts5IndexIter*)sqlite3Fts5MallocZero(&p->rc, sizeof(Fts5IndexIter));
-  if( pRet ){
-    pRet->pIndex = p;
     if( iIdx<=pConfig->nPrefix ){
+      Fts5Structure *pStruct = fts5StructureRead(p);
       buf.p[0] = FTS5_MAIN_PREFIX + iIdx;
-      pRet->pStruct = fts5StructureRead(p);
-      if( pRet->pStruct ){
-        fts5MultiIterNew(
-            p, pRet->pStruct, 1, flags, buf.p, nToken+1, -1, 0, &pRet->pMulti
-        );
+      if( pStruct ){
+        fts5MultiIterNew(p, pStruct, 1, flags, buf.p, nToken+1, -1, 0, &pRet);
+        fts5StructureRelease(pStruct);
       }
     }else{
       int bDesc = (flags & FTS5INDEX_QUERY_DESC)!=0;
       buf.p[0] = FTS5_MAIN_PREFIX;
-      fts5SetupPrefixIter(p, bDesc, buf.p, nToken+1, pRet);
+      fts5SetupPrefixIter(p, bDesc, buf.p, nToken+1, &pRet);
     }
-  }
 
-  if( p->rc ){
-    sqlite3Fts5IterClose(pRet);
-    pRet = 0;
+    if( p->rc ){
+      sqlite3Fts5IterClose(pRet);
+      pRet = 0;
+      fts5CloseReader(p);
+    }
+    *ppIter = pRet;
+    sqlite3Fts5BufferFree(&buf);
   }
-  *ppIter = pRet;
-  sqlite3Fts5BufferFree(&buf);
   return fts5IndexReturn(p);
 }
 
@@ -4658,7 +4665,7 @@ int sqlite3Fts5IndexQuery(
 */
 int sqlite3Fts5IterEof(Fts5IndexIter *pIter){
   assert( pIter->pIndex->rc==SQLITE_OK );
-  return pIter->pMulti->bEof;
+  return pIter->bEof;
 }
 
 /*
@@ -4666,7 +4673,7 @@ int sqlite3Fts5IterEof(Fts5IndexIter *pIter){
 */
 int sqlite3Fts5IterNext(Fts5IndexIter *pIter){
   assert( pIter->pIndex->rc==SQLITE_OK );
-  fts5MultiIterNext(pIter->pIndex, pIter->pMulti, 0, 0);
+  fts5MultiIterNext(pIter->pIndex, pIter, 0, 0);
   return fts5IndexReturn(pIter->pIndex);
 }
 
@@ -4675,18 +4682,16 @@ int sqlite3Fts5IterNext(Fts5IndexIter *pIter){
 */
 int sqlite3Fts5IterNextScan(Fts5IndexIter *pIter){
   Fts5Index *p = pIter->pIndex;
-  Fts5MultiSegIter *pMulti = pIter->pMulti;
 
   assert( pIter->pIndex->rc==SQLITE_OK );
-  assert( pMulti );
 
-  fts5MultiIterNext(p, pMulti, 0, 0);
+  fts5MultiIterNext(p, pIter, 0, 0);
   if( p->rc==SQLITE_OK ){
-    Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1].iFirst ];
+    Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
     if( pSeg->pLeaf && pSeg->term.p[0]!=FTS5_MAIN_PREFIX ){
       fts5DataRelease(pSeg->pLeaf);
       pSeg->pLeaf = 0;
-      pMulti->bEof = 1;
+      pIter->bEof = 1;
     }
   }
 
@@ -4699,7 +4704,7 @@ int sqlite3Fts5IterNextScan(Fts5IndexIter *pIter){
 ** in ascending or descending rowid order.
 */
 int sqlite3Fts5IterNextFrom(Fts5IndexIter *pIter, i64 iMatch){
-  fts5MultiIterNextFrom(pIter->pIndex, pIter->pMulti, iMatch);
+  fts5MultiIterNextFrom(pIter->pIndex, pIter, iMatch);
   return fts5IndexReturn(pIter->pIndex);
 }
 
@@ -4707,7 +4712,7 @@ int sqlite3Fts5IterNextFrom(Fts5IndexIter *pIter, i64 iMatch){
 ** Return the current rowid.
 */
 i64 sqlite3Fts5IterRowid(Fts5IndexIter *pIter){
-  return fts5MultiIterRowid(pIter->pMulti);
+  return fts5MultiIterRowid(pIter);
 }
 
 /*
@@ -4715,7 +4720,7 @@ i64 sqlite3Fts5IterRowid(Fts5IndexIter *pIter){
 */
 const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIter, int *pn){
   int n;
-  const char *z = (const char*)fts5MultiIterTerm(pIter->pMulti, &n);
+  const char *z = (const char*)fts5MultiIterTerm(pIter, &n);
   *pn = n-1;
   return &z[1];
 }
@@ -4735,8 +4740,7 @@ int sqlite3Fts5IterPoslist(
   int *pn,                        /* OUT: Size of position-list in bytes */
   i64 *piRowid                    /* OUT: Current rowid */
 ){
-  Fts5MultiSegIter *pMulti = pIter->pMulti;
-  Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1].iFirst ];
+  Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
   assert( pIter->pIndex->rc==SQLITE_OK );
   *piRowid = pSeg->iRowid;
   *pn = pSeg->nPos;
@@ -4757,11 +4761,10 @@ int sqlite3Fts5IterPoslist(
 */
 int sqlite3Fts5IterPoslistBuffer(Fts5IndexIter *pIter, Fts5Buffer *pBuf){
   Fts5Index *p = pIter->pIndex;
-  Fts5MultiSegIter *pMulti = pIter->pMulti;
 
   assert( p->rc==SQLITE_OK );
   fts5BufferZero(pBuf);
-  fts5MultiIterPoslist(p, pMulti, 0, pBuf);
+  fts5MultiIterPoslist(p, pIter, 0, pBuf);
   return fts5IndexReturn(p);
 }
 
@@ -4770,11 +4773,9 @@ int sqlite3Fts5IterPoslistBuffer(Fts5IndexIter *pIter, Fts5Buffer *pBuf){
 */
 void sqlite3Fts5IterClose(Fts5IndexIter *pIter){
   if( pIter ){
-    fts5MultiIterFree(pIter->pIndex, pIter->pMulti);
-    fts5StructureRelease(pIter->pStruct);
-    fts5BufferFree(&pIter->poslist);
-    fts5CloseReader(pIter->pIndex);
-    sqlite3_free(pIter);
+    Fts5Index *pIndex = pIter->pIndex;
+    fts5MultiIterFree(pIter->pIndex, pIter);
+    fts5CloseReader(pIndex);
   }
 }
 
@@ -5239,7 +5240,7 @@ static void fts5IndexIntegrityCheckSegment(
 int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){
   u64 cksum2 = 0;                 /* Checksum based on contents of indexes */
   Fts5Buffer poslist = {0,0,0};   /* Buffer used to hold a poslist */
-  Fts5MultiSegIter *pIter;        /* Used to iterate through entire index */
+  Fts5IndexIter *pIter;           /* Used to iterate through entire index */
   Fts5Structure *pStruct;         /* Index structure */
 
   /* Used by extra internal tests only run if NDEBUG is not defined */
diff --git a/ext/fts5/test/fts5version.test b/ext/fts5/test/fts5version.test
index 2176fee7d3..1ae75fd2ae 100644
--- a/ext/fts5/test/fts5version.test
+++ b/ext/fts5/test/fts5version.test
@@ -46,7 +46,6 @@ do_test 1.5 {
   catchsql { SELECT * FROM t1 WHERE t1 MATCH 'a' }
 } {1 {invalid fts5 file format (found 3, expected 2) - run 'rebuild'}}
 
-breakpoint
 do_test 1.6 {
   db close
   sqlite3 db test.db
diff --git a/manifest b/manifest
index f4db9d2282..cc7fb8f6d4 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sa\stest\scase\sto\sverify\sthat\s"PRAGMA\sdata_version"\sworks\sas\sexpected\swhen\san\sOTA\sclient\swrites\sto\sthe\sdatabase.
-D 2015-07-07T19:07:30.099
+C Merge\sthe\sFts5MultiSegIter\sand\sFts5IndexIter\sobjects.
+D 2015-07-07T19:35:11.437
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 017bf0511d1b2dd1db5e16488fbf75a17b526cbc
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -112,7 +112,7 @@ F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015
 F ext/fts5/fts5_config.c b2456e9625bca41c51d54c363e369c6356895c90
 F ext/fts5/fts5_expr.c d2e148345639c5a5583e0daa39a639bf298ae6a7
 F ext/fts5/fts5_hash.c 219f4edd72e5cf95b19c33f1058809a18fad5229
-F ext/fts5/fts5_index.c 19f6db2a43c0126267c7fa11e3ed41105eb89e2f
+F ext/fts5/fts5_index.c ab14b8a273ec72a4407eebfb787bbfb51acef565
 F ext/fts5/fts5_main.c 37b0055cb4036c4b4bb4eb36e30ebd1c21c63939
 F ext/fts5/fts5_storage.c 4cae85b5287b159d9d98174a4e70adf872b0930a
 F ext/fts5/fts5_tcl.c 85eb4e0d0fefa9420b78151496ad4599a1783e20
@@ -175,7 +175,7 @@ F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841
 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59
 F ext/fts5/test/fts5unicode3.test 35c3d02aa7acf7d43d8de3bfe32c15ba96e8928e
 F ext/fts5/test/fts5unindexed.test e9539d5b78c677315e7ed8ea911d4fd25437c680
-F ext/fts5/test/fts5version.test bed59038e937c40d3c0056d08076db7874c6cd4a
+F ext/fts5/test/fts5version.test c54a708236642bcc850d2aedc6f505fef1d9f9f1
 F ext/fts5/test/fts5vocab.test cdf97b9678484e9bad5062edf9c9106e5c3b0c5c
 F ext/fts5/tool/loadfts5.tcl 95edf0b6b92a09f9ed85595038b1108127987556
 F ext/fts5/tool/mkfts5c.tcl 5745072c7de346e18c7f491e4c3281fe8a1cfe51
@@ -1364,7 +1364,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P f37899686c032145f431f81c1072738536c61c7e
-R f128084ba3d886029accf1024856382b
+P 6441f2dc9e7f53de64bc15b1d766c885b2635112
+R f97628f043782e8117cb742bf5766746
 U dan
-Z ea7265b2d9c52b49db7f82cb72c0c2d5
+Z e1b4a5f91c01e7d2767125a8d1a40f5a
diff --git a/manifest.uuid b/manifest.uuid
index ea37ca3c4e..9704e8b452 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6441f2dc9e7f53de64bc15b1d766c885b2635112
\ No newline at end of file
+6ace73541ae48aeb19e0c2d9f6772d032d0a7eae
\ No newline at end of file

From 4297c7c4be95f3e6d906a64dd52d2329b45ff17b Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 7 Jul 2015 21:14:42 +0000
Subject: [PATCH 28/40] Add a compile-time constant for configuring the default
 pagecache setup.

FossilOrigin-Name: ad5fdfbac85682f0f0dee62e1086f40026a58be2
---
 manifest        | 14 +++++++-------
 manifest.uuid   |  2 +-
 src/global.c    |  2 +-
 src/sqliteInt.h | 10 ++++++++++
 4 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index ebdaa318e2..7c9dd2b539 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C No\sneed\sto\stest\ssqlite3GlobalConfig.bCoreMutex\swhen\sSQLITE_THREADSAFE==0.
-D 2015-07-07T15:14:16.002
+C Add\sa\scompile-time\sconstant\sfor\sconfiguring\sthe\sdefault\spagecache\ssetup.
+D 2015-07-07T21:14:42.575
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 017bf0511d1b2dd1db5e16488fbf75a17b526cbc
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -283,7 +283,7 @@ F src/expr.c c5c58e4d01c7ceb2266791d8d877f1b23a88e316
 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb
 F src/fkey.c c9b63a217d86582c22121699a47f22f524608869
 F src/func.c a98ea5880dc50e9ca6dd6f57079a37b9cfcdecf1
-F src/global.c 32c0ebfdca59db55c761f1405abf7bdfd24ecae0
+F src/global.c 508e4087f7b41d688e4762dcf4d4fe28cfbc87f9
 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5
 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094
 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08
@@ -332,7 +332,7 @@ F src/shell.c 8af3cced094aebb5f57a8ad739b9dafc7867eed7
 F src/sqlite.h.in 3d951bf985839de7fcf4d3f69568bb4df2641abe
 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad
 F src/sqlite3ext.h be1a718b7d2ce40ceba725ae92c8eb5f18003066
-F src/sqliteInt.h bc2cb1755150fc0b490d1af23ac2911b03f2f92d
+F src/sqliteInt.h d8d420d66a5c403b119696159b69f71cd53840ce
 F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46
 F src/status.c f266ad8a2892d659b74f0f50cb6a88b6e7c12179
 F src/table.c 51b46b2a62d1b3a959633d593b89bab5e2c9155e
@@ -1364,7 +1364,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 5b60df5503f61922326b3e10e94aa197b17afe4b
-R 2d7b920ac2e4dfd8e6b212c060ee3e55
+P 26db0fee06947d75621237a641b9652b312b3b37
+R eddaba760dcd1fbd940dd405a17b49da
 U drh
-Z 96d25228913b7734acfab488bf16d902
+Z 4021e6ca5eea0db9e457b6c7679c87e9
diff --git a/manifest.uuid b/manifest.uuid
index cd9a67d080..799437db10 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-26db0fee06947d75621237a641b9652b312b3b37
\ No newline at end of file
+ad5fdfbac85682f0f0dee62e1086f40026a58be2
\ No newline at end of file
diff --git a/src/global.c b/src/global.c
index 3b1e804bb8..ef4fe56ae1 100644
--- a/src/global.c
+++ b/src/global.c
@@ -186,7 +186,7 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = {
    0,                         /* nScratch */
    (void*)0,                  /* pPage */
    0,                         /* szPage */
-   100,                       /* nPage */
+   SQLITE_DEFAULT_PCACHE_INITSZ, /* nPage */
    0,                         /* mxParserStack */
    0,                         /* sharedCacheEnabled */
    SQLITE_SORTER_PMASZ,       /* szPma */
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index eaf433b7a1..199a1cc662 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -509,6 +509,16 @@
 # define SQLITE_MAX_WORKER_THREADS SQLITE_DEFAULT_WORKER_THREADS
 #endif
 
+/*
+** The default initial allocation for the pagecache when using separate
+** pagecaches for each database connection.  A positive number is the
+** number of pages.  A negative number N translations means that a buffer
+** of -1024*N bytes is allocated and used for as many pages as it will hold.
+*/
+#ifndef SQLITE_DEFAULT_PCACHE_INITSZ
+# define SQLITE_DEFAULT_PCACHE_INITSZ 100
+#endif
+
 
 /*
 ** GCC does not define the offsetof() macro so we'll have to do it

From e56d43010dc14f5a0a8651205325e4e15cbb9ff6 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 8 Jul 2015 01:22:52 +0000
Subject: [PATCH 29/40] Avoid left-shifting bits into the sign of a signed
 integer, as such behavior is technically undefined.

FossilOrigin-Name: 9417b4021b779876dcb3b9be636f01f4055eb54e
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/btree.c   |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 7c9dd2b539..77fde56543 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sa\scompile-time\sconstant\sfor\sconfiguring\sthe\sdefault\spagecache\ssetup.
-D 2015-07-07T21:14:42.575
+C Avoid\sleft-shifting\sbits\sinto\sthe\ssign\sof\sa\ssigned\sinteger,\sas\ssuch\sbehavior\nis\stechnically\sundefined.
+D 2015-07-08T01:22:52.331
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 017bf0511d1b2dd1db5e16488fbf75a17b526cbc
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -269,7 +269,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240
 F src/backup.c 4d9134dc988a87838c06056c89c0e8c4700a0452
 F src/bitvec.c d1f21d7d91690747881f03940584f4cc548c9d3d
 F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79
-F src/btree.c 740ab30df845f4a2c2bdb731ed191b4b63b2b2eb
+F src/btree.c 781deff0d5af639e8dd4f83ec963cc3bbf8cffc2
 F src/btree.h 969adc948e89e449220ff0ff724c94bb2a52e9f1
 F src/btreeInt.h 2ad754dd4528baa8d0946a593cc373b890bf859e
 F src/build.c b3f15255d5b16e42dafeaa638fd4f8a47c94ed70
@@ -1364,7 +1364,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 26db0fee06947d75621237a641b9652b312b3b37
-R eddaba760dcd1fbd940dd405a17b49da
+P ad5fdfbac85682f0f0dee62e1086f40026a58be2
+R ffdb1dc8cf76fb977c83d0c8ad3dcce5
 U drh
-Z 4021e6ca5eea0db9e457b6c7679c87e9
+Z 1578fd578fb2f64903bdcd3127f0d5d4
diff --git a/manifest.uuid b/manifest.uuid
index 799437db10..bd9adfaf63 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ad5fdfbac85682f0f0dee62e1086f40026a58be2
\ No newline at end of file
+9417b4021b779876dcb3b9be636f01f4055eb54e
\ No newline at end of file
diff --git a/src/btree.c b/src/btree.c
index f353e8df1f..17c7bb67c3 100644
--- a/src/btree.c
+++ b/src/btree.c
@@ -9119,7 +9119,7 @@ static int checkTreePage(
       assert( (u32)i<=usableSize-4 );     /* Enforced by btreeInitPage() */
       size = get2byte(&data[i+2]);
       assert( (u32)(i+size)<=usableSize );  /* Enforced by btreeInitPage() */
-      btreeHeapInsert(heap, (i<<16)|(i+size-1));
+      btreeHeapInsert(heap, (((u32)i)<<16)|(i+size-1));
       /* EVIDENCE-OF: R-58208-19414 The first 2 bytes of a freeblock are a
       ** big-endian integer which is the offset in the b-tree page of the next
       ** freeblock in the chain, or zero if the freeblock is the last on the

From 469813659440c8a9a8662ca79a69e8cfdb3c1d97 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 8 Jul 2015 12:25:38 +0000
Subject: [PATCH 30/40] Early detection of file corruption in a debugging
 routine, before the invalid data can cause problems.

FossilOrigin-Name: b27a47fef995f0ab2516e6ca779711cab1f50fb9
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/vdbeaux.c |  1 +
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index cc7fb8f6d4..b76060a668 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\sthe\sFts5MultiSegIter\sand\sFts5IndexIter\sobjects.
-D 2015-07-07T19:35:11.437
+C Early\sdetection\sof\sfile\scorruption\sin\sa\sdebugging\sroutine,\sbefore\sthe\sinvalid\ndata\scan\scause\sproblems.
+D 2015-07-08T12:25:38.106
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 017bf0511d1b2dd1db5e16488fbf75a17b526cbc
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -394,7 +394,7 @@ F src/vdbe.c 5ee4a2bf871418f61d06dc256b9b3a0084b5ec46
 F src/vdbe.h 7a75045d879118b9d3af7e8b3c108f2f27c51473
 F src/vdbeInt.h 8b54e01ad0463590e7cffabce0bc36da9ee4f816
 F src/vdbeapi.c 6a0d7757987018ff6b1b81bc5293219cd26bb299
-F src/vdbeaux.c 54bcc56d368b2d0bebc523cff514893156c09daf
+F src/vdbeaux.c 787f5f9d58f4c6f39294ed06909ba602d1a402e6
 F src/vdbeblob.c 4f2e8e075d238392df98c5e03a64342465b03f90
 F src/vdbemem.c ae38a0d35ae71cf604381a887c170466ba518090
 F src/vdbesort.c f5009e7a35e3065635d8918b9a31f498a499976b
@@ -1364,7 +1364,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 6441f2dc9e7f53de64bc15b1d766c885b2635112
-R f97628f043782e8117cb742bf5766746
-U dan
-Z e1b4a5f91c01e7d2767125a8d1a40f5a
+P 6ace73541ae48aeb19e0c2d9f6772d032d0a7eae
+R 8adde983bcb4c45839c171b768ffd3b9
+U drh
+Z b9cf714cb314f182c61af66a54b1bb68
diff --git a/manifest.uuid b/manifest.uuid
index 9704e8b452..9b6013994f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6ace73541ae48aeb19e0c2d9f6772d032d0a7eae
\ No newline at end of file
+b27a47fef995f0ab2516e6ca779711cab1f50fb9
\ No newline at end of file
diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index 5dbbadaa2b..08dc885df6 100644
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -3344,6 +3344,7 @@ static int vdbeRecordCompareDebug(
   /*  mem1.u.i = 0;  // not needed, here to silence compiler warning */
   
   idx1 = getVarint32(aKey1, szHdr1);
+  if( szHdr1>98307 ) return SQLITE_CORRUPT;
   d1 = szHdr1;
   assert( pKeyInfo->nField+pKeyInfo->nXField>=pPKey2->nField || CORRUPT_DB );
   assert( pKeyInfo->aSortOrder!=0 );

From 52d9f40b7def9c5fe30b6b4dd5a1f263b17619ae Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 8 Jul 2015 13:40:50 +0000
Subject: [PATCH 31/40] Do not do the bulk pcache1 allocations if
 SQLITE_CONFIG_PAGECACHE specifies a global page cache buffer.

FossilOrigin-Name: 75d98dae676996a05e97b09b19e34463e8f30d51
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/pcache1.c |  5 ++++-
 3 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 77fde56543..d9c3660327 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Avoid\sleft-shifting\sbits\sinto\sthe\ssign\sof\sa\ssigned\sinteger,\sas\ssuch\sbehavior\nis\stechnically\sundefined.
-D 2015-07-08T01:22:52.331
+C Do\snot\sdo\sthe\sbulk\spcache1\sallocations\sif\sSQLITE_CONFIG_PAGECACHE\sspecifies\na\sglobal\spage\scache\sbuffer.
+D 2015-07-08T13:40:50.550
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 017bf0511d1b2dd1db5e16488fbf75a17b526cbc
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -319,7 +319,7 @@ F src/pager.h 6d435f563b3f7fcae4b84433b76a6ac2730036e2
 F src/parse.y 6d60dda8f8d418b6dc034f1fbccd816c459983a8
 F src/pcache.c cde06aa50962595e412d497e22fd2e07878ba1f0
 F src/pcache.h 9968603796240cdf83da7e7bef76edf90619cea9
-F src/pcache1.c e3b0de837c9af01335e7e59fa8290dfa9338b2a3
+F src/pcache1.c 1f88fe632a67574e61a4d52f08e21338648c4016
 F src/pragma.c c1f4d012ea9f6b1ce52d341b2cd0ad72d560afd7
 F src/pragma.h b8632d7cdda7b25323fa580e3e558a4f0d4502cc
 F src/prepare.c 82e5db1013846a819f198336fed72c44c974e7b1
@@ -1364,7 +1364,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P ad5fdfbac85682f0f0dee62e1086f40026a58be2
-R ffdb1dc8cf76fb977c83d0c8ad3dcce5
+P 9417b4021b779876dcb3b9be636f01f4055eb54e
+R dc45b55e45fc287d8da6e081b45bc4d0
 U drh
-Z 1578fd578fb2f64903bdcd3127f0d5d4
+Z 0743262729e38494d19c03648108bfa3
diff --git a/manifest.uuid b/manifest.uuid
index bd9adfaf63..12460e3042 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-9417b4021b779876dcb3b9be636f01f4055eb54e
\ No newline at end of file
+75d98dae676996a05e97b09b19e34463e8f30d51
\ No newline at end of file
diff --git a/src/pcache1.c b/src/pcache1.c
index 90989bc1db..ed2700852d 100644
--- a/src/pcache1.c
+++ b/src/pcache1.c
@@ -702,7 +702,10 @@ static sqlite3_pcache *pcache1Create(int szPage, int szExtra, int bPurgeable){
     pcache1LeaveMutex(pGroup);
     /* Try to initialize the local bulk pagecache line allocation if using
     ** separate caches and if nPage!=0 */
-    if( pcache1.separateCache && sqlite3GlobalConfig.nPage!=0 ){
+    if( pcache1.separateCache
+     && sqlite3GlobalConfig.nPage!=0
+     && sqlite3GlobalConfig.pPage==0
+    ){
       int szBulk;
       char *zBulk;
       sqlite3BeginBenignMalloc();

From 7b341e6f99cd3981f1a39c1ba2a6e4973eb45e36 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 8 Jul 2015 14:13:44 +0000
Subject: [PATCH 32/40] Comment fix.  No changes to code.

FossilOrigin-Name: c1e2ed0ef4d3436d19d3203c5fb553caac53659e
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/pcache1.c |  3 ++-
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index d9c3660327..4fa0ddf103 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Do\snot\sdo\sthe\sbulk\spcache1\sallocations\sif\sSQLITE_CONFIG_PAGECACHE\sspecifies\na\sglobal\spage\scache\sbuffer.
-D 2015-07-08T13:40:50.550
+C Comment\sfix.\s\sNo\schanges\sto\scode.
+D 2015-07-08T14:13:44.573
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 017bf0511d1b2dd1db5e16488fbf75a17b526cbc
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -319,7 +319,7 @@ F src/pager.h 6d435f563b3f7fcae4b84433b76a6ac2730036e2
 F src/parse.y 6d60dda8f8d418b6dc034f1fbccd816c459983a8
 F src/pcache.c cde06aa50962595e412d497e22fd2e07878ba1f0
 F src/pcache.h 9968603796240cdf83da7e7bef76edf90619cea9
-F src/pcache1.c 1f88fe632a67574e61a4d52f08e21338648c4016
+F src/pcache1.c 3f4c87cf913f2de8189026d9a5223ddaf55eaf6b
 F src/pragma.c c1f4d012ea9f6b1ce52d341b2cd0ad72d560afd7
 F src/pragma.h b8632d7cdda7b25323fa580e3e558a4f0d4502cc
 F src/prepare.c 82e5db1013846a819f198336fed72c44c974e7b1
@@ -1364,7 +1364,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 9417b4021b779876dcb3b9be636f01f4055eb54e
-R dc45b55e45fc287d8da6e081b45bc4d0
+P 75d98dae676996a05e97b09b19e34463e8f30d51
+R e0f37cdeba600c01a32ef205d5f6df52
 U drh
-Z 0743262729e38494d19c03648108bfa3
+Z cb924bafedede368f45cad3c12c2aa68
diff --git a/manifest.uuid b/manifest.uuid
index 12460e3042..fb850df2b9 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-75d98dae676996a05e97b09b19e34463e8f30d51
\ No newline at end of file
+c1e2ed0ef4d3436d19d3203c5fb553caac53659e
\ No newline at end of file
diff --git a/src/pcache1.c b/src/pcache1.c
index ed2700852d..a345e6760f 100644
--- a/src/pcache1.c
+++ b/src/pcache1.c
@@ -41,7 +41,8 @@
 **
 ** If SQLITE_PCACHE_SEPARATE_HEADER is defined, then the extension is obtained
 ** using a separate memory allocation from the database page content.  This
-** seeks to overcome the "clownfoot" problem of allocating a few bytes more
+** seeks to overcome the "clownshoe" problem (also called "internal
+** fragmentation" in academic literature) of allocating a few bytes more
 ** than a power of two with the memory allocator rounding up to the next
 ** power of two, and leaving the rounded-up space unused.
 **

From a90579db99d0a597519570ad8bbc4ff3a6c82139 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 8 Jul 2015 17:59:08 +0000
Subject: [PATCH 33/40] Fix two problems that could cause fts3 auxiliary
 functions to occasionally misbehave if used with match expressions that
 contain both OR and NEAR.

FossilOrigin-Name: 372c1db2475f367d54270d5801aff0503745bff4
---
 ext/fts3/fts3.c       |  67 ++++++++++++++---------
 manifest              |  18 +++---
 manifest.uuid         |   2 +-
 test/fts3fault.test   |   8 ++-
 test/fts3offsets.test | 124 ++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 183 insertions(+), 36 deletions(-)
 create mode 100644 test/fts3offsets.test

diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c
index ac492ff1af..903d7d84fd 100644
--- a/ext/fts3/fts3.c
+++ b/ext/fts3/fts3.c
@@ -4351,6 +4351,7 @@ void sqlite3Fts3DoclistNext(
     p += sqlite3Fts3GetVarint(p, piDocid);
   }else{
     fts3PoslistCopy(0, &p);
+    while( p<&aDoclist[nDoclist] && *p==0 ) p++; 
     if( p>=&aDoclist[nDoclist] ){
       *pbEof = 1;
     }else{
@@ -5757,10 +5758,10 @@ int sqlite3Fts3EvalPhrasePoslist(
     int rc = SQLITE_OK;
     int bDescDoclist = pTab->bDescIdx;      /* For DOCID_CMP macro */
     int bOr = 0;
-    u8 bEof = 0;
     u8 bTreeEof = 0;
     Fts3Expr *p;                  /* Used to iterate from pExpr to root */
     Fts3Expr *pNear;              /* Most senior NEAR ancestor (or pExpr) */
+    int bMatch;
 
     /* Check if this phrase descends from an OR expression node. If not, 
     ** return NULL. Otherwise, the entry that corresponds to docid 
@@ -5794,31 +5795,47 @@ int sqlite3Fts3EvalPhrasePoslist(
     }
     if( rc!=SQLITE_OK ) return rc;
 
-    pIter = pPhrase->pOrPoslist;
-    iDocid = pPhrase->iOrDocid;
-    if( pCsr->bDesc==bDescDoclist ){
-      bEof = !pPhrase->doclist.nAll ||
-                 (pIter >= (pPhrase->doclist.aAll + pPhrase->doclist.nAll));
-      while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)<0 ) && bEof==0 ){
-        sqlite3Fts3DoclistNext(
-            bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll, 
-            &pIter, &iDocid, &bEof
-        );
-      }
-    }else{
-      bEof = !pPhrase->doclist.nAll || (pIter && pIter<=pPhrase->doclist.aAll);
-      while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)>0 ) && bEof==0 ){
-        int dummy;
-        sqlite3Fts3DoclistPrev(
-            bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll, 
-            &pIter, &iDocid, &dummy, &bEof
-        );
-      }
-    }
-    pPhrase->pOrPoslist = pIter;
-    pPhrase->iOrDocid = iDocid;
+    bMatch = 1;
+    for(p=pNear; p; p=p->pLeft){
+      u8 bEof = 0;
+      Fts3Expr *pTest = p;
+      Fts3Phrase *pPh;
+      assert( pTest->eType==FTSQUERY_NEAR || pTest->eType==FTSQUERY_PHRASE );
+      if( pTest->eType==FTSQUERY_NEAR ) pTest = pTest->pRight;
+      assert( pTest->eType==FTSQUERY_PHRASE );
+      pPh = pTest->pPhrase;
 
-    if( bEof || iDocid!=pCsr->iPrevId ) pIter = 0;
+      pIter = pPh->pOrPoslist;
+      iDocid = pPh->iOrDocid;
+      if( pCsr->bDesc==bDescDoclist ){
+        bEof = !pPh->doclist.nAll ||
+          (pIter >= (pPh->doclist.aAll + pPh->doclist.nAll));
+        while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)<0 ) && bEof==0 ){
+          sqlite3Fts3DoclistNext(
+              bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll, 
+              &pIter, &iDocid, &bEof
+          );
+        }
+      }else{
+        bEof = !pPh->doclist.nAll || (pIter && pIter<=pPh->doclist.aAll);
+        while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)>0 ) && bEof==0 ){
+          int dummy;
+          sqlite3Fts3DoclistPrev(
+              bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll, 
+              &pIter, &iDocid, &dummy, &bEof
+              );
+        }
+      }
+      pPh->pOrPoslist = pIter;
+      pPh->iOrDocid = iDocid;
+      if( bEof || iDocid!=pCsr->iPrevId ) bMatch = 0;
+    }
+
+    if( bMatch ){
+      pIter = pPhrase->pOrPoslist;
+    }else{
+      pIter = 0;
+    }
   }
   if( pIter==0 ) return SQLITE_OK;
 
diff --git a/manifest b/manifest
index 8742d04df9..cd16d59a37 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Enhance\sthe\spcache1\spage\scache\sso\sthat\sit\stries\sto\sallocate\sa\sblock\sof\nSQLITE_DEFAULT_PCACHE_INITSZ\spages\sfrom\smalloc()\son\sstartup,\sand\suses\sthose\npreallocated\spages\swhen\spossible\srather\sthan\sgoing\sto\smalloc()\sfor\seach\nindividual\spage.\s\sAbout\sa\s5%\sperformance\sincrease\sfor\ssome\sworkloads.
-D 2015-07-08T16:22:42.453
+C Fix\stwo\sproblems\sthat\scould\scause\sfts3\sauxiliary\sfunctions\sto\soccasionally\smisbehave\sif\sused\swith\smatch\sexpressions\sthat\scontain\sboth\sOR\sand\sNEAR.
+D 2015-07-08T17:59:08.894
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 017bf0511d1b2dd1db5e16488fbf75a17b526cbc
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -78,7 +78,7 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51
 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
 F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314
 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
-F ext/fts3/fts3.c a95de5190cf52f4fa9d5952890399cab63e632b9
+F ext/fts3/fts3.c d2f7981f4d7dfeb76aac82a15c7f37f425329c0f
 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
 F ext/fts3/fts3Int.h 601743955ac43a0e82e6828a931c07bb3b0c95ff
 F ext/fts3/fts3_aux.c 9edc3655fcb287f0467d0a4b886a01c6185fe9f1
@@ -691,13 +691,14 @@ F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a
 F test/fts3expr3.test 9e91b8edbcb197bf2e92161aa7696446d96dce5f
 F test/fts3expr4.test e1be1248566f43c252d4404d52914f1fc4bfa065
 F test/fts3expr5.test f9abfffbf5e53d48a33e12a1e8f8ba2c551c9b49
-F test/fts3fault.test cb72dccb0a3b9f730f16c5240f3fcb9303eb1660
+F test/fts3fault.test da49627b280b210ebc6657f76344c7851f10ce66
 F test/fts3fault2.test f953bb3cf903988172270a9a0aafd5a890b0f98f
 F test/fts3first.test dbdedd20914c8d539aa3206c9b34a23775644641
 F test/fts3join.test 53e66a0c21eb568580674a43b21c059acb26f499
 F test/fts3malloc.test b0e4c133b8d61d4f6d112d8110f8320e9e453ef6
 F test/fts3matchinfo.test 07009313ad6c082f94d8c9c3228eb8940c93ac71
 F test/fts3near.test 7e3354d46f155a822b59c0e957fd2a70c1d7e905
+F test/fts3offsets.test 5b8ec5be27dd2070af3538b23c67f1ca8c822853
 F test/fts3prefix.test fa794eaab0bdae466494947b0b153d7844478ab2
 F test/fts3prefix2.test e1f0a822ca661dced7f12ce392e14eaf65609dce
 F test/fts3query.test f33eb71a1fe1084ea585eeb7ee76b390729f5170
@@ -1364,8 +1365,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P b27a47fef995f0ab2516e6ca779711cab1f50fb9 c1e2ed0ef4d3436d19d3203c5fb553caac53659e
-R 0716d0744f0e72c8cdadbca41133bc14
-T +closed c1e2ed0ef4d3436d19d3203c5fb553caac53659e
-U drh
-Z eea1350426d30f63377f69d37a46ee1a
+P 5348ffc3fda5168c1e9e14aa88b0c6aedbda7c94
+R bb6c5e63b90e2eefb82b18295e953873
+U dan
+Z bead1c8f07e0e547f66cbd142b1cf6b7
diff --git a/manifest.uuid b/manifest.uuid
index a2762d1d63..7344ce86e2 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5348ffc3fda5168c1e9e14aa88b0c6aedbda7c94
\ No newline at end of file
+372c1db2475f367d54270d5801aff0503745bff4
\ No newline at end of file
diff --git a/test/fts3fault.test b/test/fts3fault.test
index ab82daa935..7d94332059 100644
--- a/test/fts3fault.test
+++ b/test/fts3fault.test
@@ -18,6 +18,8 @@ set ::testprefix fts3fault
 # If SQLITE_ENABLE_FTS3 is not defined, omit this file.
 ifcapable !fts3 { finish_test ; return }
 
+if 0 {
+
 # Test error handling in the sqlite3Fts3Init() function. This is the 
 # function that registers the FTS3 module and various support functions
 # with SQLite.
@@ -157,6 +159,9 @@ do_faultsim_test 7.3 -prep {
                        {1 {SQL logic error or missing database}}
 }
 
+
+}
+
 proc mit {blob} {
   set scan(littleEndian) i*
   set scan(bigEndian) I*
@@ -176,7 +181,7 @@ do_test 8.0 {
   faultsim_save_and_close
 } {}
 
-do_faultsim_test 8.1 -prep { 
+do_faultsim_test 8.1 -faults oom-t* -prep { 
   faultsim_restore_and_reopen
   db func mit mit
 } -body {
@@ -184,6 +189,7 @@ do_faultsim_test 8.1 -prep {
 } -test {
   faultsim_test_result {0 {{1 1 1 1 4 2 1 5 5}}}
 }
+
 do_faultsim_test 8.2 -faults oom-t* -prep { 
   faultsim_restore_and_reopen
   db func mit mit
diff --git a/test/fts3offsets.test b/test/fts3offsets.test
new file mode 100644
index 0000000000..184321ac01
--- /dev/null
+++ b/test/fts3offsets.test
@@ -0,0 +1,124 @@
+# 2010 November 02
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+# If SQLITE_ENABLE_FTS3 is not defined, omit this file.
+ifcapable !fts3 { finish_test ; return }
+
+set testprefix fts3offsets
+set sqlite_fts3_enable_parentheses 1
+
+proc extract {offsets text} {
+  set res ""
+
+  set off [list]
+  foreach {t i s n} $offsets {
+    lappend off [list $s $n]
+  }
+  set off [lsort -integer -index 0 $off]
+
+  set iOff 0
+  foreach e $off {
+    foreach {s n} $e {}
+    append res [string range $text $iOff $s-1]
+    append res "("
+    append res [string range $text $s [expr $s+$n-1]]
+    append res ")"
+    set iOff [expr $s+$n]
+  }
+  append res [string range $text $iOff end]
+  
+  set res
+}
+db func extract extract
+
+
+do_execsql_test 1.1.0 {
+  CREATE VIRTUAL TABLE xx USING fts3(x);
+  INSERT INTO xx VALUES('A x x x B C x x');
+  INSERT INTO xx VALUES('A B C x B x x C');
+  INSERT INTO xx VALUES('A x x B C x x x');
+}
+do_execsql_test 1.1.1 {
+  SELECT oid,extract(offsets(xx), x) FROM xx WHERE xx MATCH 'a OR (b NEAR/1 c)';
+} {
+  1 {(A) x x x (B) (C) x x} 
+  2 {(A) (B) (C) x (B) x x C} 
+  3 {(A) x x (B) (C) x x x}
+}
+
+do_execsql_test 1.2 {
+  DELETE FROM xx;
+  INSERT INTO xx VALUES('A x x x B C x x');
+  INSERT INTO xx VALUES('A x x C x x x C');
+  INSERT INTO xx VALUES('A x x B C x x x');
+}
+do_execsql_test 1.2.1 {
+  SELECT oid,extract(offsets(xx), x) FROM xx WHERE xx MATCH 'a OR (b NEAR/1 c)';
+} {
+  1 {(A) x x x (B) (C) x x}
+  2 {(A) x x C x x x C} 
+  3 {(A) x x (B) (C) x x x}
+}
+
+do_execsql_test 1.3 {
+  DELETE FROM xx;
+  INSERT INTO xx(rowid, x) VALUES(1, 'A B C');
+  INSERT INTO xx(rowid, x) VALUES(2, 'A x');
+  INSERT INTO xx(rowid, x) VALUES(3, 'A B C');
+  INSERT INTO xx(rowid, x) VALUES(4, 'A B C x x x x x x x B');
+  INSERT INTO xx(rowid, x) VALUES(5, 'A x x x x x x x x x C');
+  INSERT INTO xx(rowid, x) VALUES(6, 'A x x x x x x x x x x x B');
+  INSERT INTO xx(rowid, x) VALUES(7, 'A B C');
+}
+do_execsql_test 1.3.1 {
+  SELECT oid,extract(offsets(xx), x) FROM xx WHERE xx MATCH 'a OR (b NEAR/1 c)';
+} {
+  1 {(A) (B) (C)}
+  2 {(A) x}
+  3 {(A) (B) (C)}
+  4 {(A) (B) (C) x x x x x x x B}
+  5 {(A) x x x x x x x x x C}
+  6 {(A) x x x x x x x x x x x B} 
+  7 {(A) (B) (C)}
+}
+
+
+do_execsql_test 1.4 {
+  DELETE FROM xx;
+  INSERT INTO xx(rowid, x) VALUES(7, 'A B C');
+  INSERT INTO xx(rowid, x) VALUES(6, 'A x');
+  INSERT INTO xx(rowid, x) VALUES(5, 'A B C');
+  INSERT INTO xx(rowid, x) VALUES(4, 'A B C x x x x x x x B');
+  INSERT INTO xx(rowid, x) VALUES(3, 'A x x x x x x x x x C');
+  INSERT INTO xx(rowid, x) VALUES(2, 'A x x x x x x x x x x x B');
+  INSERT INTO xx(rowid, x) VALUES(1, 'A B C');
+}
+do_execsql_test 1.4.1 {
+  SELECT oid,extract(offsets(xx), x) FROM xx WHERE xx MATCH 'a OR (b NEAR/1 c)'
+  ORDER BY docid DESC;
+} {
+  7 {(A) (B) (C)}
+  6 {(A) x}
+  5 {(A) (B) (C)}
+  4 {(A) (B) (C) x x x x x x x B}
+  3 {(A) x x x x x x x x x C}
+  2 {(A) x x x x x x x x x x x B} 
+  1 {(A) (B) (C)}
+}
+
+
+set sqlite_fts3_enable_parentheses 0
+finish_test
+

From 3dbd2397f45988968bb547fb820098197c897db6 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 9 Jul 2015 19:02:19 +0000
Subject: [PATCH 34/40] Reduce the number of calls to malloc() made by fts5.

FossilOrigin-Name: 898618ccf61d3ad166d9fc742e132d518338b5c3
---
 ext/fts5/fts5Int.h            |  1 +
 ext/fts5/fts5_index.c         | 81 +++++++++++++++++++++++++++--------
 ext/fts5/test/fts5_common.tcl |  4 +-
 ext/fts5/test/fts5aa.test     |  2 +-
 ext/fts5/test/fts5rowid.test  | 13 +++---
 ext/fts5/tool/showfts5.tcl    | 22 ++++++----
 manifest                      | 22 +++++-----
 manifest.uuid                 |  2 +-
 8 files changed, 99 insertions(+), 48 deletions(-)

diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h
index c66984b286..074a2713f4 100644
--- a/ext/fts5/fts5Int.h
+++ b/ext/fts5/fts5Int.h
@@ -36,6 +36,7 @@ typedef sqlite3_uint64 u64;
 #define NEVER(x) 0
 
 #define MIN(x,y) (((x) < (y)) ? (x) : (y))
+#define MAX(x,y) (((x) > (y)) ? (x) : (y))
 
 #endif
 
diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 966ea0abd1..f2f68baa93 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -605,6 +605,21 @@ struct Fts5BtreeIter {
 };
 
 
+/*
+** The first argument passed to this macro is a pointer to an Fts5Buffer
+** object.
+*/
+#define fts5BufferSize(pBuf,n) {                \
+  if( pBuf->nSpace<n ) {                        \
+    u8 *pNew = sqlite3_realloc(pBuf->p, n);     \
+    if( pNew==0 ){                              \
+      sqlite3_free(pBuf->p);                    \
+    }                                           \
+    pBuf->nSpace = n;                           \
+    pBuf->p = pNew;                             \
+  }                                             \
+}
+
 static void fts5PutU16(u8 *aOut, u16 iVal){
   aOut[0] = (iVal>>8);
   aOut[1] = (iVal&0xFF);
@@ -724,16 +739,20 @@ static Fts5Data *fts5DataReadOrBuffer(
       u8 *aOut;                   /* Read blob data into this buffer */
       int nByte = sqlite3_blob_bytes(p->pReader);
       if( pBuf ){
-        fts5BufferZero(pBuf);
-        fts5BufferGrow(&rc, pBuf, nByte);
-        aOut = pBuf->p;
+        fts5BufferSize(pBuf, MAX(nByte, p->pConfig->pgsz) + 20);
         pBuf->n = nByte;
+        aOut = pBuf->p;
+        if( aOut==0 ){
+          rc = SQLITE_NOMEM;
+        }
       }else{
         int nSpace = nByte + FTS5_DATA_ZERO_PADDING;
-        pRet = (Fts5Data*)sqlite3Fts5MallocZero(&rc, nSpace+sizeof(Fts5Data));
+        pRet = (Fts5Data*)sqlite3_malloc(nSpace+sizeof(Fts5Data));
         if( pRet ){
           pRet->n = nByte;
           aOut = pRet->p = (u8*)&pRet[1];
+        }else{
+          rc = SQLITE_NOMEM;
         }
       }
 
@@ -1019,18 +1038,20 @@ static void fts5StructureExtendLevel(
 static Fts5Structure *fts5StructureRead(Fts5Index *p){
   Fts5Config *pConfig = p->pConfig;
   Fts5Structure *pRet = 0;        /* Object to return */
-  Fts5Data *pData;                /* %_data entry containing structure record */
   int iCookie;                    /* Configuration cookie */
+  Fts5Buffer buf = {0, 0, 0};
 
-  pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID);
-  if( !pData ) return 0;
-  p->rc = fts5StructureDecode(pData->p, pData->n, &iCookie, &pRet);
+  fts5DataBuffer(p, &buf, FTS5_STRUCTURE_ROWID);
+  if( buf.p==0 ) return 0;
+  assert( buf.nSpace>=(buf.n + FTS5_DATA_ZERO_PADDING) );
+  memset(&buf.p[buf.n], 0, FTS5_DATA_ZERO_PADDING);
+  p->rc = fts5StructureDecode(buf.p, buf.n, &iCookie, &pRet);
 
   if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){
     p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie);
   }
 
-  fts5DataRelease(pData);
+  fts5BufferFree(&buf);
   if( p->rc!=SQLITE_OK ){
     fts5StructureRelease(pRet);
     pRet = 0;
@@ -2028,7 +2049,7 @@ static void fts5SegIterLoadDlidx(Fts5Index *p, Fts5SegIter *pIter){
 
 #ifdef SQLITE_DEBUG
 static void fts5AssertNodeSeekOk(
-  Fts5Data *pNode,
+  Fts5Buffer *pNode,
   const u8 *pTerm, int nTerm,     /* Term to search for */
   int iExpectPg,
   int bExpectDlidx
@@ -2070,7 +2091,7 @@ static void fts5AssertNodeSeekOk(
 ** returned child page number has a doclist-index. Or left as is otherwise.
 */
 static int fts5NodeSeek(
-  Fts5Data *pNode,                /* Node to search */
+  Fts5Buffer *pNode,              /* Node to search */
   const u8 *pTerm, int nTerm,     /* Term to search for */
   int *pbDlidx                    /* OUT: True if dlidx flag is set */
 ){
@@ -2094,6 +2115,7 @@ static int fts5NodeSeek(
       *pbDlidx = (*pPtr==0x01);
       pPtr++;
       pPtr += fts5GetVarint32(pPtr, nEmpty);
+      if( pPtr>=pEnd ) break;
     }
 
     /* Read the next "term" pointer. Set nKeep to the number of bytes to
@@ -2230,7 +2252,10 @@ static void fts5LeafSeek(
       /* Skip past position list */
       fts5IndexGetVarint32(a, iOff, nPos);
       iOff += (nPos >> 1);
-      if( iOff>=n ) goto search_failed;
+      if( iOff>=(n-1) ){
+        iOff = n;
+        goto search_failed;
+      }
 
       /* If this is the end of the doclist, break out of the loop */
       if( a[iOff]==0x00 ){
@@ -2287,6 +2312,7 @@ static void fts5LeafSeek(
 */
 static void fts5SegIterSeekInit(
   Fts5Index *p,                   /* FTS5 backend */
+  Fts5Buffer *pBuf,               /* Buffer to use for loading pages */
   const u8 *pTerm, int nTerm,     /* Term to seek to */
   int flags,                      /* Mask of FTS5INDEX_XXX flags */
   Fts5StructureSegment *pSeg,     /* Description of segment */
@@ -2298,6 +2324,9 @@ static void fts5SegIterSeekInit(
   int bDlidx = 0;                 /* True if there is a doclist-index */
   Fts5Data *pLeaf;
 
+  static int nCall = 0;
+  nCall++;
+
   assert( bGe==0 || (flags & FTS5INDEX_QUERY_DESC)==0 );
   assert( pTerm && nTerm );
   memset(pIter, 0, sizeof(*pIter));
@@ -2307,11 +2336,9 @@ static void fts5SegIterSeekInit(
   ** contain term (pTerm/nTerm), if it is present in the segment. */
   for(h=pSeg->nHeight-1; h>0; h--){
     i64 iRowid = FTS5_SEGMENT_ROWID(pSeg->iSegid, h, iPg);
-    Fts5Data *pNode = fts5DataRead(p, iRowid);
-    if( pNode==0 ) break;
-
-    iPg = fts5NodeSeek(pNode, pTerm, nTerm, &bDlidx);
-    fts5DataRelease(pNode);
+    fts5DataBuffer(p, pBuf, iRowid);
+    if( p->rc ) break;
+    iPg = fts5NodeSeek(pBuf, pTerm, nTerm, &bDlidx);
   }
 
   if( iPg<pSeg->pgnoFirst ){
@@ -2340,6 +2367,20 @@ static void fts5SegIterSeekInit(
       }
     }
   }
+
+  /* Either:
+  **
+  **   1) an error has occurred, or
+  **   2) the iterator points to EOF, or
+  **   3) the iterator points to an entry with term (pTerm/nTerm), or
+  **   4) the FTS5INDEX_QUERY_SCAN flag was set and the iterator points
+  **      to an entry with a term greater than or equal to (pTerm/nTerm).
+  */
+  assert( p->rc!=SQLITE_OK                                          /* 1 */
+   || pIter->pLeaf==0                                               /* 2 */
+   || fts5BufferCompareBlob(&pIter->term, pTerm, nTerm)==0          /* 3 */
+   || (bGe && fts5BufferCompareBlob(&pIter->term, pTerm, nTerm)>0)  /* 4 */
+  );
 }
 
 /*
@@ -2795,6 +2836,7 @@ static void fts5MultiIterNew(
   int nSeg = 0;                   /* Number of segment-iters in use */
   int iIter = 0;                  /* */
   int iSeg;                       /* Used to iterate through segments */
+  Fts5Buffer buf = {0,0,0};       /* Buffer used by fts5SegIterSeekInit() */
   Fts5StructureLevel *pLvl;
   Fts5IndexIter *pNew;
 
@@ -2832,7 +2874,7 @@ static void fts5MultiIterNew(
         if( pTerm==0 ){
           fts5SegIterInit(p, pSeg, pIter);
         }else{
-          fts5SegIterSeekInit(p, pTerm, nTerm, flags, pSeg, pIter);
+          fts5SegIterSeekInit(p, &buf, pTerm, nTerm, flags, pSeg, pIter);
         }
       }
     }
@@ -2866,6 +2908,7 @@ static void fts5MultiIterNew(
     fts5MultiIterFree(p, pNew);
     *ppOut = 0;
   }
+  fts5BufferFree(&buf);
 }
 
 /*
@@ -5391,7 +5434,7 @@ static void fts5DebugStructure(
   for(iLvl=0; iLvl<p->nLevel; iLvl++){
     Fts5StructureLevel *pLvl = &p->aLevel[iLvl];
     sqlite3Fts5BufferAppendPrintf(pRc, pBuf, 
-        " {lvl=%d nMerge=%d", iLvl, pLvl->nMerge
+        " {lvl=%d nMerge=%d nSeg=%d", iLvl, pLvl->nMerge, pLvl->nSeg
     );
     for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){
       Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
diff --git a/ext/fts5/test/fts5_common.tcl b/ext/fts5/test/fts5_common.tcl
index a86222a94e..22aa3187fe 100644
--- a/ext/fts5/test/fts5_common.tcl
+++ b/ext/fts5/test/fts5_common.tcl
@@ -124,7 +124,7 @@ proc fts5_level_segs {tbl} {
   set sql "SELECT fts5_decode(rowid,block) aS r FROM ${tbl}_data WHERE rowid=10"
   set ret [list]
   foreach L [lrange [db one $sql] 1 end] {
-    lappend ret [expr [llength $L] - 2]
+    lappend ret [expr [llength $L] - 3]
   }
   set ret
 } 
@@ -134,7 +134,7 @@ proc fts5_level_segids {tbl} {
   set ret [list]
   foreach L [lrange [db one $sql] 1 end] {
     set lvl [list]
-    foreach S [lrange $L 2 end] {
+    foreach S [lrange $L 3 end] {
       regexp {id=([1234567890]*)} $S -> segid
       lappend lvl $segid
     }
diff --git a/ext/fts5/test/fts5aa.test b/ext/fts5/test/fts5aa.test
index a18005cfdb..70e086e8c9 100644
--- a/ext/fts5/test/fts5aa.test
+++ b/ext/fts5/test/fts5aa.test
@@ -49,7 +49,7 @@ do_execsql_test 2.1 {
 }
 do_test 2.2 {
   execsql { SELECT fts5_decode(id, block) FROM t1_data WHERE id==10 }
-} {/{\(structure\) {lvl=0 nMerge=0 {id=[0123456789]* h=1 leaves=1..1}}}/}
+} {/{\(structure\) {lvl=0 nMerge=0 nSeg=1 {id=[0123456789]* h=1 leaves=1..1}}}/}
 
 foreach w {a b c d e f} {
   do_execsql_test 2.3.$w.asc {
diff --git a/ext/fts5/test/fts5rowid.test b/ext/fts5/test/fts5rowid.test
index 9ea5272d5b..453d79867b 100644
--- a/ext/fts5/test/fts5rowid.test
+++ b/ext/fts5/test/fts5rowid.test
@@ -71,7 +71,8 @@ do_execsql_test 2.3 {
 } $res
 do_execsql_test 2.4 {
   UPDATE x1_data SET block = X'';
-  SELECT count(fts5_decode(rowid, block)) FROM x1_data;
+  -- SELECT count(fts5_decode(rowid, block)) FROM x1_data;
+  SELECT count(*) FROM x1_data;
 } $res
 
 do_execsql_test 2.5 {
@@ -83,10 +84,12 @@ set res [db one {SELECT count(*) FROM x1_data}]
 do_execsql_test 2.6 {
   SELECT count(fts5_decode(rowid, block)) FROM x1_data;
 } $res
-do_execsql_test 2.7 {
-  UPDATE x1_data SET block = X'';
-  SELECT count(fts5_decode(rowid, block)) FROM x1_data;
-} $res
+
+# This is really a corruption test...
+#do_execsql_test 2.7 {
+#  UPDATE x1_data SET block = X'';
+#  SELECT count(fts5_decode(rowid, block)) FROM x1_data;
+#} $res
 
 #-------------------------------------------------------------------------
 # Tests with very large tokens.
diff --git a/ext/fts5/tool/showfts5.tcl b/ext/fts5/tool/showfts5.tcl
index 57ed863b4e..846902b3be 100644
--- a/ext/fts5/tool/showfts5.tcl
+++ b/ext/fts5/tool/showfts5.tcl
@@ -1,28 +1,32 @@
 
 
+
+#-------------------------------------------------------------------------
+# Process command line arguments.
+#
 proc usage {} {
   puts stderr "usage: $::argv0 database table"
   puts stderr ""
   exit 1
 }
-
-set o(vtab)       fts5
-set o(tok)        ""
-set o(limit)      0
-set o(automerge)  -1
-set o(crisismerge)  -1
-
 if {[llength $argv]!=2} usage
-
 set database [lindex $argv 0]
 set tbl [lindex $argv 1]
 
+
+
+#-------------------------------------------------------------------------
+# Start of main program.
+#
 sqlite3 db $database
 catch { load_static_extension db fts5 }
 
 db eval "SELECT fts5_decode(rowid, block) AS d FROM ${tbl}_data WHERE id=10" {
   foreach lvl [lrange $d 1 end] {
-    puts $lvl
+    puts [lrange $lvl 0 2]
+    foreach seg [lrange $lvl 3 end] {
+      puts "        $seg"
+    }
   }
 }
 
diff --git a/manifest b/manifest
index cd16d59a37..78ab54d28a 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\stwo\sproblems\sthat\scould\scause\sfts3\sauxiliary\sfunctions\sto\soccasionally\smisbehave\sif\sused\swith\smatch\sexpressions\sthat\scontain\sboth\sOR\sand\sNEAR.
-D 2015-07-08T17:59:08.894
+C Reduce\sthe\snumber\sof\scalls\sto\smalloc()\smade\sby\sfts5.
+D 2015-07-09T19:02:19.523
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 017bf0511d1b2dd1db5e16488fbf75a17b526cbc
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -106,13 +106,13 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252
 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
 F ext/fts5/extract_api_docs.tcl 55a6d648d516f35d9a1e580ac00de27154e1904a
 F ext/fts5/fts5.h 81d1a92fc2b4bd477af7e4e0b38b456f3e199fba
-F ext/fts5/fts5Int.h c91773920639c01571d6870ac0c20e960798f79c
+F ext/fts5/fts5Int.h 582eb73064b5a2837d44e72907f84f94e86a2f79
 F ext/fts5/fts5_aux.c 7cd0e2858171dacf505fea4e2e84ee6126854c3d
 F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015
 F ext/fts5/fts5_config.c b2456e9625bca41c51d54c363e369c6356895c90
 F ext/fts5/fts5_expr.c d2e148345639c5a5583e0daa39a639bf298ae6a7
 F ext/fts5/fts5_hash.c 219f4edd72e5cf95b19c33f1058809a18fad5229
-F ext/fts5/fts5_index.c ab14b8a273ec72a4407eebfb787bbfb51acef565
+F ext/fts5/fts5_index.c ba397fe86f0b5ffdafe8eff71d5dbc85cc1180da
 F ext/fts5/fts5_main.c 37b0055cb4036c4b4bb4eb36e30ebd1c21c63939
 F ext/fts5/fts5_storage.c 4cae85b5287b159d9d98174a4e70adf872b0930a
 F ext/fts5/fts5_tcl.c 85eb4e0d0fefa9420b78151496ad4599a1783e20
@@ -122,8 +122,8 @@ F ext/fts5/fts5_varint.c 3f86ce09cab152e3d45490d7586b7ed2e40c13f1
 F ext/fts5/fts5_vocab.c 4e268a3fcbc099e50e335a1135be985a41ff6f7f
 F ext/fts5/fts5parse.y 833db1101b78c0c47686ab1b84918e38c36e9452
 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
-F ext/fts5/test/fts5_common.tcl 9553cce0757092d194307c2168d4edd100eab578
-F ext/fts5/test/fts5aa.test f7f057811eb0113d3259e059218bc85cef444280
+F ext/fts5/test/fts5_common.tcl e0b4a846a7670f6232a644ece69ef25a5c19c0e8
+F ext/fts5/test/fts5aa.test 4e896b9154764fed48179a87ba0bdf3650d7f49d
 F ext/fts5/test/fts5ab.test 6fe3a56731d15978afbb74ae51b355fc9310f2ad
 F ext/fts5/test/fts5ac.test 9737992d08c56bfd4803e933744d2d764e23795c
 F ext/fts5/test/fts5ad.test b2edee8b7de0c21d2c88f8a18c195034aad6952d
@@ -169,7 +169,7 @@ F ext/fts5/test/fts5prefix.test 552a462f0e8595676611f41643de217fb4ac2808
 F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1
 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b
 F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17
-F ext/fts5/test/fts5rowid.test f7674e19a40987bf59624d8db9827114cb7f7a3e
+F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460
 F ext/fts5/test/fts5tokenizer.test 83e7e01a21ec7fdf814d51f6184cc26bb77d7695
 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841
 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59
@@ -179,7 +179,7 @@ F ext/fts5/test/fts5version.test c54a708236642bcc850d2aedc6f505fef1d9f9f1
 F ext/fts5/test/fts5vocab.test cdf97b9678484e9bad5062edf9c9106e5c3b0c5c
 F ext/fts5/tool/loadfts5.tcl 95edf0b6b92a09f9ed85595038b1108127987556
 F ext/fts5/tool/mkfts5c.tcl 5745072c7de346e18c7f491e4c3281fe8a1cfe51
-F ext/fts5/tool/showfts5.tcl 8478fbe6cb93a8ff37335c9b48a38c4197dcfe5e
+F ext/fts5/tool/showfts5.tcl fb62e8eae6d862afdd22f367e286fb886d5e1ab6
 F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43
 F ext/icu/icu.c b2732aef0b076e4276d9b39b5a33cec7a05e1413
 F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37
@@ -1365,7 +1365,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 5348ffc3fda5168c1e9e14aa88b0c6aedbda7c94
-R bb6c5e63b90e2eefb82b18295e953873
+P 372c1db2475f367d54270d5801aff0503745bff4
+R cd5eb7bfe39c7cb521d188bde2e79a4e
 U dan
-Z bead1c8f07e0e547f66cbd142b1cf6b7
+Z 5190374d0e2aa885aa84b71a9146c526
diff --git a/manifest.uuid b/manifest.uuid
index 7344ce86e2..fb8de8c580 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-372c1db2475f367d54270d5801aff0503745bff4
\ No newline at end of file
+898618ccf61d3ad166d9fc742e132d518338b5c3
\ No newline at end of file

From d2454f47e768a980799b5efe005bb3f535bc8470 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 9 Jul 2015 20:46:35 +0000
Subject: [PATCH 35/40] Improve the performance of docid merges in fts5.

FossilOrigin-Name: b2de77a01cc5edcea2f98f7916e64cb33e6bd414
---
 ext/fts5/fts5Int.h    |  6 +++++
 ext/fts5/fts5_index.c | 60 ++++++++++++++++++++++++++++++-------------
 ext/fts5/fts5_main.c  |  8 ------
 manifest              | 16 ++++++------
 manifest.uuid         |  2 +-
 5 files changed, 57 insertions(+), 35 deletions(-)

diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h
index 074a2713f4..149f6b6694 100644
--- a/ext/fts5/fts5Int.h
+++ b/ext/fts5/fts5Int.h
@@ -38,6 +38,12 @@ typedef sqlite3_uint64 u64;
 #define MIN(x,y) (((x) < (y)) ? (x) : (y))
 #define MAX(x,y) (((x) > (y)) ? (x) : (y))
 
+/*
+** Constants for the largest and smallest possible 64-bit signed integers.
+*/
+# define LARGEST_INT64  (0xffffffff|(((i64)0x7fffffff)<<32))
+# define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64)
+
 #endif
 
 
diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index f2f68baa93..0b8f137ebf 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -514,6 +514,8 @@ struct Fts5IndexIter {
   int bRev;                       /* True to iterate in reverse order */
   int bSkipEmpty;                 /* True to skip deleted entries */
   int bEof;                       /* True at EOF */
+
+  i64 iSwitchRowid;               /* Firstest rowid of other than aFirst[1] */
   Fts5CResult *aFirst;            /* Current merge state (see above) */
   Fts5SegIter aSeg[1];            /* Array of segment iterators */
 };
@@ -2499,9 +2501,21 @@ static void fts5AssertComparisonResult(
 */
 static void fts5AssertMultiIterSetup(Fts5Index *p, Fts5IndexIter *pIter){
   if( p->rc==SQLITE_OK ){
+    Fts5SegIter *pFirst = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
     int i;
 
-    assert( (pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf==0)==pIter->bEof );
+    assert( (pFirst->pLeaf==0)==pIter->bEof );
+
+    /* Check that pIter->iSwitchRowid is set correctly. */
+    for(i=0; i<pIter->nSeg; i++){
+      Fts5SegIter *p1 = &pIter->aSeg[i];
+      assert( p1==pFirst 
+           || p1->pLeaf==0 
+           || fts5BufferCompare(&pFirst->term, &p1->term) 
+           || p1->iRowid==pIter->iSwitchRowid
+           || (p1->iRowid<pIter->iSwitchRowid)==pIter->bRev
+      );
+    }
 
     for(i=0; i<pIter->nSeg; i+=2){
       Fts5SegIter *p1 = &pIter->aSeg[i];
@@ -2719,27 +2733,35 @@ static int fts5MultiIterAdvanceRowid(
   Fts5IndexIter *pIter,           /* Iterator to update aFirst[] array for */
   int iChanged                    /* Index of sub-iterator just advanced */
 ){
-  int i;
   Fts5SegIter *pNew = &pIter->aSeg[iChanged];
-  Fts5SegIter *pOther = &pIter->aSeg[iChanged ^ 0x0001];
 
-  for(i=(pIter->nSeg+iChanged)/2; 1; i=i/2){
-    Fts5CResult *pRes = &pIter->aFirst[i];
+  if( pNew->iRowid==pIter->iSwitchRowid
+   || (pNew->iRowid<pIter->iSwitchRowid)==pIter->bRev
+  ){
+    int i;
+    Fts5SegIter *pOther = &pIter->aSeg[iChanged ^ 0x0001];
+    pIter->iSwitchRowid = pIter->bRev ? SMALLEST_INT64 : LARGEST_INT64;
+    for(i=(pIter->nSeg+iChanged)/2; 1; i=i/2){
+      Fts5CResult *pRes = &pIter->aFirst[i];
 
-    assert( pNew->pLeaf );
-    assert( pRes->bTermEq==0 || pOther->pLeaf );
-    
-    if( pRes->bTermEq ){
-      if( pNew->iRowid==pOther->iRowid ){
-        return 1;
-      }else if( (pOther->iRowid>pNew->iRowid)==pIter->bRev ){
-        pNew = pOther;
+      assert( pNew->pLeaf );
+      assert( pRes->bTermEq==0 || pOther->pLeaf );
+
+      if( pRes->bTermEq ){
+        if( pNew->iRowid==pOther->iRowid ){
+          return 1;
+        }else if( (pOther->iRowid>pNew->iRowid)==pIter->bRev ){
+          pIter->iSwitchRowid = pOther->iRowid;
+          pNew = pOther;
+        }else if( (pOther->iRowid>pIter->iSwitchRowid)==pIter->bRev ){
+          pIter->iSwitchRowid = pOther->iRowid;
+        }
       }
-    }
-    pRes->iFirst = (pNew - pIter->aSeg);
-    if( i==1 ) break;
+      pRes->iFirst = (pNew - pIter->aSeg);
+      if( i==1 ) break;
 
-    pOther = &pIter->aSeg[ pIter->aFirst[i ^ 0x0001].iFirst ];
+      pOther = &pIter->aSeg[ pIter->aFirst[i ^ 0x0001].iFirst ];
+    }
   }
 
   return 0;
@@ -2749,7 +2771,9 @@ static int fts5MultiIterAdvanceRowid(
 ** Set the pIter->bEof variable based on the state of the sub-iterators.
 */
 static void fts5MultiIterSetEof(Fts5IndexIter *pIter){
-  pIter->bEof = pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf==0;
+  Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
+  pIter->bEof = pSeg->pLeaf==0;
+  pIter->iSwitchRowid = pSeg->iRowid;
 }
 
 /*
diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c
index 05ae344479..ccb94bd8d6 100644
--- a/ext/fts5/fts5_main.c
+++ b/ext/fts5/fts5_main.c
@@ -227,14 +227,6 @@ struct Fts5Cursor {
 #define BitFlagAllTest(x,y) (((x) & (y))==(y))
 #define BitFlagTest(x,y)    (((x) & (y))!=0)
 
-/*
-** Constants for the largest and smallest possible 64-bit signed integers.
-** These are copied from sqliteInt.h.
-*/
-#ifndef SQLITE_AMALGAMATION
-# define LARGEST_INT64  (0xffffffff|(((i64)0x7fffffff)<<32))
-# define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64)
-#endif
 
 /*
 ** Macros to Set(), Clear() and Test() cursor flags.
diff --git a/manifest b/manifest
index 78ab54d28a..56929d7dc1 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Reduce\sthe\snumber\sof\scalls\sto\smalloc()\smade\sby\sfts5.
-D 2015-07-09T19:02:19.523
+C Improve\sthe\sperformance\sof\sdocid\smerges\sin\sfts5.
+D 2015-07-09T20:46:35.829
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 017bf0511d1b2dd1db5e16488fbf75a17b526cbc
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -106,14 +106,14 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252
 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
 F ext/fts5/extract_api_docs.tcl 55a6d648d516f35d9a1e580ac00de27154e1904a
 F ext/fts5/fts5.h 81d1a92fc2b4bd477af7e4e0b38b456f3e199fba
-F ext/fts5/fts5Int.h 582eb73064b5a2837d44e72907f84f94e86a2f79
+F ext/fts5/fts5Int.h 8d9bce1847a10df2e4ed9492ea4f3868276748fb
 F ext/fts5/fts5_aux.c 7cd0e2858171dacf505fea4e2e84ee6126854c3d
 F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015
 F ext/fts5/fts5_config.c b2456e9625bca41c51d54c363e369c6356895c90
 F ext/fts5/fts5_expr.c d2e148345639c5a5583e0daa39a639bf298ae6a7
 F ext/fts5/fts5_hash.c 219f4edd72e5cf95b19c33f1058809a18fad5229
-F ext/fts5/fts5_index.c ba397fe86f0b5ffdafe8eff71d5dbc85cc1180da
-F ext/fts5/fts5_main.c 37b0055cb4036c4b4bb4eb36e30ebd1c21c63939
+F ext/fts5/fts5_index.c 1a1fd996dfe2d632df1dd00689553bc0d205497d
+F ext/fts5/fts5_main.c 2e43726b3ef40b3d5efc0adc7c279d857b1c74fe
 F ext/fts5/fts5_storage.c 4cae85b5287b159d9d98174a4e70adf872b0930a
 F ext/fts5/fts5_tcl.c 85eb4e0d0fefa9420b78151496ad4599a1783e20
 F ext/fts5/fts5_tokenize.c 30f97a8c74683797b4cd233790444fbefb3b0708
@@ -1365,7 +1365,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 372c1db2475f367d54270d5801aff0503745bff4
-R cd5eb7bfe39c7cb521d188bde2e79a4e
+P 898618ccf61d3ad166d9fc742e132d518338b5c3
+R 67cbb00207cd0d10d2422bd26a9385a8
 U dan
-Z 5190374d0e2aa885aa84b71a9146c526
+Z ab9be791b136092de6b55e47372c8b3c
diff --git a/manifest.uuid b/manifest.uuid
index fb8de8c580..b82d7bb75b 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-898618ccf61d3ad166d9fc742e132d518338b5c3
\ No newline at end of file
+b2de77a01cc5edcea2f98f7916e64cb33e6bd414
\ No newline at end of file

From 3eb9024d68e70fd3a5816f17c2e293c922d37de2 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 10 Jul 2015 17:55:42 +0000
Subject: [PATCH 36/40] Fix inconsistencies in formatting of fts5 docs.

FossilOrigin-Name: 5fb4c77163be8d5ff1e2934dda3cc2ad0a6267e8
---
 ext/fts5/extract_api_docs.tcl | 19 ++++++++++++++-----
 manifest                      | 12 ++++++------
 manifest.uuid                 |  2 +-
 3 files changed, 21 insertions(+), 12 deletions(-)

diff --git a/ext/fts5/extract_api_docs.tcl b/ext/fts5/extract_api_docs.tcl
index 27f136a99b..afb2699be5 100644
--- a/ext/fts5/extract_api_docs.tcl
+++ b/ext/fts5/extract_api_docs.tcl
@@ -134,6 +134,7 @@ proc get_api_docs {data} {
   #
   set D [get_struct_docs $data [array names M]]
   
+  output "<dl>"
   foreach {sub docs} $D {
     if {[info exists M($sub)]} {
       set hdr $M($sub)
@@ -142,12 +143,17 @@ proc get_api_docs {data} {
       set link ""
     }
 
-    output "<hr color=#eeeee style=\"margin:1em 8.4ex 0 8.4ex;\"$link>"
-    set style "padding-left:6ex;font-size:1.4em;display:block"
-    output "<h style=\"$style\"><pre>$hdr</pre></h>"
+    #output "<hr color=#eeeee style=\"margin:1em 8.4ex 0 8.4ex;\"$link>"
+    #set style "padding-left:6ex;font-size:1.4em;display:block"
+    #output "<h style=\"$style\"><pre>$hdr</pre></h>"
+
+    regsub -line {^  *[)]} $hdr ")" hdr
+    output "<dt style=\"white-space:pre;font-family:monospace;font-size:120%\""
+    output "$link>"
+    output "<b>$hdr</b></dt><dd>"
   
     set mode ""
-    set bEmpty 1
+    set margin " style=margin-top:0.1em"
     foreach line [split [string trim $docs] "\n"] {
       if {[string trim $line]==""} {
         if {$mode != ""} {output "</$mode>"}
@@ -158,12 +164,15 @@ proc get_api_docs {data} {
         } else {
           set mode p
         }
-        output "<$mode>"
+        output "<$mode$margin>"
+        set margin ""
       }
       output $line
     }
     if {$mode != ""} {output "</$mode>"}
+    output "</dd>"
   }
+  output "</dl>"
 }
 
 proc get_fts5_struct {data start end} {
diff --git a/manifest b/manifest
index 56929d7dc1..7d1324ea39 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Improve\sthe\sperformance\sof\sdocid\smerges\sin\sfts5.
-D 2015-07-09T20:46:35.829
+C Fix\sinconsistencies\sin\sformatting\sof\sfts5\sdocs.
+D 2015-07-10T17:55:42.512
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 017bf0511d1b2dd1db5e16488fbf75a17b526cbc
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -104,7 +104,7 @@ F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c
 F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
 F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252
 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
-F ext/fts5/extract_api_docs.tcl 55a6d648d516f35d9a1e580ac00de27154e1904a
+F ext/fts5/extract_api_docs.tcl 06583c935f89075ea0b32f85efa5dd7619fcbd03
 F ext/fts5/fts5.h 81d1a92fc2b4bd477af7e4e0b38b456f3e199fba
 F ext/fts5/fts5Int.h 8d9bce1847a10df2e4ed9492ea4f3868276748fb
 F ext/fts5/fts5_aux.c 7cd0e2858171dacf505fea4e2e84ee6126854c3d
@@ -1365,7 +1365,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 898618ccf61d3ad166d9fc742e132d518338b5c3
-R 67cbb00207cd0d10d2422bd26a9385a8
+P b2de77a01cc5edcea2f98f7916e64cb33e6bd414
+R 7c08478f15e7cb3428a462d66cb461c5
 U dan
-Z ab9be791b136092de6b55e47372c8b3c
+Z eba5e30eba51a8866d4bf4dc68e41c5e
diff --git a/manifest.uuid b/manifest.uuid
index b82d7bb75b..f05b61bf94 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b2de77a01cc5edcea2f98f7916e64cb33e6bd414
\ No newline at end of file
+5fb4c77163be8d5ff1e2934dda3cc2ad0a6267e8
\ No newline at end of file

From 3547c5184c569e927a330c29d181b63ec319f831 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 13 Jul 2015 18:01:14 +0000
Subject: [PATCH 37/40] Fix the header comment on the loadfts.c utility
 program.

FossilOrigin-Name: 955ad4ae1978eeee1be9014fe2760c8519ad6206
---
 manifest       | 14 +++++++-------
 manifest.uuid  |  2 +-
 tool/loadfts.c |  6 +++++-
 3 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index 7d1324ea39..1abb778338 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sinconsistencies\sin\sformatting\sof\sfts5\sdocs.
-D 2015-07-10T17:55:42.512
+C Fix\sthe\sheader\scomment\son\sthe\sloadfts.c\sutility\sprogram.
+D 2015-07-13T18:01:14.191
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 017bf0511d1b2dd1db5e16488fbf75a17b526cbc
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -1322,7 +1322,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5
 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
 F tool/lemon.c b9109f59b57e7b6f101c4fe644c8361ba6dee969
 F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc
-F tool/loadfts.c 76b6589ab5efcdc9cfe16d43ab5a6c2618e44bd4
+F tool/loadfts.c f6443dd32e23fd9c846d0c286c1f6c7cb2c8c6ea
 F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6
 F tool/mkautoconfamal.sh d1a2da0e15b2ed33d60af35c7e9d483f13a8eb9f
 F tool/mkkeywordhash.c dfff09dbbfaf950e89af294f48f902181b144670
@@ -1365,7 +1365,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P b2de77a01cc5edcea2f98f7916e64cb33e6bd414
-R 7c08478f15e7cb3428a462d66cb461c5
-U dan
-Z eba5e30eba51a8866d4bf4dc68e41c5e
+P 5fb4c77163be8d5ff1e2934dda3cc2ad0a6267e8
+R d3824c43622cd5af3eb2d4500e444b7b
+U drh
+Z 5617f1fcb2290edff5681fccc612ced5
diff --git a/manifest.uuid b/manifest.uuid
index f05b61bf94..87dcea2412 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5fb4c77163be8d5ff1e2934dda3cc2ad0a6267e8
\ No newline at end of file
+955ad4ae1978eeee1be9014fe2760c8519ad6206
\ No newline at end of file
diff --git a/tool/loadfts.c b/tool/loadfts.c
index 5b2ed5dc6a..e4ccd1b329 100644
--- a/tool/loadfts.c
+++ b/tool/loadfts.c
@@ -1,5 +1,5 @@
 /*
-** 2013-06-10
+** 2015-07-28
 **
 ** The author disclaims copyright to this source code.  In place of
 ** a legal notice, here is a blessing:
@@ -9,6 +9,10 @@
 **    May you share freely, never taking more than you give.
 **
 *************************************************************************
+**
+** This file implements a utility program that will load many disk
+** files (all files under a given directory) into a FTS table.  This is
+** used for performance testing of FTS3, FTS4, and FTS5.
 */
 
 #include <stdio.h>

From fcd4325398ee1d02249f38097f106f4e3dd29dd2 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 13 Jul 2015 23:55:08 +0000
Subject: [PATCH 38/40] Fix the origination date on the loadfts.c utility
 program.

FossilOrigin-Name: 5c76c062c0cbf7c95897c5de4868172292023303
---
 manifest       | 12 ++++++------
 manifest.uuid  |  2 +-
 tool/loadfts.c |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 1abb778338..00f46b14c8 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sthe\sheader\scomment\son\sthe\sloadfts.c\sutility\sprogram.
-D 2015-07-13T18:01:14.191
+C Fix\sthe\sorigination\sdate\son\sthe\sloadfts.c\sutility\sprogram.
+D 2015-07-13T23:55:08.368
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 017bf0511d1b2dd1db5e16488fbf75a17b526cbc
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -1322,7 +1322,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5
 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
 F tool/lemon.c b9109f59b57e7b6f101c4fe644c8361ba6dee969
 F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc
-F tool/loadfts.c f6443dd32e23fd9c846d0c286c1f6c7cb2c8c6ea
+F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862
 F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6
 F tool/mkautoconfamal.sh d1a2da0e15b2ed33d60af35c7e9d483f13a8eb9f
 F tool/mkkeywordhash.c dfff09dbbfaf950e89af294f48f902181b144670
@@ -1365,7 +1365,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 5fb4c77163be8d5ff1e2934dda3cc2ad0a6267e8
-R d3824c43622cd5af3eb2d4500e444b7b
+P 955ad4ae1978eeee1be9014fe2760c8519ad6206
+R fb02aef78cc5a7388816eff210ab1fba
 U drh
-Z 5617f1fcb2290edff5681fccc612ced5
+Z 21eb497bf685df9f637399e6a83cd3d7
diff --git a/manifest.uuid b/manifest.uuid
index 87dcea2412..cf262adb03 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-955ad4ae1978eeee1be9014fe2760c8519ad6206
\ No newline at end of file
+5c76c062c0cbf7c95897c5de4868172292023303
\ No newline at end of file
diff --git a/tool/loadfts.c b/tool/loadfts.c
index e4ccd1b329..0000797b88 100644
--- a/tool/loadfts.c
+++ b/tool/loadfts.c
@@ -1,5 +1,5 @@
 /*
-** 2015-07-28
+** 2014-07-28
 **
 ** The author disclaims copyright to this source code.  In place of
 ** a legal notice, here is a blessing:

From ebad80e3733d223b5b54dc3c76bb0ef818715bd2 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Tue, 14 Jul 2015 00:36:34 +0000
Subject: [PATCH 39/40] Refine cleaning of FTS5 files by Makefile targets.

FossilOrigin-Name: e548d77b3c91cdf11c78d1a688fd768e209bdbf5
---
 Makefile.in   |  2 +-
 Makefile.msc  |  2 +-
 main.mk       |  2 +-
 manifest      | 18 +++++++++---------
 manifest.uuid |  2 +-
 5 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/Makefile.in b/Makefile.in
index 3aaa4b5827..0cbd80cc4b 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -1189,7 +1189,7 @@ clean:
 	rm -f fuzzershell fuzzershell.exe
 	rm -f fuzzcheck fuzzcheck.exe
 	rm -f sqldiff sqldiff.exe
-	rm -f fts5.c fts5.h fts5parse.c fts5parse.h
+	rm -f fts5.c fts5.h fts5parse.*
 
 distclean:	clean
 	rm -f config.log config.status libtool Makefile sqlite3.pc
diff --git a/Makefile.msc b/Makefile.msc
index 7a57489870..02a3330a24 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -1833,7 +1833,7 @@ clean:
 	del /Q sqlite3_analyzer.exe sqlite3_analyzer.c 2>NUL
 	del /Q sqlite-*-output.vsix 2>NUL
 	del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe 2>NUL
-	del /Q fts5.c fts5.h fts5parse.c fts5parse.h 2>NUL
+	del /Q fts5.c fts5.h fts5parse.* 2>NUL
 
 # Dynamic link library section.
 #
diff --git a/main.mk b/main.mk
index 3c248996df..4b84873316 100644
--- a/main.mk
+++ b/main.mk
@@ -869,4 +869,4 @@ clean:
 	rm -f fuzzershell fuzzershell.exe
 	rm -f fuzzcheck fuzzcheck.exe
 	rm -f sqldiff sqldiff.exe
-	rm -f fts5.c fts5.h fts5parse.c fts5parse.h
+	rm -f fts5.c fts5.h fts5parse.*
diff --git a/manifest b/manifest
index 00f46b14c8..c87e7f0d2b 100644
--- a/manifest
+++ b/manifest
@@ -1,9 +1,9 @@
-C Fix\sthe\sorigination\sdate\son\sthe\sloadfts.c\sutility\sprogram.
-D 2015-07-13T23:55:08.368
+C Refine\scleaning\sof\sFTS5\sfiles\sby\sMakefile\stargets.
+D 2015-07-14T00:36:34.209
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
-F Makefile.in 017bf0511d1b2dd1db5e16488fbf75a17b526cbc
+F Makefile.in 2a4a94d9c6ad8cde388b672d89d1463e6f38a6da
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
-F Makefile.msc e97727cdfab1d4b890eb1f7e9d3f11a6717d329d
+F Makefile.msc 86926e2330ad4b7ae6d4674762b53b293873b3fb
 F Makefile.vxworks e1b65dea203f054e71653415bd8f96dcaed47858
 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7
 F VERSION ce0ae95abd7121c534f6917c1c8f2b70d9acd4db
@@ -248,7 +248,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 2fda07d2e73021aaa98aeaf98e4b39b2ef2d5ccb
+F main.mk 551665e0bf7bd275a3e49dc8569ccb17098f7179
 F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea
 F mkopcodeh.awk 0e7f04a8eb90f92259e47d80110e4e98d7ce337a
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
@@ -1365,7 +1365,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 955ad4ae1978eeee1be9014fe2760c8519ad6206
-R fb02aef78cc5a7388816eff210ab1fba
-U drh
-Z 21eb497bf685df9f637399e6a83cd3d7
+P 5c76c062c0cbf7c95897c5de4868172292023303
+R 426b5fccc68ae98e50db8b48fd4e7d32
+U mistachkin
+Z 30f4df410df2e184bde2b63910ad9db6
diff --git a/manifest.uuid b/manifest.uuid
index cf262adb03..649bb12860 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5c76c062c0cbf7c95897c5de4868172292023303
\ No newline at end of file
+e548d77b3c91cdf11c78d1a688fd768e209bdbf5
\ No newline at end of file

From 201e0c68f7d5d19ba759f56fadd0d58c838c41f9 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 14 Jul 2015 14:48:50 +0000
Subject: [PATCH 40/40] Always invoke the profile callback even if the
 statement does not run to completion.

FossilOrigin-Name: 202479aa0a62067343e724487960b8a039e2e978
---
 manifest        | 16 ++++++++--------
 manifest.uuid   |  2 +-
 src/vdbeapi.c   | 50 +++++++++++++++++++++++++++++++++++++------------
 test/trace.test |  8 ++++++++
 4 files changed, 55 insertions(+), 21 deletions(-)

diff --git a/manifest b/manifest
index c87e7f0d2b..a07ab9a0dd 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Refine\scleaning\sof\sFTS5\sfiles\sby\sMakefile\stargets.
-D 2015-07-14T00:36:34.209
+C Always\sinvoke\sthe\sprofile\scallback\seven\sif\sthe\sstatement\sdoes\snot\srun\sto\ncompletion.
+D 2015-07-14T14:48:50.217
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 2a4a94d9c6ad8cde388b672d89d1463e6f38a6da
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -393,7 +393,7 @@ F src/vacuum.c 2ddd5cad2a7b9cef7f9e431b8c7771634c6b1701
 F src/vdbe.c 5ee4a2bf871418f61d06dc256b9b3a0084b5ec46
 F src/vdbe.h 7a75045d879118b9d3af7e8b3c108f2f27c51473
 F src/vdbeInt.h 8b54e01ad0463590e7cffabce0bc36da9ee4f816
-F src/vdbeapi.c 6a0d7757987018ff6b1b81bc5293219cd26bb299
+F src/vdbeapi.c 2fa7ed7520313d6dd1dc6e6495999201c630adfc
 F src/vdbeaux.c 787f5f9d58f4c6f39294ed06909ba602d1a402e6
 F src/vdbeblob.c 4f2e8e075d238392df98c5e03a64342465b03f90
 F src/vdbemem.c ae38a0d35ae71cf604381a887c170466ba518090
@@ -1183,7 +1183,7 @@ F test/tkt3997.test a335fa41ca3985660a139df7b734a26ef53284bd
 F test/tkt4018.test 7c2c9ba4df489c676a0a7a0e809a1fb9b2185bd1
 F test/tokenize.test ce430a7aed48fc98301611429595883fdfcab5d7
 F test/tpch01.test 04adbf8d8300fa60a222f28d901abd76e7be6dd4
-F test/trace.test 73a5508100f7fccfbc3f8018d5f6963ed478eea0
+F test/trace.test 6f676313e3ebd2a50585036d2f212a3319dd5836
 F test/trace2.test f5cb67ad3bc09e0c58e8cca78dfd0b5639259983
 F test/trans.test 6e1b4c6a42dba31bd65f8fa5e61a2708e08ddde6
 F test/trans2.test 62bd045bfc7a1c14c5ba83ba64d21ade31583f76
@@ -1365,7 +1365,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 5c76c062c0cbf7c95897c5de4868172292023303
-R 426b5fccc68ae98e50db8b48fd4e7d32
-U mistachkin
-Z 30f4df410df2e184bde2b63910ad9db6
+P e548d77b3c91cdf11c78d1a688fd768e209bdbf5
+R f434f0b9092147e4e9389e2958ccbd33
+U drh
+Z 3bea78814fd8f9c4678ec13302736bdc
diff --git a/manifest.uuid b/manifest.uuid
index 649bb12860..e4e6a3512a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e548d77b3c91cdf11c78d1a688fd768e209bdbf5
\ No newline at end of file
+202479aa0a62067343e724487960b8a039e2e978
\ No newline at end of file
diff --git a/src/vdbeapi.c b/src/vdbeapi.c
index 010fbfba16..4687aac510 100644
--- a/src/vdbeapi.c
+++ b/src/vdbeapi.c
@@ -53,6 +53,31 @@ static int vdbeSafetyNotNull(Vdbe *p){
   }
 }
 
+#ifndef SQLITE_OMIT_TRACE
+/*
+** Invoke the profile callback.  This routine is only called if we already
+** know that the profile callback is defined and needs to be invoked.
+*/
+static SQLITE_NOINLINE void invokeProfileCallback(sqlite3 *db, Vdbe *p){
+  sqlite3_int64 iNow;
+  assert( p->startTime>0 );
+  assert( db->xProfile!=0 );
+  assert( db->init.busy==0 );
+  assert( p->zSql!=0 );
+  sqlite3OsCurrentTimeInt64(db->pVfs, &iNow);
+  db->xProfile(db->pProfileArg, p->zSql, (iNow - p->startTime)*1000000);
+  p->startTime = 0;
+}
+/*
+** The checkProfileCallback(DB,P) macro checks to see if a profile callback
+** is needed, and it invokes the callback if it is needed.
+*/
+# define checkProfileCallback(DB,P) \
+   if( ((P)->startTime)>0 ){ invokeProfileCallback(DB,P); }
+#else
+# define checkProfileCallback(DB,P)  /*no-op*/
+#endif
+
 /*
 ** The following routine destroys a virtual machine that is created by
 ** the sqlite3_compile() routine. The integer returned is an SQLITE_
@@ -73,6 +98,7 @@ int sqlite3_finalize(sqlite3_stmt *pStmt){
     sqlite3 *db = v->db;
     if( vdbeSafety(v) ) return SQLITE_MISUSE_BKPT;
     sqlite3_mutex_enter(db->mutex);
+    checkProfileCallback(db, v);
     rc = sqlite3VdbeFinalize(v);
     rc = sqlite3ApiExit(db, rc);
     sqlite3LeaveMutexAndCloseZombie(db);
@@ -94,12 +120,14 @@ int sqlite3_reset(sqlite3_stmt *pStmt){
     rc = SQLITE_OK;
   }else{
     Vdbe *v = (Vdbe*)pStmt;
-    sqlite3_mutex_enter(v->db->mutex);
+    sqlite3 *db = v->db;
+    sqlite3_mutex_enter(db->mutex);
+    checkProfileCallback(db, v);
     rc = sqlite3VdbeReset(v);
     sqlite3VdbeRewind(v);
-    assert( (rc & (v->db->errMask))==rc );
-    rc = sqlite3ApiExit(v->db, rc);
-    sqlite3_mutex_leave(v->db->mutex);
+    assert( (rc & (db->errMask))==rc );
+    rc = sqlite3ApiExit(db, rc);
+    sqlite3_mutex_leave(db->mutex);
   }
   return rc;
 }
@@ -450,6 +478,7 @@ static int doWalCallbacks(sqlite3 *db){
   return rc;
 }
 
+
 /*
 ** Execute the statement pStmt, either until a row of data is ready, the
 ** statement is completely executed or an error occurs.
@@ -518,8 +547,10 @@ static int sqlite3Step(Vdbe *p){
     );
 
 #ifndef SQLITE_OMIT_TRACE
-    if( db->xProfile && !db->init.busy ){
+    if( db->xProfile && !db->init.busy && p->zSql ){
       sqlite3OsCurrentTimeInt64(db->pVfs, &p->startTime);
+    }else{
+      assert( p->startTime==0 );
     }
 #endif
 
@@ -543,13 +574,8 @@ static int sqlite3Step(Vdbe *p){
   }
 
 #ifndef SQLITE_OMIT_TRACE
-  /* Invoke the profile callback if there is one
-  */
-  if( rc!=SQLITE_ROW && db->xProfile && !db->init.busy && p->zSql ){
-    sqlite3_int64 iNow;
-    sqlite3OsCurrentTimeInt64(db->pVfs, &iNow);
-    db->xProfile(db->pProfileArg, p->zSql, (iNow - p->startTime)*1000000);
-  }
+  /* If the statement completed successfully, invoke the profile callback */
+  if( rc!=SQLITE_ROW ) checkProfileCallback(db, p);
 #endif
 
   if( rc==SQLITE_DONE ){
diff --git a/test/trace.test b/test/trace.test
index a64cc333fa..fd51d7ab64 100644
--- a/test/trace.test
+++ b/test/trace.test
@@ -169,6 +169,14 @@ do_test trace-4.5 {
 } {{SELECT * FROM t1}}
 catch {sqlite3_finalize $STMT}
 
+# 3.8.11: Profile output even if the statement is not run to completion.
+do_test trace-4.6 {
+  set TRACE_OUT {}
+  db eval {SELECT * FROM t1} {} {if {$a>=1} break}
+  set TRACE_OUT
+} {{SELECT * FROM t1}}
+
+
 # Trigger tracing.
 #
 ifcapable trigger {