Merge latest trunk changes into this branch.

FossilOrigin-Name: 2ac72114a1f5344b42472b941c60f460c28c981a22ea40909b30f7bf4eb4b11b

.fossil-settings/empty-dirs

@@ -0,0 +1 @@
+compat

.fossil-settings/ignore-glob

@@ -0,0 +1 @@
+compat/*

Makefile.in

@@ -89,6 +89,9 @@ TCC += $(OPT_FEATURE_FLAGS)
 # ie.  make "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1".
 TCC += $(OPTS)
 
+# Add in compile-time options for some libraries used by extensions
+TCC += @HAVE_ZLIB@
+
 # Version numbers and release number for the SQLite being compiled.
 #
 VERSION = @VERSION@
@@ -166,7 +169,8 @@ USE_AMALGAMATION = @USE_AMALGAMATION@
 #
 LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \
          backup.lo bitvec.lo btmutex.lo btree.lo build.lo \
-         callback.lo complete.lo ctime.lo date.lo dbstat.lo delete.lo \
+         callback.lo complete.lo ctime.lo \
+         date.lo dbpage.lo dbstat.lo delete.lo \
          expr.lo fault.lo fkey.lo \
          fts3.lo fts3_aux.lo fts3_expr.lo fts3_hash.lo fts3_icu.lo \
          fts3_porter.lo fts3_snippet.lo fts3_tokenizer.lo fts3_tokenizer1.lo \
@@ -176,17 +180,17 @@ LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \
          func.lo global.lo hash.lo \
          icu.lo insert.lo json1.lo legacy.lo loadext.lo \
          main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \
-         memjournal.lo \
+         memdb.lo memjournal.lo \
          mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \
          notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \
          pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
          random.lo resolve.lo rowset.lo rtree.lo \
          sqlite3session.lo select.lo sqlite3rbu.lo status.lo stmt.lo \
          table.lo threads.lo tokenize.lo treeview.lo trigger.lo \
-         update.lo util.lo vacuum.lo \
+         update.lo upsert.lo util.lo vacuum.lo \
          vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
          vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \
-         utf.lo vtab.lo
+         window.lo utf.lo vtab.lo
 
 # Object files for the amalgamation.
 #
@@ -215,6 +219,7 @@ SRC = \
   $(TOP)/src/complete.c \
   $(TOP)/src/ctime.c \
   $(TOP)/src/date.c \
+  $(TOP)/src/dbpage.c \
   $(TOP)/src/dbstat.c \
   $(TOP)/src/delete.c \
   $(TOP)/src/expr.c \
@@ -235,6 +240,7 @@ SRC = \
   $(TOP)/src/mem2.c \
   $(TOP)/src/mem3.c \
   $(TOP)/src/mem5.c \
+  $(TOP)/src/memdb.c \
   $(TOP)/src/memjournal.c \
   $(TOP)/src/msvc.h \
   $(TOP)/src/mutex.c \
@@ -265,7 +271,7 @@ SRC = \
   $(TOP)/src/rowset.c \
   $(TOP)/src/select.c \
   $(TOP)/src/status.c \
-  $(TOP)/src/shell.c \
+  $(TOP)/src/shell.c.in \
   $(TOP)/src/sqlite.h.in \
   $(TOP)/src/sqlite3ext.h \
   $(TOP)/src/sqliteInt.h \
@@ -278,6 +284,7 @@ SRC = \
   $(TOP)/src/trigger.c \
   $(TOP)/src/utf.c \
   $(TOP)/src/update.c \
+  $(TOP)/src/upsert.c \
   $(TOP)/src/util.c \
   $(TOP)/src/vacuum.c \
   $(TOP)/src/vdbe.c \
@@ -297,7 +304,8 @@ SRC = \
   $(TOP)/src/where.c \
   $(TOP)/src/wherecode.c \
   $(TOP)/src/whereexpr.c \
-  $(TOP)/src/whereInt.h
+  $(TOP)/src/whereInt.h \
+  $(TOP)/src/window.c
 
 # Source code for extensions
 #
@@ -342,7 +350,8 @@ SRC += \
   $(TOP)/ext/icu/icu.c
 SRC += \
   $(TOP)/ext/rtree/rtree.h \
-  $(TOP)/ext/rtree/rtree.c
+  $(TOP)/ext/rtree/rtree.c \
+  $(TOP)/ext/rtree/geopoly.c
 SRC += \
   $(TOP)/ext/session/sqlite3session.c \
   $(TOP)/ext/session/sqlite3session.h
@@ -362,6 +371,7 @@ SRC += \
   parse.c \
   parse.h \
   config.h \
+  shell.c \
   sqlite3.h
 
 # Source code to the test files.
@@ -393,6 +403,7 @@ TESTSRC = \
   $(TOP)/src/test_intarray.c \
   $(TOP)/src/test_journal.c \
   $(TOP)/src/test_malloc.c \
+  $(TOP)/src/test_md5.c \
   $(TOP)/src/test_multiplex.c \
   $(TOP)/src/test_mutex.c \
   $(TOP)/src/test_onefile.c \
@@ -404,10 +415,12 @@ TESTSRC = \
   $(TOP)/src/test_server.c \
   $(TOP)/src/test_superlock.c \
   $(TOP)/src/test_syscall.c \
+  $(TOP)/src/test_tclsh.c \
   $(TOP)/src/test_tclvar.c \
   $(TOP)/src/test_thread.c \
   $(TOP)/src/test_vfs.c \
   $(TOP)/src/test_windirent.c \
+  $(TOP)/src/test_window.c \
   $(TOP)/src/test_wsd.c       \
   $(TOP)/ext/fts3/fts3_term.c \
   $(TOP)/ext/fts3/fts3_test.c  \
@@ -417,18 +430,23 @@ TESTSRC = \
 # Statically linked extensions
 #
 TESTSRC += \
+  $(TOP)/ext/expert/sqlite3expert.c \
+  $(TOP)/ext/expert/test_expert.c \
   $(TOP)/ext/misc/amatch.c \
   $(TOP)/ext/misc/carray.c \
   $(TOP)/ext/misc/closure.c \
   $(TOP)/ext/misc/csv.c \
   $(TOP)/ext/misc/eval.c \
+  $(TOP)/ext/misc/explain.c \
   $(TOP)/ext/misc/fileio.c \
   $(TOP)/ext/misc/fuzzer.c \
   $(TOP)/ext/fts5/fts5_tcl.c \
   $(TOP)/ext/fts5/fts5_test_mi.c \
   $(TOP)/ext/fts5/fts5_test_tok.c \
   $(TOP)/ext/misc/ieee754.c \
+  $(TOP)/ext/misc/mmapwarm.c \
   $(TOP)/ext/misc/nextchar.c \
+  $(TOP)/ext/misc/normalize.c \
   $(TOP)/ext/misc/percentile.c \
   $(TOP)/ext/misc/regexp.c \
   $(TOP)/ext/misc/remember.c \
@@ -436,7 +454,8 @@ TESTSRC += \
   $(TOP)/ext/misc/spellfix.c \
   $(TOP)/ext/misc/totype.c \
   $(TOP)/ext/misc/unionvtab.c \
-  $(TOP)/ext/misc/wholenumber.c
+  $(TOP)/ext/misc/wholenumber.c \
+  $(TOP)/ext/misc/zipfile.c
 
 # Source code to the library files needed by the test fixture
 #
@@ -448,6 +467,7 @@ TESTSRC2 = \
   $(TOP)/src/build.c \
   $(TOP)/src/ctime.c \
   $(TOP)/src/date.c \
+  $(TOP)/src/dbpage.c \
   $(TOP)/src/dbstat.c \
   $(TOP)/src/expr.c \
   $(TOP)/src/func.c \
@@ -477,6 +497,7 @@ TESTSRC2 = \
   $(TOP)/src/where.c \
   $(TOP)/src/wherecode.c \
   $(TOP)/src/whereexpr.c \
+  $(TOP)/src/window.c \
   parse.c \
   $(TOP)/ext/fts3/fts3.c \
   $(TOP)/ext/fts3/fts3_aux.c \
@@ -533,7 +554,8 @@ EXTHDR += \
   $(TOP)/ext/fts3/fts3_hash.h \
   $(TOP)/ext/fts3/fts3_tokenizer.h
 EXTHDR += \
-  $(TOP)/ext/rtree/rtree.h
+  $(TOP)/ext/rtree/rtree.h \
+  $(TOP)/ext/rtree/geopoly.c
 EXTHDR += \
   $(TOP)/ext/icu/sqliteicu.h
 EXTHDR += \
@@ -546,7 +568,8 @@ TESTPROGS = \
   sqlite3$(TEXE) \
   sqlite3_analyzer$(TEXE) \
   sqldiff$(TEXE) \
-  dbhash$(TEXE)
+  dbhash$(TEXE) \
+  sqltclsh$(TEXE)
 
 # Databases containing fuzzer test cases
 #
@@ -555,7 +578,8 @@ FUZZDATA = \
   $(TOP)/test/fuzzdata2.db \
   $(TOP)/test/fuzzdata3.db \
   $(TOP)/test/fuzzdata4.db \
-  $(TOP)/test/fuzzdata5.db
+  $(TOP)/test/fuzzdata5.db \
+  $(TOP)/test/fuzzdata6.db
 
 # Standard options to testfixture
 #
@@ -564,13 +588,19 @@ TESTOPTS = --verbose=file --output=test-out.txt
 # Extra compiler options for various shell tools
 #
 SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4
-# SHELL_OPT += -DSQLITE_ENABLE_FTS5
+#SHELL_OPT += -DSQLITE_ENABLE_FTS5
+SHELL_OPT += -DSQLITE_ENABLE_RTREE
 SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS
 SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
 SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB
+SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB
+SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
+SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC
+SHELL_OPT += -DSQLITE_INTROSPECTION_PRAGMAS
 FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
 FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ
 FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
+FUZZCHECK_OPT += -DSQLITE_PRINTF_PRECISION_LIMIT=1000
 FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c
 DBFUZZ_OPT = 
 
@@ -596,9 +626,9 @@ libtclsqlite3.la:	tclsqlite.lo libsqlite3.la
 		-version-info "8:6:8" \
 		-avoid-version
 
-sqlite3$(TEXE):	$(TOP)/src/shell.c sqlite3.c
+sqlite3$(TEXE):	shell.c sqlite3.c
 	$(LTLINK) $(READLINE_FLAGS) $(SHELL_OPT) -o $@ \
-		$(TOP)/src/shell.c sqlite3.c \
+		shell.c sqlite3.c \
 		$(LIBREADLINE) $(TLIBS) -rpath "$(libdir)"
 
 sqldiff$(TEXE):	$(TOP)/tool/sqldiff.c sqlite3.lo sqlite3.h
@@ -628,6 +658,9 @@ ossshell$(TEXE):	$(TOP)/test/ossfuzz.c $(TOP)/test/ossshell.c sqlite3.c sqlite3.
 	$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/ossshell.c \
              $(TOP)/test/ossfuzz.c sqlite3.c $(TLIBS)
 
+sessionfuzz$(TEXE):	$(TOP)/test/sessionfuzz.c sqlite3.c sqlite3.h
+	$(CC) $(CFLAGS) -I. -o $@ $(TOP)/test/sessionfuzz.c $(TLIBS)
+
 dbfuzz$(TEXE):	$(TOP)/test/dbfuzz.c sqlite3.c sqlite3.h
 	$(LTLINK) -o $@ $(DBFUZZ_OPT) $(TOP)/test/dbfuzz.c sqlite3.c $(TLIBS)
 
@@ -667,7 +700,7 @@ mptest:	mptester$(TEXE)
 
 sqlite3.c:	.target_source $(TOP)/tool/mksqlite3c.tcl
 	$(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl
-	cp tsrc/shell.c tsrc/sqlite3ext.h .
+	cp tsrc/sqlite3ext.h .
 	cp $(TOP)/ext/session/sqlite3session.h .
 
 sqlite3ext.h:	.target_source
@@ -693,6 +726,11 @@ lemon$(BEXE):	$(TOP)/tool/lemon.c $(TOP)/tool/lempar.c
 	$(BCC) -o $@ $(TOP)/tool/lemon.c
 	cp $(TOP)/tool/lempar.c .
 
+# Rules to build the program that generates the source-id
+#
+mksourceid$(BEXE):	$(TOP)/tool/mksourceid.c
+	$(BCC) -o $@ $(TOP)/tool/mksourceid.c
+
 # Rules to build individual *.o files from generated *.c files. This
 # applies to:
 #
@@ -746,6 +784,9 @@ ctime.lo:	$(TOP)/src/ctime.c $(HDR)
 date.lo:	$(TOP)/src/date.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/date.c
 
+dbpage.lo:	$(TOP)/src/dbpage.c $(HDR)
+	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/dbpage.c
+
 dbstat.lo:	$(TOP)/src/dbstat.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/dbstat.c
 
@@ -800,6 +841,9 @@ mem3.lo:	$(TOP)/src/mem3.c $(HDR)
 mem5.lo:	$(TOP)/src/mem5.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/mem5.c
 
+memdb.lo:	$(TOP)/src/memdb.c $(HDR)
+	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/memdb.c
+
 memjournal.lo:	$(TOP)/src/memjournal.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/memjournal.c
 
@@ -878,6 +922,9 @@ trigger.lo:	$(TOP)/src/trigger.c $(HDR)
 update.lo:	$(TOP)/src/update.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/update.c
 
+upsert.lo:	$(TOP)/src/upsert.c $(HDR)
+	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/upsert.c
+
 utf.lo:	$(TOP)/src/utf.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/utf.c
 
@@ -926,11 +973,14 @@ wherecode.lo:	$(TOP)/src/wherecode.c $(HDR)
 whereexpr.lo:	$(TOP)/src/whereexpr.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/whereexpr.c
 
+window.lo:	$(TOP)/src/window.c $(HDR)
+	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/window.c
+
 tclsqlite.lo:	$(TOP)/src/tclsqlite.c $(HDR)
 	$(LTCOMPILE) -DUSE_TCL_STUBS=1 -c $(TOP)/src/tclsqlite.c
 
 tclsqlite-shell.lo:	$(TOP)/src/tclsqlite.c $(HDR)
-	$(LTCOMPILE) -DTCLSH=1 -o $@ -c $(TOP)/src/tclsqlite.c
+	$(LTCOMPILE) -DTCLSH -o $@ -c $(TOP)/src/tclsqlite.c
 
 tclsqlite-stubs.lo:	$(TOP)/src/tclsqlite.c $(HDR)
 	$(LTCOMPILE) -DUSE_TCL_STUBS=1 -o $@ -c $(TOP)/src/tclsqlite.c
@@ -958,13 +1008,30 @@ parse.c:	$(TOP)/src/parse.y lemon$(BEXE) $(TOP)/tool/addopcodes.tcl
 	mv parse.h parse.h.temp
 	$(TCLSH_CMD) $(TOP)/tool/addopcodes.tcl parse.h.temp >parse.h
 
-sqlite3.h:	$(TOP)/src/sqlite.h.in $(TOP)/manifest.uuid $(TOP)/VERSION
+sqlite3.h:	$(TOP)/src/sqlite.h.in $(TOP)/manifest mksourceid$(BEXE) $(TOP)/VERSION
 	$(TCLSH_CMD) $(TOP)/tool/mksqlite3h.tcl $(TOP) >sqlite3.h
 
 keywordhash.h:	$(TOP)/tool/mkkeywordhash.c
 	$(BCC) -o mkkeywordhash$(BEXE) $(OPT_FEATURE_FLAGS) $(OPTS) $(TOP)/tool/mkkeywordhash.c
 	./mkkeywordhash$(BEXE) >keywordhash.h
 
+# Source files that go into making shell.c
+SHELL_SRC = \
+	$(TOP)/src/shell.c.in \
+        $(TOP)/ext/misc/appendvfs.c \
+	$(TOP)/ext/misc/shathree.c \
+	$(TOP)/ext/misc/fileio.c \
+	$(TOP)/ext/misc/completion.c \
+	$(TOP)/ext/misc/sqlar.c \
+	$(TOP)/ext/expert/sqlite3expert.c \
+	$(TOP)/ext/expert/sqlite3expert.h \
+	$(TOP)/ext/misc/zipfile.c \
+        $(TOP)/src/test_windirent.c
+
+shell.c:	$(SHELL_SRC) $(TOP)/tool/mkshellc.tcl
+	$(TCLSH_CMD) $(TOP)/tool/mkshellc.tcl >shell.c
+
+
 
 
 # Rules to build the extension objects.
@@ -1085,12 +1152,14 @@ sqlite3rbu.lo:	$(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR)
 # necessary because the test fixture requires non-API symbols which are
 # hidden when the library is built via the amalgamation).
 #
-TESTFIXTURE_FLAGS  = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
+TESTFIXTURE_FLAGS  = -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
+TESTFIXTURE_FLAGS += -DTCLSH_INIT_PROC=sqlite3TestInit
 TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE 
 TESTFIXTURE_FLAGS += -DBUILD_sqlite
 TESTFIXTURE_FLAGS += -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
 TESTFIXTURE_FLAGS += -DSQLITE_DEFAULT_PAGE_SIZE=1024
 TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_STMTVTAB
+TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_DBPAGE_VTAB
 
 TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.la
 TESTFIXTURE_SRC1 = sqlite3.c
@@ -1114,14 +1183,17 @@ fulltestonly:	$(TESTPROGS) fuzztest
 	./testfixture$(TEXE) $(TOP)/test/full.test
 
 # Fuzz testing
-fuzztest:	fuzzcheck$(TEXE) $(FUZZDATA)
+fuzztest:	fuzzcheck$(TEXE) $(FUZZDATA) sessionfuzz$(TEXE) $(TOP)/test/sessionfuzz-data1.db
 	./fuzzcheck$(TEXE) $(FUZZDATA)
+	./sessionfuzz$(TEXE) run $(TOP)/test/sessionfuzz-data1.db
 
-fastfuzztest:	fuzzcheck$(TEXE) $(FUZZDATA)
+fastfuzztest:	fuzzcheck$(TEXE) $(FUZZDATA) sessionfuzz$(TEXE) $(TOP)/test/sessionfuzz-data1.db
 	./fuzzcheck$(TEXE) --limit-mem 100M $(FUZZDATA)
+	./sessionfuzz$(TEXE) run $(TOP)/test/sessionfuzz-data1.db
 
-valgrindfuzz:	fuzzcheck$(TEXT) $(FUZZDATA)
+valgrindfuzz:	fuzzcheck$(TEXT) $(FUZZDATA) sessionfuzz$(TEXE) $(TOP)/test/sessionfuzz-data1.db
 	valgrind ./fuzzcheck$(TEXE) --cell-size-check --limit-mem 10M --timeout 600 $(FUZZDATA)
+	valgrind ./sessionfuzz$(TEXE) run $(TOP)/test/sessionfuzz-data1.db
 
 # The veryquick.test TCL tests.
 #
@@ -1151,18 +1223,37 @@ valgrindtest:	$(TESTPROGS) valgrindfuzz
 smoketest:	$(TESTPROGS) fuzzcheck$(TEXE)
 	./testfixture$(TEXE) $(TOP)/test/main.test $(TESTOPTS)
 
-sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl
+sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in
-	echo "#define TCLSH 2" > $@
+	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in >sqlite3_analyzer.c
-	echo "#define SQLITE_ENABLE_DBSTAT_VTAB 1" >> $@
-	cat sqlite3.c $(TOP)/src/tclsqlite.c >> $@
-	echo "static const char *tclsh_main_loop(void){" >> $@
-	echo "static const char *zMainloop = " >> $@
-	$(TCLSH_CMD) $(TOP)/tool/tostr.tcl $(TOP)/tool/spaceanal.tcl >> $@
-	echo "; return zMainloop; }" >> $@
 
 sqlite3_analyzer$(TEXE): sqlite3_analyzer.c
 	$(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS)
 
+sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in	
+	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in >sqltclsh.c
+
+sqltclsh$(TEXE): sqltclsh.c
+	$(LTLINK) sqltclsh.c -o $@ $(LIBTCL) $(TLIBS)
+
+sqlite3_expert$(TEXE): $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c
+	$(LTLINK)	$(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c -o sqlite3_expert $(TLIBS)
+
+CHECKER_DEPS =\
+  $(TOP)/tool/mkccode.tcl \
+  sqlite3.c \
+  $(TOP)/src/tclsqlite.c \
+  $(TOP)/ext/repair/sqlite3_checker.tcl \
+  $(TOP)/ext/repair/checkindex.c \
+  $(TOP)/ext/repair/checkfreelist.c \
+  $(TOP)/ext/misc/btreeinfo.c \
+  $(TOP)/ext/repair/sqlite3_checker.c.in
+
+sqlite3_checker.c:	$(CHECKER_DEPS)
+	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/ext/repair/sqlite3_checker.c.in >$@
+
+sqlite3_checker$(TEXE):	sqlite3_checker.c
+	$(LTLINK) sqlite3_checker.c -o $@ $(LIBTCL) $(TLIBS)
+
 dbdump$(TEXE): $(TOP)/ext/misc/dbdump.c sqlite3.lo
 	$(LTLINK) -DDBDUMP_STANDALONE -o $@ \
            $(TOP)/ext/misc/dbdump.c sqlite3.lo $(TLIBS)
@@ -1179,12 +1270,18 @@ showjournal$(TEXE):	$(TOP)/tool/showjournal.c sqlite3.lo
 showwal$(TEXE):	$(TOP)/tool/showwal.c sqlite3.lo
 	$(LTLINK) -o $@ $(TOP)/tool/showwal.c sqlite3.lo $(TLIBS)
 
+showshm$(TEXE):	$(TOP)/tool/showshm.c
+	$(LTLINK) -o $@ $(TOP)/tool/showshm.c
+
 changeset$(TEXE):	$(TOP)/ext/session/changeset.c sqlite3.lo
 	$(LTLINK) -o $@ $(TOP)/ext/session/changeset.c sqlite3.lo $(TLIBS)
 
 rollback-test$(TEXE):	$(TOP)/tool/rollback-test.c sqlite3.lo
 	$(LTLINK) -o $@ $(TOP)/tool/rollback-test.c sqlite3.lo $(TLIBS)
 
+atrc$(TEXX): $(TOP)/test/atrc.c sqlite3.lo
+	$(LTLINK) -o $@ $(TOP)/test/atrc.c sqlite3.lo $(TLIBS)
+
 LogEst$(TEXE):	$(TOP)/tool/logest.c sqlite3.h
 	$(LTLINK) -I. -o $@ $(TOP)/tool/logest.c
 

Makefile.msc

@@ -92,6 +92,29 @@ SPLIT_AMALGAMATION = 0
 !ENDIF
 
 # <<mark>>
+# Set this non-0 to have this makefile assume the Tcl shell executable
+# (tclsh*.exe) is available in the PATH.  By default, this is disabled
+# for compatibility with older build environments.  This setting only
+# applies if TCLSH_CMD is not set manually.
+#
+!IFNDEF USE_TCLSH_IN_PATH
+USE_TCLSH_IN_PATH = 0
+!ENDIF
+
+# Set this non-0 to use zlib, possibly compiling it from source code.
+#
+!IFNDEF USE_ZLIB
+USE_ZLIB = 0
+!ENDIF
+
+# Set this non-0 to build zlib from source code.  This is enabled by
+# default and in that case it will be assumed that the ZLIBDIR macro
+# points to the top-level source code directory for zlib.
+#
+!IFNDEF BUILD_ZLIB
+BUILD_ZLIB = 1
+!ENDIF
+
 # Set this non-0 to use the International Components for Unicode (ICU).
 #
 !IFNDEF USE_ICU
@@ -316,6 +339,12 @@ SQLITE_TCL_DEP =
 !IF $(MINIMAL_AMALGAMATION)==0
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_GEOPOLY=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_JSON1=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBSTAT_VTAB=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_INTROSPECTION_PRAGMAS=1
 !ENDIF
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1
 !ENDIF
@@ -597,6 +626,10 @@ SHELL_COMPILE_OPTS = $(SHELL_CCONV_OPTS)
 !IFNDEF SHELL_CORE_SRC
 !IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
 SHELL_CORE_SRC =
+# <<mark>>
+!ELSEIF $(USE_AMALGAMATION)==0
+SHELL_CORE_SRC =
+# <</mark>>
 !ELSE
 SHELL_CORE_SRC = $(SQLITE3C)
 !ENDIF
@@ -607,16 +640,33 @@ SHELL_CORE_SRC = $(SQLITE3C)
 !IFNDEF SHELL_CORE_DEP
 !IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
 SHELL_CORE_DEP = $(SQLITE3DLL)
+# <<mark>>
+!ELSEIF $(USE_AMALGAMATION)==0
+SHELL_CORE_DEP = libsqlite3.lib
+# <</mark>>
 !ELSE
 SHELL_CORE_DEP =
 !ENDIF
 !ENDIF
 
+# <<mark>>
+# If zlib support is enabled, add the dependencies for it.
+#
+!IF $(USE_ZLIB)!=0 && $(BUILD_ZLIB)!=0
+SHELL_CORE_DEP = zlib $(SHELL_CORE_DEP)
+TESTFIXTURE_DEP = zlib $(TESTFIXTURE_DEP)
+!ENDIF
+# <</mark>>
+
 # This is the core library that the shell executable should link with.
 #
 !IFNDEF SHELL_CORE_LIB
 !IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
 SHELL_CORE_LIB = $(SQLITE3LIB)
+# <<mark>>
+!ELSEIF $(USE_AMALGAMATION)==0
+SHELL_CORE_LIB = libsqlite3.lib
+# <</mark>>
 !ELSE
 SHELL_CORE_LIB =
 !ENDIF
@@ -802,12 +852,16 @@ RCC = $(RCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1
 # prior to running nmake in order to match the actual installed location and
 # version on this machine.
 #
+!IFNDEF TCLDIR
+TCLDIR = $(TOP)\compat\tcl
+!ENDIF
+
 !IFNDEF TCLINCDIR
-TCLINCDIR = c:\tcl\include
+TCLINCDIR = $(TCLDIR)\include
 !ENDIF
 
 !IFNDEF TCLLIBDIR
-TCLLIBDIR = c:\tcl\lib
+TCLLIBDIR = $(TCLDIR)\lib
 !ENDIF
 
 !IFNDEF LIBTCL
@@ -819,7 +873,32 @@ LIBTCLSTUB = tclstub86.lib
 !ENDIF
 
 !IFNDEF LIBTCLPATH
-LIBTCLPATH = c:\tcl\bin
+LIBTCLPATH = $(TCLDIR)\bin
+!ENDIF
+
+# The locations of the zlib header and library files.  These variables
+# (ZLIBINCDIR, ZLIBLIBDIR, and ZLIBLIB) may be overridden via the environment
+# prior to running nmake in order to match the actual installed (or source
+# code) location on this machine.
+#
+!IFNDEF ZLIBDIR
+ZLIBDIR = $(TOP)\compat\zlib
+!ENDIF
+
+!IFNDEF ZLIBINCDIR
+ZLIBINCDIR = $(ZLIBDIR)
+!ENDIF
+
+!IFNDEF ZLIBLIBDIR
+ZLIBLIBDIR = $(ZLIBDIR)
+!ENDIF
+
+!IFNDEF ZLIBLIB
+!IF $(DYNAMIC_SHELL)!=0
+ZLIBLIB = zdll.lib
+!ELSE
+ZLIBLIB = zlib.lib
+!ENDIF
 !ENDIF
 
 # The locations of the ICU header and library files.  These variables
@@ -827,12 +906,16 @@ LIBTCLPATH = c:\tcl\bin
 # prior to running nmake in order to match the actual installed location on
 # this machine.
 #
+!IFNDEF ICUDIR
+ICUDIR = $(TOP)\compat\icu
+!ENDIF
+
 !IFNDEF ICUINCDIR
-ICUINCDIR = c:\icu\include
+ICUINCDIR = $(ICUDIR)\include
 !ENDIF
 
 !IFNDEF ICULIBDIR
-ICULIBDIR = c:\icu\lib
+ICULIBDIR = $(ICUDIR)\lib
 !ENDIF
 
 !IFNDEF LIBICU
@@ -845,7 +928,11 @@ LIBICU = icuuc.lib icuin.lib
 # specific Tcl shell to use.
 #
 !IFNDEF TCLSH_CMD
+!IF $(USE_TCLSH_IN_PATH)!=0 || !EXIST("$(TCLDIR)\bin\tclsh.exe")
 TCLSH_CMD = tclsh
+!ELSE
+TCLSH_CMD = $(TCLDIR)\bin\tclsh.exe
+!ENDIF
 !ENDIF
 # <</mark>>
 
@@ -951,6 +1038,15 @@ BCC = $(BCC) -Zi
 !ENDIF
 
 # <<mark>>
+# If zlib support is enabled, add the compiler options for it.
+#
+!IF $(USE_ZLIB)!=0
+TCC = $(TCC) -DSQLITE_HAVE_ZLIB=1
+RCC = $(RCC) -DSQLITE_HAVE_ZLIB=1
+TCC = $(TCC) -I$(ZLIBINCDIR)
+RCC = $(RCC) -I$(ZLIBINCDIR)
+!ENDIF
+
 # If ICU support is enabled, add the compiler options for it.
 #
 !IF $(USE_ICU)!=0
@@ -974,7 +1070,7 @@ LTLINK = $(TCC) -Fe$@
 # If requested, link to the RPCRT4 library.
 #
 !IF $(USE_RPCRT4_LIB)!=0
-LTLINK = $(LTLINK) rpcrt4.lib
+LTLIBS = $(LTLIBS) rpcrt4.lib
 !ENDIF
 
 # If a platform was set, force the linker to target that.
@@ -1071,8 +1167,15 @@ LDFLAGS = $(LDOPTS)
 # Start with the Tcl related linker options.
 #
 !IF $(NO_TCL)==0
-LTLIBPATHS = /LIBPATH:$(TCLLIBDIR)
+TCLLIBPATHS = $(TCLLIBPATHS) /LIBPATH:$(TCLLIBDIR)
-LTLIBS = $(LIBTCL)
+TCLLIBS = $(TCLLIBS) $(LIBTCL)
+!ENDIF
+
+# If zlib support is enabled, add the linker options for it.
+#
+!IF $(USE_ZLIB)!=0
+LTLIBPATHS = $(LTLIBPATHS) /LIBPATH:$(ZLIBLIBDIR)
+LTLIBS = $(LTLIBS) $(ZLIBLIB)
 !ENDIF
 
 # If ICU support is enabled, add the linker options for it.
@@ -1091,26 +1194,27 @@ LTLIBS = $(LTLIBS) $(LIBICU)
 #
 LIBOBJS0 = vdbe.lo parse.lo alter.lo analyze.lo attach.lo auth.lo \
          backup.lo bitvec.lo btmutex.lo btree.lo build.lo \
-         callback.lo complete.lo ctime.lo date.lo dbstat.lo delete.lo \
+         callback.lo complete.lo ctime.lo \
+         date.lo dbpage.lo dbstat.lo delete.lo \
          expr.lo fault.lo fkey.lo \
          fts3.lo fts3_aux.lo fts3_expr.lo fts3_hash.lo fts3_icu.lo \
          fts3_porter.lo fts3_snippet.lo fts3_tokenizer.lo fts3_tokenizer1.lo \
          fts3_tokenize_vtab.lo fts3_unicode.lo fts3_unicode2.lo fts3_write.lo \
          fts5.lo \
          func.lo global.lo hash.lo \
-         icu.lo insert.lo legacy.lo loadext.lo \
+         icu.lo insert.lo json1.lo legacy.lo loadext.lo \
          main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \
-         memjournal.lo \
+         memdb.lo memjournal.lo \
          mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \
          notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \
          pager.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
          random.lo resolve.lo rowset.lo rtree.lo \
-         sqlite3session.lo select.lo sqlite3rbu.lo status.lo \
+         sqlite3session.lo select.lo sqlite3rbu.lo status.lo stmt.lo \
          table.lo threads.lo tokenize.lo treeview.lo trigger.lo \
-         update.lo util.lo vacuum.lo \
+         update.lo upsert.lo util.lo vacuum.lo \
          vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
          vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \
-         utf.lo vtab.lo
+         window.lo utf.lo vtab.lo
 # <</mark>>
 
 # Object files for the amalgamation.
@@ -1154,6 +1258,7 @@ SRC00 = \
   $(TOP)\src\complete.c \
   $(TOP)\src\ctime.c \
   $(TOP)\src\date.c \
+  $(TOP)\src\dbpage.c \
   $(TOP)\src\dbstat.c \
   $(TOP)\src\delete.c \
   $(TOP)\src\expr.c \
@@ -1172,6 +1277,7 @@ SRC00 = \
   $(TOP)\src\mem2.c \
   $(TOP)\src\mem3.c \
   $(TOP)\src\mem5.c \
+  $(TOP)\src\memdb.c \
   $(TOP)\src\memjournal.c \
   $(TOP)\src\mutex.c \
   $(TOP)\src\mutex_noop.c \
@@ -1204,6 +1310,7 @@ SRC01 = \
   $(TOP)\src\trigger.c \
   $(TOP)\src\utf.c \
   $(TOP)\src\update.c \
+  $(TOP)\src\upsert.c \
   $(TOP)\src\util.c \
   $(TOP)\src\vacuum.c \
   $(TOP)\src\vdbe.c \
@@ -1218,12 +1325,8 @@ SRC01 = \
   $(TOP)\src\walker.c \
   $(TOP)\src\where.c \
   $(TOP)\src\wherecode.c \
-  $(TOP)\src\whereexpr.c
+  $(TOP)\src\whereexpr.c \
-
+  $(TOP)\src\window.c
-# Shell source code files.
-#
-SRC02 = \
-  $(TOP)\src\shell.c
 
 # Core miscellaneous files.
 #
@@ -1316,6 +1419,7 @@ SRC09 = \
   $(TOP)\ext\fts3\fts3_tokenizer.h \
   $(TOP)\ext\icu\sqliteicu.h \
   $(TOP)\ext\rtree\rtree.h \
+  $(TOP)\ext\rtree\geopoly.c \
   $(TOP)\ext\rbu\sqlite3rbu.h \
   $(TOP)\ext\session\sqlite3session.h
 
@@ -1331,6 +1435,7 @@ SRC11 = \
   keywordhash.h \
   opcodes.h \
   parse.h \
+  shell.c \
   $(SQLITE3H)
 
 # Generated Tcl header files
@@ -1345,7 +1450,7 @@ SRC12 =
 
 # All source code files.
 #
-SRC = $(SRC00) $(SRC01) $(SRC02) $(SRC03) $(SRC04) $(SRC05) $(SRC06) $(SRC07) $(SRC08) $(SRC09) $(SRC10) $(SRC11)
+SRC = $(SRC00) $(SRC01) $(SRC03) $(SRC04) $(SRC05) $(SRC06) $(SRC07) $(SRC08) $(SRC09) $(SRC10) $(SRC11)
 
 # Source code to the test files.
 #
@@ -1376,6 +1481,7 @@ TESTSRC = \
   $(TOP)\src\test_intarray.c \
   $(TOP)\src\test_journal.c \
   $(TOP)\src\test_malloc.c \
+  $(TOP)\src\test_md5.c \
   $(TOP)\src\test_multiplex.c \
   $(TOP)\src\test_mutex.c \
   $(TOP)\src\test_onefile.c \
@@ -1387,10 +1493,12 @@ TESTSRC = \
   $(TOP)\src\test_server.c \
   $(TOP)\src\test_superlock.c \
   $(TOP)\src\test_syscall.c \
+  $(TOP)\src\test_tclsh.c \
   $(TOP)\src\test_tclvar.c \
   $(TOP)\src\test_thread.c \
   $(TOP)\src\test_vfs.c \
   $(TOP)\src\test_windirent.c \
+  $(TOP)\src\test_window.c \
   $(TOP)\src\test_wsd.c \
   $(TOP)\ext\fts3\fts3_term.c \
   $(TOP)\ext\fts3\fts3_test.c \
@@ -1400,18 +1508,23 @@ TESTSRC = \
 # Statically linked extensions.
 #
 TESTEXT = \
+  $(TOP)\ext\expert\sqlite3expert.c \
+  $(TOP)\ext\expert\test_expert.c \
   $(TOP)\ext\misc\amatch.c \
   $(TOP)\ext\misc\carray.c \
   $(TOP)\ext\misc\closure.c \
   $(TOP)\ext\misc\csv.c \
   $(TOP)\ext\misc\eval.c \
+  $(TOP)\ext\misc\explain.c \
   $(TOP)\ext\misc\fileio.c \
   $(TOP)\ext\misc\fuzzer.c \
   $(TOP)\ext\fts5\fts5_tcl.c \
   $(TOP)\ext\fts5\fts5_test_mi.c \
   $(TOP)\ext\fts5\fts5_test_tok.c \
   $(TOP)\ext\misc\ieee754.c \
+  $(TOP)\ext\misc\mmapwarm.c \
   $(TOP)\ext\misc\nextchar.c \
+  $(TOP)\ext\misc\normalize.c \
   $(TOP)\ext\misc\percentile.c \
   $(TOP)\ext\misc\regexp.c \
   $(TOP)\ext\misc\remember.c \
@@ -1421,6 +1534,12 @@ TESTEXT = \
   $(TOP)\ext\misc\unionvtab.c \
   $(TOP)\ext\misc\wholenumber.c
 
+# If use of zlib is enabled, add the "zipfile.c" source file.
+#
+!IF $(USE_ZLIB)!=0
+TESTEXT = $(TESTEXT) $(TOP)\ext\misc\zipfile.c
+!ENDIF
+
 # Source code to the library files needed by the test fixture
 # (non-amalgamation)
 #
@@ -1476,7 +1595,8 @@ EXTHDR = $(EXTHDR) \
   $(TOP)\ext\fts3\fts3_hash.h \
   $(TOP)\ext\fts3\fts3_tokenizer.h
 EXTHDR = $(EXTHDR) \
-  $(TOP)\ext\rtree\rtree.h
+  $(TOP)\ext\rtree\rtree.h \
+  $(TOP)\ext\rtree\geopoly.c
 EXTHDR = $(EXTHDR) \
   $(TOP)\ext\icu\sqliteicu.h
 EXTHDR = $(EXTHDR) \
@@ -1490,8 +1610,10 @@ TESTPROGS = \
   testfixture.exe \
   $(SQLITE3EXE) \
   sqlite3_analyzer.exe \
+  sqlite3_checker.exe \
   sqldiff.exe \
-  dbhash.exe
+  dbhash.exe \
+  sqltclsh.exe
 
 # Databases containing fuzzer test cases
 #
@@ -1500,27 +1622,29 @@ FUZZDATA = \
   $(TOP)\test\fuzzdata2.db \
   $(TOP)\test\fuzzdata3.db \
   $(TOP)\test\fuzzdata4.db \
-  $(TOP)\test\fuzzdata5.db
+  $(TOP)\test\fuzzdata5.db \
+  $(TOP)\test\fuzzdata6.db
 # <</mark>>
 
 # Additional compiler options for the shell.  These are only effective
 # when the shell is not being dynamically linked.
 #
 !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
-SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1
 !ENDIF
 
 # <<mark>>
 # Extra compiler options for various test tools.
 #
-MPTESTER_COMPILE_OPTS = -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS5
+MPTESTER_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS5
 FUZZERSHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1
-FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ -DSQLITE_MAX_MEMORY=50000000
+FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ -DSQLITE_MAX_MEMORY=50000000 -DSQLITE_PRINTF_PRECISION_LIMIT=1000
 FUZZCHECK_SRC = $(TOP)\test\fuzzcheck.c $(TOP)\test\ossfuzz.c
 OSSSHELL_SRC = $(TOP)\test\ossshell.c $(TOP)\test\ossfuzz.c
 DBFUZZ_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION
 KV_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_DIRECT_OVERFLOW_READ
-DBSELFTEST_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5
 ST_COMPILE_OPTS = -DSQLITE_THREADSAFE=0
 
 # Standard options to testfixture.
@@ -1539,7 +1663,15 @@ ALL_TCL_TARGETS =
 # This is the default Makefile target.  The objects listed here
 # are what get build when you type just "make" with no arguments.
 #
-all:	dll libsqlite3.lib shell $(ALL_TCL_TARGETS)
+core:	dll libsqlite3.lib shell
+
+# Targets that require the Tcl library.
+#
+tcl:	$(ALL_TCL_TARGETS)
+
+# This Makefile target builds all of the standard binaries.
+#
+all:	core tcl
 
 # Dynamic link library section.
 #
@@ -1564,12 +1696,12 @@ $(SQLITE3DLL):	$(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP)
 sqlite3.def:	libsqlite3.lib
 	echo EXPORTS > sqlite3.def
 	dumpbin /all libsqlite3.lib \
-		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3(?:session|changeset|changegroup)?_[^@]*)(?:@\d+)?$$" \1 \
+		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3(?:session|changeset|changegroup|rebaser)?_[^@]*)(?:@\d+)?$$" \1 \
 		| sort >> sqlite3.def
 # <</block2>>
 
-$(SQLITE3EXE):	$(TOP)\src\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
+$(SQLITE3EXE):	shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
-	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\src\shell.c $(SHELL_CORE_SRC) \
+	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) shell.c $(SHELL_CORE_SRC) \
 		/link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS)
 
 # <<mark>>
@@ -1580,13 +1712,13 @@ dbhash.exe:	$(TOP)\tool\dbhash.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) $(TOP)\tool\dbhash.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
 
 scrub.exe:	$(TOP)\ext\misc\scrub.c $(SQLITE3C) $(SQLITE3H)
-	$(LTLINK) $(NO_WARN) $(TOP)\ext\misc\scrub.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
+	$(LTLINK) $(NO_WARN) -DSCRUB_STANDALONE=1 $(TOP)\ext\misc\scrub.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
 
 srcck1.exe:	$(TOP)\tool\srcck1.c
 	$(BCC) $(NO_WARN) -Fe$@ $(TOP)\tool\srcck1.c
 
-sourcetest:	srcck1.exe sqlite3.c
+sourcetest:	srcck1.exe $(SQLITE3C)
-	srcck1.exe sqlite3.c
+	srcck1.exe $(SQLITE3C)
 
 fuzzershell.exe:	$(TOP)\tool\fuzzershell.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) $(FUZZERSHELL_COMPILE_OPTS) $(TOP)\tool\fuzzershell.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
@@ -1600,6 +1732,9 @@ fuzzcheck.exe:	$(FUZZCHECK_SRC) $(SQLITE3C) $(SQLITE3H)
 ossshell.exe:	$(OSSSHELL_SRC) $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) $(FUZZCHECK_COMPILE_OPTS) $(OSSSHELL_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
 
+sessionfuzz.exe:	zlib $(TOP)\test\sessionfuzz.c $(SQLITE3C) $(SQLITE3H)
+	$(LTLINK) $(NO_WARN) -I$(ZLIBINCDIR) $(TOP)\test\sessionfuzz.c /link $(LDFLAGS) $(LTLINKOPTS) /LIBPATH:$(ZLIBLIBDIR) $(ZLIBLIB)
+
 mptester.exe:	$(TOP)\mptest\mptest.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) $(MPTESTER_COMPILE_OPTS) $(TOP)\mptest\mptest.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
 
@@ -1628,7 +1763,6 @@ mptest:	mptester.exe
 	-mkdir tsrc
 	for %i in ($(SRC00)) do copy /Y %i tsrc
 	for %i in ($(SRC01)) do copy /Y %i tsrc
-	for %i in ($(SRC02)) do copy /Y %i tsrc
 	for %i in ($(SRC03)) do copy /Y %i tsrc
 	for %i in ($(SRC04)) do copy /Y %i tsrc
 	for %i in ($(SRC05)) do copy /Y %i tsrc
@@ -1648,7 +1782,6 @@ mptest:	mptester.exe
 
 sqlite3.c:	.target_source sqlite3ext.h $(MKSQLITE3C_TOOL)
 	$(TCLSH_CMD) $(MKSQLITE3C_TOOL) $(MKSQLITE3C_ARGS)
-	copy tsrc\shell.c .
 	copy $(TOP)\ext\session\sqlite3session.h .
 
 sqlite3-all.c:	sqlite3.c $(TOP)\tool\split-sqlite3c.tcl
@@ -1670,6 +1803,12 @@ lemon.exe:	$(TOP)\tool\lemon.c lempar.c
 	$(BCC) $(NO_WARN) -Daccess=_access \
 		-Fe$@ $(TOP)\tool\lemon.c /link $(LDFLAGS) $(NLTLINKOPTS) $(NLTLIBPATHS)
 
+# <<mark>>
+# Rules to build the source-id generator tool
+#
+mksourceid.exe:	$(TOP)\tool\mksourceid.c
+	$(BCC) $(NO_WARN) -Fe$@ $(TOP)\tool\mksourceid.c /link $(LDFLAGS) $(NLTLINKOPTS) $(NLTLIBPATHS)
+
 # Rules to build individual *.lo files from generated *.c files. This
 # applies to:
 #
@@ -1740,7 +1879,10 @@ ctime.lo:	$(TOP)\src\ctime.c $(HDR)
 date.lo:	$(TOP)\src\date.c $(HDR)
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\date.c
 
-dbstat.lo:	$(TOP)\src\date.c $(HDR)
+dbpage.lo:	$(TOP)\src\dbpage.c $(HDR)
+	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\dbpage.c
+
+dbstat.lo:	$(TOP)\src\dbstat.c $(HDR)
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\dbstat.c
 
 delete.lo:	$(TOP)\src\delete.c $(HDR)
@@ -1794,6 +1936,9 @@ mem3.lo:	$(TOP)\src\mem3.c $(HDR)
 mem5.lo:	$(TOP)\src\mem5.c $(HDR)
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\mem5.c
 
+memdb.lo:	$(TOP)\src\memdb.c $(HDR)
+	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\memdb.c
+
 memjournal.lo:	$(TOP)\src\memjournal.c $(HDR)
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\memjournal.c
 
@@ -1872,6 +2017,9 @@ trigger.lo:	$(TOP)\src\trigger.c $(HDR)
 update.lo:	$(TOP)\src\update.c $(HDR)
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\update.c
 
+upsert.lo:	$(TOP)\src\upsert.c $(HDR)
+	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\upsert.c
+
 utf.lo:	$(TOP)\src\utf.c $(HDR)
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\utf.c
 
@@ -1920,14 +2068,17 @@ wherecode.lo:	$(TOP)\src\wherecode.c $(HDR)
 whereexpr.lo:	$(TOP)\src\whereexpr.c $(HDR)
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\whereexpr.c
 
+window.lo:	$(TOP)\src\window.c $(HDR)
+	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\window.c
+
 tclsqlite.lo:	$(TOP)\src\tclsqlite.c $(HDR) $(SQLITE_TCL_DEP)
 	$(LTCOMPILE) $(NO_WARN) -DUSE_TCL_STUBS=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c
 
 tclsqlite-shell.lo:	$(TOP)\src\tclsqlite.c $(HDR) $(SQLITE_TCL_DEP)
-	$(LTCOMPILE) $(NO_WARN) -DTCLSH=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c
+	$(LTCOMPILE) $(NO_WARN) -DTCLSH -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c
 
 tclsqlite3.exe:	tclsqlite-shell.lo $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS)
-	$(LTLINK) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /OUT:$@ tclsqlite-shell.lo $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
+	$(LTLINK) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) /OUT:$@ tclsqlite-shell.lo $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS)
 
 # Rules to build opcodes.c and opcodes.h
 #
@@ -1948,7 +2099,7 @@ parse.c:	$(TOP)\src\parse.y lemon.exe $(TOP)\tool\addopcodes.tcl
 	move parse.h parse.h.temp
 	$(TCLSH_CMD) $(TOP)\tool\addopcodes.tcl parse.h.temp > parse.h
 
-$(SQLITE3H):	$(TOP)\src\sqlite.h.in $(TOP)\manifest.uuid $(TOP)\VERSION
+$(SQLITE3H):	$(TOP)\src\sqlite.h.in $(TOP)\manifest mksourceid.exe $(TOP)\VERSION
 	$(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP:\=/) > $(SQLITE3H) $(MKSQLITE3H_ARGS)
 
 sqlite3ext.h:	.target_source
@@ -1967,7 +2118,29 @@ mkkeywordhash.exe:	$(TOP)\tool\mkkeywordhash.c
 keywordhash.h:	$(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe
 	.\mkkeywordhash.exe > keywordhash.h
 
+# Source files that go into making shell.c
+SHELL_SRC = \
+	$(TOP)\src\shell.c.in \
+	$(TOP)\ext\misc\appendvfs.c \
+	$(TOP)\ext\misc\shathree.c \
+	$(TOP)\ext\misc\fileio.c \
+	$(TOP)\ext\misc\completion.c \
+	$(TOP)\ext\expert\sqlite3expert.c \
+	$(TOP)\ext\expert\sqlite3expert.h \
+	$(TOP)\src\test_windirent.c
 
+# If use of zlib is enabled, add the "zipfile.c" source file.
+#
+!IF $(USE_ZLIB)!=0
+SHELL_SRC = $(SHELL_SRC) $(TOP)\ext\misc\sqlar.c
+SHELL_SRC = $(SHELL_SRC) $(TOP)\ext\misc\zipfile.c
+!ENDIF
+
+shell.c:	$(SHELL_SRC) $(TOP)\tool\mkshellc.tcl
+	$(TCLSH_CMD) $(TOP)\tool\mkshellc.tcl > shell.c
+
+zlib:
+	pushd $(ZLIBDIR) && $(MAKE) /f win32\Makefile.msc clean $(ZLIBLIB) && popd
 
 # Rules to build the extension objects.
 #
@@ -2031,6 +2204,12 @@ fts3_unicode2.lo:	$(TOP)\ext\fts3\fts3_unicode2.c $(HDR) $(EXTHDR)
 fts3_write.lo:	$(TOP)\ext\fts3\fts3_write.c $(HDR) $(EXTHDR)
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_write.c
 
+json1.lo:	$(TOP)\ext\misc\json1.c $(HDR) $(EXTHDR)
+	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\misc\json1.c
+
+stmt.lo:	$(TOP)\ext\misc\stmt.c $(HDR) $(EXTHDR)
+	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\misc\stmt.c
+
 rtree.lo:	$(TOP)\ext\rtree\rtree.c $(HDR) $(EXTHDR)
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\rtree\rtree.c
 
@@ -2056,6 +2235,24 @@ FTS5_SRC = \
    $(TOP)\ext\fts5\fts5_varint.c \
    $(TOP)\ext\fts5\fts5_vocab.c
 
+LSM1_SRC = \
+   $(TOP)\ext\lsm1\lsm.h \
+   $(TOP)\ext\lsm1\lsmInt.h \
+   $(TOP)\ext\lsm1\lsm_ckpt.c \
+   $(TOP)\ext\lsm1\lsm_file.c \
+   $(TOP)\ext\lsm1\lsm_log.c \
+   $(TOP)\ext\lsm1\lsm_main.c \
+   $(TOP)\ext\lsm1\lsm_mem.c \
+   $(TOP)\ext\lsm1\lsm_mutex.c \
+   $(TOP)\ext\lsm1\lsm_shared.c \
+   $(TOP)\ext\lsm1\lsm_sorted.c \
+   $(TOP)\ext\lsm1\lsm_str.c \
+   $(TOP)\ext\lsm1\lsm_tree.c \
+   $(TOP)\ext\lsm1\lsm_unix.c \
+   $(TOP)\ext\lsm1\lsm_varint.c \
+   $(TOP)\ext\lsm1\lsm_vtab.c \
+   $(TOP)\ext\lsm1\lsm_win32.c
+
 fts5parse.c:	$(TOP)\ext\fts5\fts5parse.y lemon.exe
 	copy $(TOP)\ext\fts5\fts5parse.y .
 	del /Q fts5parse.h 2>NUL
@@ -2067,6 +2264,10 @@ fts5.c:	$(FTS5_SRC)
 	$(TCLSH_CMD) $(TOP)\ext\fts5\tool\mkfts5c.tcl
 	copy $(TOP)\ext\fts5\fts5.h .
 
+lsm1.c:	$(LSM1_SRC)
+	$(TCLSH_CMD) $(TOP)\ext\lsm1\tool\mklsm1c.tcl
+	copy $(TOP)\ext\lsm1\lsm.h .
+
 fts5.lo:	fts5.c $(HDR) $(EXTHDR)
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c fts5.c
 
@@ -2086,12 +2287,14 @@ sqlite3rbu.lo:	$(TOP)\ext\rbu\sqlite3rbu.c $(HDR) $(EXTHDR)
 # necessary because the test fixture requires non-API symbols which are
 # hidden when the library is built via the amalgamation).
 #
-TESTFIXTURE_FLAGS = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
+TESTFIXTURE_FLAGS = -DTCLSH_INIT_PROC=sqlite3TestInit -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERVER=1 -DSQLITE_PRIVATE=""
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_CORE $(NO_WARN)
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_DEFAULT_PAGE_SIZE=1024
-TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB
+TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1
+TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1
+TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_JSON1=1
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) $(TEST_CCONV_OPTS)
 
 TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2)
@@ -2122,11 +2325,11 @@ sqlite_tcl.h:
 		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact "Tcl_HashEntry *(*createProc)" "Tcl_HashEntry *(SQLITE_TCLAPI *createProc)" >> $(SQLITETCLH)
 !ENDIF
 
-testfixture.exe:	$(TESTFIXTURE_SRC) $(SQLITE3H) $(LIBRESOBJS) $(HDR) $(SQLITE_TCL_DEP)
+testfixture.exe:	$(TESTFIXTURE_SRC) $(TESTFIXTURE_DEP) $(SQLITE3H) $(LIBRESOBJS) $(HDR) $(SQLITE_TCL_DEP)
 	$(LTLINK) -DSQLITE_NO_SYNC=1 $(TESTFIXTURE_FLAGS) \
 		-DBUILD_sqlite -I$(TCLINCDIR) \
 		$(TESTFIXTURE_SRC) \
-		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
+		/link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS)
 
 extensiontest:	testfixture.exe testloadext.dll
 	@set PATH=$(LIBTCLPATH);$(PATH)
@@ -2171,24 +2374,45 @@ smoketest:	$(TESTPROGS)
 	@set PATH=$(LIBTCLPATH);$(PATH)
 	.\testfixture.exe $(TOP)\test\main.test $(TESTOPTS)
 
-sqlite3_analyzer.c:	$(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl $(SQLITE_TCL_DEP)
+sqlite3_analyzer.c:	$(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in $(SQLITE_TCL_DEP)
-	echo #define TCLSH 2 > $@
+	$(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in > $@
-	echo #define SQLITE_ENABLE_DBSTAT_VTAB 1 >> $@
-	copy $@ + $(SQLITE3C) + $(TOP)\src\tclsqlite.c $@
-	echo static const char *tclsh_main_loop(void){ >> $@
-	echo static const char *zMainloop = >> $@
-	$(TCLSH_CMD) $(TOP)\tool\tostr.tcl $(TOP)\tool\spaceanal.tcl >> $@
-	echo ; return zMainloop; } >> $@
 
 sqlite3_analyzer.exe:	sqlite3_analyzer.c $(LIBRESOBJS)
 	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_analyzer.c \
-		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
+		/link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS)
 
-dbdump.exe:	$(TOP)\ext\misc\dbdump.c $(SQLITE3C) $(SQLITE3H)
+sqltclsh.c: sqlite3.c $(TOP)\src\tclsqlite.c $(TOP)\tool\sqltclsh.tcl $(TOP)\ext\misc\appendvfs.c $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqltclsh.c.in
+	$(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqltclsh.c.in >sqltclsh.c
+
+sqltclsh.exe: sqltclsh.c  $(SHELL_CORE_DEP) $(LIBRESOBJS)
+	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqltclsh.c \
+		/link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS)
+
+sqlite3_expert.exe: $(SQLITE3C) $(TOP)\ext\expert\sqlite3expert.h $(TOP)\ext\expert\sqlite3expert.c $(TOP)\ext\expert\expert.c
+	$(LTLINK) $(NO_WARN)	$(TOP)\ext\expert\sqlite3expert.c $(TOP)\ext\expert\expert.c $(SQLITE3C) $(TLIBS)
+
+CHECKER_DEPS =\
+  $(TOP)/tool/mkccode.tcl \
+  sqlite3.c \
+  $(TOP)/src/tclsqlite.c \
+  $(TOP)/ext/repair/sqlite3_checker.tcl \
+  $(TOP)/ext/repair/checkindex.c \
+  $(TOP)/ext/repair/checkfreelist.c \
+  $(TOP)/ext/misc/btreeinfo.c \
+  $(TOP)/ext/repair/sqlite3_checker.c.in
+
+sqlite3_checker.c:	$(CHECKER_DEPS)
+	$(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\ext\repair\sqlite3_checker.c.in > $@
+
+sqlite3_checker.exe:	sqlite3_checker.c $(LIBRESOBJS)
+	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_checker.c \
+		/link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS)
+
+dbdump.exe:	$(TOP)\ext\misc\dbdump.c $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS)
 	$(LTLINK) $(NO_WARN) -DDBDUMP_STANDALONE $(TOP)\ext\misc\dbdump.c $(SQLITE3C) \
 		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS)
 
-testloadext.lo:	$(TOP)\src\test_loadext.c
+testloadext.lo:	$(TOP)\src\test_loadext.c $(SQLITE3H)
 	$(LTCOMPILE) $(NO_WARN) -c $(TOP)\src\test_loadext.c
 
 testloadext.dll:	testloadext.lo
@@ -2210,6 +2434,9 @@ showwal.exe:	$(TOP)\tool\showwal.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
 		$(TOP)\tool\showwal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
 
+showshm.exe:	$(TOP)\tool\showshm.c
+	$(LTLINK) $(NO_WARN)	$(TOP)\tool\showshm.c /link $(LDFLAGS) $(LTLINKOPTS)
+
 changeset.exe:	$(TOP)\ext\session\changeset.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
 		-DSQLITE_ENABLE_SESSION=1 -DSQLITE_ENABLE_PREUPDATE_HOOK=1 \
@@ -2223,6 +2450,10 @@ rollback-test.exe:	$(TOP)\tool\rollback-test.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
 		$(TOP)\tool\rollback-test.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
 
+atrc.exe:	$(TOP)\test\atrc.c $(SQLITE3C) $(SQLITE3H)
+	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
+		$(TOP)\test\atrc.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
+
 LogEst.exe:	$(TOP)\tool\logest.c $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) $(TOP)\tool\LogEst.c /link $(LDFLAGS) $(LTLINKOPTS)
 
@@ -2238,9 +2469,6 @@ kvtest.exe:	$(TOP)\test\kvtest.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) $(KV_COMPILE_OPTS) \
 		$(TOP)\test\kvtest.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
 
-dbselftest.exe:	$(TOP)\test\dbselftest.c $(SQLITE3C) $(SQLITE3H)
-	$(LTLINK) $(NO_WARN) $(DBSELFTEST_COMPILE_OPTS) $(TOP)\test\dbselftest.c $(SQLITE3C)
-
 rbu.exe:	$(TOP)\ext\rbu\rbu.c $(TOP)\ext\rbu\sqlite3rbu.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) -DSQLITE_ENABLE_RBU \
 		$(TOP)\ext\rbu\rbu.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
@@ -2257,10 +2485,9 @@ clean:
 	del /Q *.bsc *.def *.cod *.da *.bb *.bbg *.vc gmon.out 2>NUL
 	del /Q $(SQLITE3EXE) $(SQLITE3DLL) Replace.exe 2>NUL
 # <<mark>>
-	del /Q sqlite3.c sqlite3.h 2>NUL
 	del /Q opcodes.c opcodes.h 2>NUL
 	del /Q lemon.* lempar.c parse.* 2>NUL
-	del /Q mkkeywordhash.* keywordhash.h 2>NUL
+	del /Q mksourceid.* mkkeywordhash.* keywordhash.h 2>NUL
 	del /Q notasharedlib.* 2>NUL
 	-rmdir /Q/S .deps 2>NUL
 	-rmdir /Q/S .libs 2>NUL
@@ -2274,11 +2501,16 @@ clean:
 	del /Q changeset.exe 2>NUL
 	del /Q showjournal.exe showstat4.exe showwal.exe speedtest1.exe 2>NUL
 	del /Q mptester.exe wordcount.exe rbu.exe srcck1.exe 2>NUL
-	del /Q sqlite3.c sqlite3-*.c 2>NUL
+	del /Q sqlite3.c sqlite3-*.c sqlite3.h 2>NUL
 	del /Q sqlite3rc.h 2>NUL
 	del /Q shell.c sqlite3ext.h sqlite3session.h 2>NUL
 	del /Q sqlite3_analyzer.exe sqlite3_analyzer.c 2>NUL
 	del /Q sqlite-*-output.vsix 2>NUL
 	del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe dbhash.exe 2>NUL
+	del /Q sqltclsh.* 2>NUL
+	del /Q dbfuzz.exe sessionfuzz.exe 2>NUL
+	del /Q kvtest.exe ossshell.exe scrub.exe 2>NUL
+	del /Q showshm.exe sqlite3_checker.* sqlite3_expert.exe 2>NUL
 	del /Q fts5.* fts5parse.* 2>NUL
+	del /Q lsm.h lsm1.c 2>NUL
 # <</mark>>

README.md

@@ -1,12 +1,14 @@
 <h1 align="center">SQLite Source Repository</h1>
 
-This repository contains the complete source code for the SQLite database
+This repository contains the complete source code for the 
-engine.  Some test scripts are also include.  However, many other test scripts
+[SQLite database engine](https://sqlite.org/).  Some test scripts 
+are also included.  However, many other test scripts
 and most of the documentation are managed separately.
 
-If you are reading this on a Git mirror someplace, you are doing it wrong.
+SQLite [does not use Git](https://sqlite.org/whynotgit.html).
-The [official repository](https://www.sqlite.org/src/) is better.  Go there
+If you are reading this on GitHub, then you are looking at an
-now.
+unofficial mirror. See <https://sqlite.org/src> for the official
+repository.
 
 ## Obtaining The Code
 
@@ -14,15 +16,17 @@ SQLite sources are managed using the
 [Fossil](https://www.fossil-scm.org/), a distributed version control system
 that was specifically designed to support SQLite development.
 If you do not want to use Fossil, you can download tarballs or ZIP
-archives as follows:
+archives or [SQLite archives](https://sqlite.org/cli.html#sqlar) as follows:
 
-  *  Lastest trunk check-in:
+  *  Lastest trunk check-in as
-     <https://www.sqlite.org/src/tarball/sqlite.tar.gz> or
+     [Tarball](https://www.sqlite.org/src/tarball/sqlite.tar.gz),
-     <https://www.sqlite.org/src/zip/sqlite.zip>.
+     [ZIP-archive](https://www.sqlite.org/src/zip/sqlite.zip), or
+     [SQLite-archive](https://www.sqlite.org/src/sqlar/sqlite.sqlar).
 
-  *  Latest release:
+  *  Latest release as
-     <https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=release> or
+     [Tarball](https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=release),
-     <https://www.sqlite.org/src/zip/sqlite.zip?r=release>.
+     [ZIP-archive](https://www.sqlite.org/src/zip/sqlite.zip?r=release), or
+     [SQLite-archive](https://www.sqlite.org/src/sqlar/sqlite.sqlar?r=release).
 
   *  For other check-ins, substitute an appropriate branch name or
      tag or hash prefix for "release" in the URLs of the previous
@@ -104,7 +108,6 @@ recommended.
 SQLite does not require [Tcl](http://www.tcl.tk/) to run, but a Tcl installation
 is required by the makefiles (including those for MSVC).  SQLite contains
 a lot of generated code and Tcl is used to do much of that code generation.
-The makefiles also require AWK.
 
 ## Source Code Tour
 
@@ -116,7 +119,7 @@ The **src/** also contains the "shell.c" file
 which is the main program for the "sqlite3.exe"
 [command-line shell](https://sqlite.org/cli.html) and
 the "tclsqlite.c" file which implements the
-[TCL bindings](https://sqlite.org/tclsqlite.html) for SQLite.
+[Tcl bindings](https://sqlite.org/tclsqlite.html) for SQLite.
 (Historical note:  SQLite began as a Tcl
 extension and only later escaped to the wild as an independent library.)
 
@@ -163,14 +166,14 @@ template for generating its parser.
 
 Lemon also generates the **parse.h** header file, at the same time it
 generates parse.c. But the parse.h header file is
-modified further (to add additional symbols) using the ./addopcodes.awk
+modified further (to add additional symbols) using the ./addopcodes.tcl
-AWK script.
+Tcl script.
 
 The **opcodes.h** header file contains macros that define the numbers
 corresponding to opcodes in the "VDBE" virtual machine.  The opcodes.h
 file is generated by the scanning the src/vdbe.c source file.  The
-AWK script at ./mkopcodeh.awk does this scan and generates opcodes.h.
+Tcl script at ./mkopcodeh.tcl does this scan and generates opcodes.h.
-A second AWK script, ./mkopcodec.awk, then scans opcodes.h to generate
+A second Tcl script, ./mkopcodec.tcl, then scans opcodes.h to generate
 the **opcodes.c** source file, which contains a reverse mapping from
 opcode-number to opcode-name that is used for EXPLAIN output.
 
@@ -207,8 +210,8 @@ The amalgamation source file is more than 200K lines long.  Some symbolic
 debuggers (most notably MSVC) are unable to deal with files longer than 64K
 lines.  To work around this, a separate Tcl script, tool/split-sqlite3c.tcl,
 can be run on the amalgamation to break it up into a single small C file
-called **sqlite3-all.c** that does #include on about five other files
+called **sqlite3-all.c** that does #include on about seven other files
-named **sqlite3-1.c**, **sqlite3-2.c**, ..., **sqlite3-5.c**.  In this way,
+named **sqlite3-1.c**, **sqlite3-2.c**, ..., **sqlite3-7.c**.  In this way,
 all of the source code is contained within a single translation unit so
 that the compiler can do extra cross-procedure optimization, but no
 individual source file exceeds 32K lines in length.
@@ -237,7 +240,8 @@ Key files:
      trying to understand how the library works internally.
 
   *  **sqliteInt.h** - this header file defines many of the data objects
-     used internally by SQLite.
+     used internally by SQLite.  In addition to "sqliteInt.h", some
+     subsystems have their own header files.
 
   *  **parse.y** - This file describes the LALR(1) grammar that SQLite uses
      to parse SQL statements, and the actions that are taken at each step
@@ -249,29 +253,44 @@ Key files:
      which defines internal data objects.  The rest of SQLite interacts
      with the VDBE through an interface defined by vdbe.h.
 
-  *  **where.c** - This file analyzes the WHERE clause and generates
+  *  **where.c** - This file (together with its helper files named
+     by "where*.c") analyzes the WHERE clause and generates
      virtual machine code to run queries efficiently.  This file is
      sometimes called the "query optimizer".  It has its own private
      header file, whereInt.h, that defines data objects used internally.
 
   *  **btree.c** - This file contains the implementation of the B-Tree
-     storage engine used by SQLite.
+     storage engine used by SQLite.  The interface to the rest of the system
+     is defined by "btree.h".  The "btreeInt.h" header defines objects
+     used internally by btree.c and not published to the rest of the system.
 
   *  **pager.c** - This file contains the "pager" implementation, the
-     module that implements transactions.
+     module that implements transactions.  The "pager.h" header file
+     defines the interface between pager.c and the rest of the system.
 
   *  **os_unix.c** and **os_win.c** - These two files implement the interface
      between SQLite and the underlying operating system using the run-time
      pluggable VFS interface.
 
-  *  **shell.c** - This file is not part of the core SQLite library.  This
+  *  **shell.c.in** - This file is not part of the core SQLite library.  This
      is the file that, when linked against sqlite3.a, generates the
-     "sqlite3.exe" command-line shell.
+     "sqlite3.exe" command-line shell.  The "shell.c.in" file is transformed
+     into "shell.c" as part of the build process.
 
   *  **tclsqlite.c** - This file implements the Tcl bindings for SQLite.  It
      is not part of the core SQLite library.  But as most of the tests in this
      repository are written in Tcl, the Tcl language bindings are important.
 
+  *  **test*.c** - Files in the src/ folder that begin with "test" go into
+     building the "testfixture.exe" program.  The testfixture.exe program is
+     an enhanced Tcl shell.  The testfixture.exe program runs scripts in the
+     test/ folder to validate the core SQLite code.  The testfixture program
+     (and some other test programs too) is build and run when you type
+     "make test".
+
+  *  **ext/misc/json1.c** - This file implements the various JSON functions
+     that are build into SQLite.
+
 There are many other source files.  Each has a succinct header comment that
 describes its purpose and role within the larger system.
 

VERSION

@@ -1 +1 @@
-3.21.0
+3.26.0

autoconf/Makefile.am

@@ -1,6 +1,5 @@
 
-AM_CFLAGS = @THREADSAFE_FLAGS@ @DYNAMIC_EXTENSION_FLAGS@ @FTS5_FLAGS@ @JSON1_FLAGS@ @SESSION_FLAGS@ -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE
+AM_CFLAGS = @BUILD_CFLAGS@ 
-
 lib_LTLIBRARIES = libsqlite3.la
 libsqlite3_la_SOURCES = sqlite3.c
 libsqlite3_la_LDFLAGS = -no-undefined -version-info 8:6:8
@@ -10,11 +9,11 @@ sqlite3_SOURCES = shell.c sqlite3.h
 EXTRA_sqlite3_SOURCES = sqlite3.c
 sqlite3_LDADD = @EXTRA_SHELL_OBJ@ @READLINE_LIBS@
 sqlite3_DEPENDENCIES = @EXTRA_SHELL_OBJ@
-sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS
+sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_STMTVTAB -DSQLITE_ENABLE_DBSTAT_VTAB $(SHELL_CFLAGS)
 
 include_HEADERS = sqlite3.h sqlite3ext.h
 
-EXTRA_DIST = sqlite3.1 tea Makefile.msc sqlite3.rc README.txt Replace.cs
+EXTRA_DIST = sqlite3.1 tea Makefile.msc sqlite3.rc README.txt Replace.cs Makefile.fallback
 pkgconfigdir = ${libdir}/pkgconfig
 pkgconfig_DATA = sqlite3.pc
 

autoconf/Makefile.fallback

@@ -0,0 +1,19 @@
+#!/usr/bin/make
+#
+# If the configure script does not work, then this Makefile is available
+# as a backup.  Manually configure the variables below.
+#
+# Note:  This makefile works out-of-the-box on MacOS 10.2 (Jaguar)
+#
+CC = gcc
+CFLAGS = -O0 -I.
+LIBS = -lz
+COPTS += -D_BSD_SOURCE
+COPTS += -DSQLITE_ENABLE_LOCKING_STYLE=0
+COPTS += -DSQLITE_THREADSAFE=0
+COPTS += -DSQLITE_OMIT_LOAD_EXTENSION
+COPTS += -DSQLITE_WITHOUT_ZONEMALLOC
+COPTS += -DSQLITE_ENABLE_RTREE
+
+sqlite3:	shell.c sqlite3.c
+	$(CC) $(CFLAGS) $(COPTS) -o sqlite3 shell.c sqlite3.c $(LIBS)

autoconf/Makefile.msc

@@ -277,6 +277,12 @@ SQLITE3EXEPDB = /pdb:sqlite3sh.pdb
 !IF $(MINIMAL_AMALGAMATION)==0
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_GEOPOLY=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_JSON1=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBSTAT_VTAB=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_INTROSPECTION_PRAGMAS=1
 !ENDIF
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1
 !ENDIF
@@ -561,6 +567,7 @@ SHELL_CORE_DEP =
 !ENDIF
 !ENDIF
 
+
 # This is the core library that the shell executable should link with.
 #
 !IFNDEF SHELL_CORE_LIB
@@ -808,7 +815,7 @@ LTLINK = $(TCC) -Fe$@
 # If requested, link to the RPCRT4 library.
 #
 !IF $(USE_RPCRT4_LIB)!=0
-LTLINK = $(LTLINK) rpcrt4.lib
+LTLIBS = $(LTLIBS) rpcrt4.lib
 !ENDIF
 
 # If a platform was set, force the linker to target that.
@@ -927,14 +934,24 @@ LIBRESOBJS =
 # when the shell is not being dynamically linked.
 #
 !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
-SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1
 !ENDIF
 
 
 # This is the default Makefile target.  The objects listed here
 # are what get build when you type just "make" with no arguments.
 #
-all:	dll shell
+core:	dll shell
+
+# Targets that require the Tcl library.
+#
+tcl:	$(ALL_TCL_TARGETS)
+
+# This Makefile target builds all of the standard binaries.
+#
+all:	core tcl
 
 # Dynamic link library section.
 #
@@ -954,11 +971,11 @@ Replace.exe:
 sqlite3.def:	Replace.exe $(LIBOBJ)
 	echo EXPORTS > sqlite3.def
 	dumpbin /all $(LIBOBJ) \
-		| .\Replace.exe "^\s+/EXPORT:_?(sqlite3(?:session|changeset|changegroup)?_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \
+		| .\Replace.exe "^\s+/EXPORT:_?(sqlite3(?:session|changeset|changegroup|rebaser)?_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \
 		| sort >> sqlite3.def
 
-$(SQLITE3EXE):	$(TOP)\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
+$(SQLITE3EXE):	shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
-	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\shell.c $(SHELL_CORE_SRC) \
+	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) shell.c $(SHELL_CORE_SRC) \
 		/link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS)
 
 
@@ -973,7 +990,7 @@ sqlite3.lo:	$(SQLITE3C)
 !IF $(USE_RC)!=0
 _HASHCHAR=^#
 !IF ![echo !IFNDEF VERSION > rcver.vc] && \
-    ![for /F "delims=" %V in ('type "$(SQLITE3H)" ^| find "$(_HASHCHAR)define SQLITE_VERSION "') do (echo VERSION = ^^%V >> rcver.vc)] && \
+    ![for /F "delims=" %V in ('type "$(SQLITE3H)" ^| "%SystemRoot%\System32\find.exe" "$(_HASHCHAR)define SQLITE_VERSION "') do (echo VERSION = ^^%V >> rcver.vc)] && \
     ![echo !ENDIF >> rcver.vc]
 !INCLUDE rcver.vc
 !ENDIF

autoconf/configure.ac

@@ -12,6 +12,7 @@
 AC_PREREQ(2.61)
 AC_INIT(sqlite, --SQLITE-VERSION--, http://www.sqlite.org)
 AC_CONFIG_SRCDIR([sqlite3.c])
+AC_CONFIG_AUX_DIR([.])
 
 # Use automake.
 AM_INIT_AUTOMAKE([foreign])
@@ -28,6 +29,7 @@ AC_CHECK_FUNCS([fdatasync usleep fullfsync localtime_r gmtime_r])
 AC_FUNC_STRERROR_R
 
 AC_CONFIG_FILES([Makefile sqlite3.pc])
+BUILD_CFLAGS=
 AC_SUBST(BUILD_CFLAGS)
 
 #-------------------------------------------------------------------------
@@ -85,13 +87,11 @@ AC_SUBST(READLINE_LIBS)
 AC_ARG_ENABLE(threadsafe, [AS_HELP_STRING(
   [--enable-threadsafe], [build a thread-safe library [default=yes]])], 
   [], [enable_threadsafe=yes])
-THREADSAFE_FLAGS=-DSQLITE_THREADSAFE=0
 if test x"$enable_threadsafe" != "xno"; then
-  THREADSAFE_FLAGS="-D_REENTRANT=1 -DSQLITE_THREADSAFE=1"
+  BUILD_CFLAGS="$BUILD_CFLAGS -D_REENTRANT=1 -DSQLITE_THREADSAFE=1"
   AC_SEARCH_LIBS(pthread_create, pthread)
   AC_SEARCH_LIBS(pthread_mutexattr_init, pthread)
 fi
-AC_SUBST(THREADSAFE_FLAGS)
 #-----------------------------------------------------------------------
 
 #-----------------------------------------------------------------------
@@ -103,36 +103,66 @@ AC_ARG_ENABLE(dynamic-extensions, [AS_HELP_STRING(
 if test x"$enable_dynamic_extensions" != "xno"; then
   AC_SEARCH_LIBS(dlopen, dl)
 else
-  DYNAMIC_EXTENSION_FLAGS=-DSQLITE_OMIT_LOAD_EXTENSION=1
+  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_OMIT_LOAD_EXTENSION=1"
 fi
 AC_MSG_CHECKING([for whether to support dynamic extensions])
 AC_MSG_RESULT($enable_dynamic_extensions)
-AC_SUBST(DYNAMIC_EXTENSION_FLAGS)
+#-----------------------------------------------------------------------
+
+#-----------------------------------------------------------------------
+#   --enable-fts4
+#
+AC_ARG_ENABLE(fts4, [AS_HELP_STRING(
+  [--enable-fts4], [include fts4 support [default=yes]])], 
+  [], [enable_fts4=yes])
+if test x"$enable_fts4" = "xyes"; then
+  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_FTS4"
+fi
+#-----------------------------------------------------------------------
+
+#-----------------------------------------------------------------------
+#   --enable-fts3
+#
+AC_ARG_ENABLE(fts3, [AS_HELP_STRING(
+  [--enable-fts3], [include fts3 support [default=no]])], 
+  [], [])
+if test x"$enable_fts3" = "xyes" -a x"$enable_fts4" = "xno"; then
+  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_FTS3"
+fi
 #-----------------------------------------------------------------------
 
 #-----------------------------------------------------------------------
 #   --enable-fts5
 #
 AC_ARG_ENABLE(fts5, [AS_HELP_STRING(
-  [--enable-fts5], [include fts5 support [default=no]])], 
+  [--enable-fts5], [include fts5 support [default=yes]])], 
-  [], [enable_fts5=no])
+  [], [enable_fts5=yes])
 if test x"$enable_fts5" = "xyes"; then
   AC_SEARCH_LIBS(log, m)
-  FTS5_FLAGS=-DSQLITE_ENABLE_FTS5
+  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_FTS5"
 fi
-AC_SUBST(FTS5_FLAGS)
 #-----------------------------------------------------------------------
 
 #-----------------------------------------------------------------------
 #   --enable-json1
 #
 AC_ARG_ENABLE(json1, [AS_HELP_STRING(
-  [--enable-json1], [include json1 support [default=no]])], 
+  [--enable-json1], [include json1 support [default=yes]])], 
-  [], [enable_json1=no])
+  [],[enable_json1=yes])
 if test x"$enable_json1" = "xyes"; then
-  JSON1_FLAGS=-DSQLITE_ENABLE_JSON1
+  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_JSON1"
+fi
+#-----------------------------------------------------------------------
+
+#-----------------------------------------------------------------------
+#   --enable-rtree
+#
+AC_ARG_ENABLE(rtree, [AS_HELP_STRING(
+  [--enable-rtree], [include rtree support [default=yes]])], 
+  [], [enable_rtree=yes])
+if test x"$enable_rtree" = "xyes"; then
+  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_RTREE"
 fi
-AC_SUBST(JSON1_FLAGS)
 #-----------------------------------------------------------------------
 
 #-----------------------------------------------------------------------
@@ -140,11 +170,22 @@ AC_SUBST(JSON1_FLAGS)
 #
 AC_ARG_ENABLE(session, [AS_HELP_STRING(
   [--enable-session], [enable the session extension [default=no]])], 
-  [], [enable_session=no])
+  [], [])
 if test x"$enable_session" = "xyes"; then
-  SESSION_FLAGS="-DSQLITE_ENABLE_SESSION -DSQLITE_ENABLE_PREUPDATE_HOOK"
+  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_SESSION -DSQLITE_ENABLE_PREUPDATE_HOOK"
+fi
+#-----------------------------------------------------------------------
+
+#-----------------------------------------------------------------------
+#   --enable-debug
+#
+AC_ARG_ENABLE(debug, [AS_HELP_STRING(
+  [--enable-debug], [build with debugging features enabled [default=no]])], 
+  [], [])
+if test x"$enable_debug" = "xyes"; then
+  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_DEBUG -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE"
+  CFLAGS="-g -O0"
 fi
-AC_SUBST(SESSION_FLAGS)
 #-----------------------------------------------------------------------
 
 #-----------------------------------------------------------------------
@@ -163,6 +204,12 @@ AC_SUBST(EXTRA_SHELL_OBJ)
 #-----------------------------------------------------------------------
 
 AC_CHECK_FUNCS(posix_fallocate)
+AC_CHECK_HEADERS(zlib.h,[
+  AC_SEARCH_LIBS(deflate,z,[BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_HAVE_ZLIB"])
+])
+
+AC_SEARCH_LIBS(system,,,[SHELL_CFLAGS="-DSQLITE_NOHAVE_SYSTEM"])
+AC_SUBST(SHELL_CFLAGS)
 
 #-----------------------------------------------------------------------
 # UPDATE: Maybe it's better if users just set CFLAGS before invoking

configure

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for sqlite 3.21.0.
+# Generated by GNU Autoconf 2.69 for sqlite 3.26.0.
 #
 #
 # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -726,8 +726,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='sqlite'
 PACKAGE_TARNAME='sqlite'
-PACKAGE_VERSION='3.21.0'
+PACKAGE_VERSION='3.26.0'
-PACKAGE_STRING='sqlite 3.21.0'
+PACKAGE_STRING='sqlite 3.26.0'
 PACKAGE_BUGREPORT=''
 PACKAGE_URL=''
 
@@ -772,6 +772,7 @@ LIBOBJS
 BUILD_CFLAGS
 USE_GCOV
 OPT_FEATURE_FLAGS
+HAVE_ZLIB
 USE_AMALGAMATION
 TARGET_DEBUG
 TARGET_HAVE_EDITLINE
@@ -909,6 +910,8 @@ enable_fts3
 enable_fts4
 enable_fts5
 enable_json1
+enable_update_limit
+enable_geopoly
 enable_rtree
 enable_session
 enable_gcov
@@ -1463,7 +1466,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures sqlite 3.21.0 to adapt to many kinds of systems.
+\`configure' configures sqlite 3.26.0 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1528,7 +1531,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of sqlite 3.21.0:";;
+     short | recursive ) echo "Configuration of sqlite 3.26.0:";;
    esac
   cat <<\_ACEOF
 
@@ -1560,6 +1563,8 @@ Optional Features:
   --enable-fts4           Enable the FTS4 extension
   --enable-fts5           Enable the FTS5 extension
   --enable-json1          Enable the JSON1 extension
+  --enable-update-limit   Enable the UPDATE/DELETE LIMIT clause
+  --enable-geopoly        Enable the GEOPOLY extension
   --enable-rtree          Enable the RTREE extension
   --enable-session        Enable the SESSION extension
   --enable-gcov           Enable coverage testing using gcov
@@ -1652,7 +1657,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-sqlite configure 3.21.0
+sqlite configure 3.26.0
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -2071,7 +2076,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by sqlite $as_me 3.21.0, which was
+It was created by sqlite $as_me 3.26.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -3929,13 +3934,13 @@ if ${lt_cv_nm_interface+:} false; then :
 else
   lt_cv_nm_interface="BSD nm"
   echo "int some_variable = 0;" > conftest.$ac_ext
-  (eval echo "\"\$as_me:3932: $ac_compile\"" >&5)
+  (eval echo "\"\$as_me:3937: $ac_compile\"" >&5)
   (eval "$ac_compile" 2>conftest.err)
   cat conftest.err >&5
-  (eval echo "\"\$as_me:3935: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
+  (eval echo "\"\$as_me:3940: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
   (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out)
   cat conftest.err >&5
-  (eval echo "\"\$as_me:3938: output\"" >&5)
+  (eval echo "\"\$as_me:3943: output\"" >&5)
   cat conftest.out >&5
   if $GREP 'External.*some_variable' conftest.out > /dev/null; then
     lt_cv_nm_interface="MS dumpbin"
@@ -5141,7 +5146,7 @@ ia64-*-hpux*)
   ;;
 *-*-irix6*)
   # Find out which ABI we are using.
-  echo '#line 5144 "configure"' > conftest.$ac_ext
+  echo '#line 5149 "configure"' > conftest.$ac_ext
   if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5
   (eval $ac_compile) 2>&5
   ac_status=$?
@@ -6666,11 +6671,11 @@ else
    -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
    -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
    -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:6669: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:6674: $lt_compile\"" >&5)
    (eval "$lt_compile" 2>conftest.err)
    ac_status=$?
    cat conftest.err >&5
-   echo "$as_me:6673: \$? = $ac_status" >&5
+   echo "$as_me:6678: \$? = $ac_status" >&5
    if (exit $ac_status) && test -s "$ac_outfile"; then
      # The compiler can only warn and ignore the option if not recognized
      # So say no if there are warnings other than the usual output.
@@ -7005,11 +7010,11 @@ else
    -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
    -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
    -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7008: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7013: $lt_compile\"" >&5)
    (eval "$lt_compile" 2>conftest.err)
    ac_status=$?
    cat conftest.err >&5
-   echo "$as_me:7012: \$? = $ac_status" >&5
+   echo "$as_me:7017: \$? = $ac_status" >&5
    if (exit $ac_status) && test -s "$ac_outfile"; then
      # The compiler can only warn and ignore the option if not recognized
      # So say no if there are warnings other than the usual output.
@@ -7110,11 +7115,11 @@ else
    -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
    -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
    -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7113: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7118: $lt_compile\"" >&5)
    (eval "$lt_compile" 2>out/conftest.err)
    ac_status=$?
    cat out/conftest.err >&5
-   echo "$as_me:7117: \$? = $ac_status" >&5
+   echo "$as_me:7122: \$? = $ac_status" >&5
    if (exit $ac_status) && test -s out/conftest2.$ac_objext
    then
      # The compiler can only warn and ignore the option if not recognized
@@ -7165,11 +7170,11 @@ else
    -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
    -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
    -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7168: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7173: $lt_compile\"" >&5)
    (eval "$lt_compile" 2>out/conftest.err)
    ac_status=$?
    cat out/conftest.err >&5
-   echo "$as_me:7172: \$? = $ac_status" >&5
+   echo "$as_me:7177: \$? = $ac_status" >&5
    if (exit $ac_status) && test -s out/conftest2.$ac_objext
    then
      # The compiler can only warn and ignore the option if not recognized
@@ -9545,7 +9550,7 @@ else
   lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
   lt_status=$lt_dlunknown
   cat > conftest.$ac_ext <<_LT_EOF
-#line 9548 "configure"
+#line 9553 "configure"
 #include "confdefs.h"
 
 #if HAVE_DLFCN_H
@@ -9641,7 +9646,7 @@ else
   lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
   lt_status=$lt_dlunknown
   cat > conftest.$ac_ext <<_LT_EOF
-#line 9644 "configure"
+#line 9649 "configure"
 #include "confdefs.h"
 
 #if HAVE_DLFCN_H
@@ -10302,7 +10307,7 @@ USE_AMALGAMATION=1
 # if not, then we fall back to plain tclsh.
 # TODO: try other versions before falling back?
 #
-for ac_prog in tclsh8.6 tclsh8.5 tclsh
+for ac_prog in tclsh8.7 tclsh8.6 tclsh8.5 tclsh
 do
   # Extract the first word of "$ac_prog", so it can be a program name with args.
 set dummy $ac_prog; ac_word=$2
@@ -10452,8 +10457,6 @@ fi
 # Check whether --enable-threadsafe was given.
 if test "${enable_threadsafe+set}" = set; then :
   enableval=$enable_threadsafe;
-else
-  enable_threadsafe=yes
 fi
 
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support threadsafe operation" >&5
@@ -11246,12 +11249,10 @@ fi
 # check for debug enabled
 # Check whether --enable-debug was given.
 if test "${enable_debug+set}" = set; then :
-  enableval=$enable_debug; use_debug=$enableval
+  enableval=$enable_debug;
-else
-  use_debug=no
 fi
 
-if test "${use_debug}" = "yes" ; then
+if test "${enable_debug}" = "yes" ; then
   TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0"
 else
   TARGET_DEBUG="-DNDEBUG"
@@ -11262,26 +11263,98 @@ fi
 # See whether we should use the amalgamation to build
 # Check whether --enable-amalgamation was given.
 if test "${enable_amalgamation+set}" = set; then :
-  enableval=$enable_amalgamation; use_amalgamation=$enableval
+  enableval=$enable_amalgamation;
-else
-  use_amalgamation=yes
 fi
 
-if test "${use_amalgamation}" != "yes" ; then
+if test "${enable_amalgamation}" == "no" ; then
   USE_AMALGAMATION=0
 fi
 
 
+#########
+# Look for zlib.  Only needed by extensions and by the sqlite3.exe shell
+for ac_header in zlib.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "zlib.h" "ac_cv_header_zlib_h" "$ac_includes_default"
+if test "x$ac_cv_header_zlib_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_ZLIB_H 1
+_ACEOF
+
+fi
+
+done
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing deflate" >&5
+$as_echo_n "checking for library containing deflate... " >&6; }
+if ${ac_cv_search_deflate+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  ac_func_search_save_LIBS=$LIBS
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+/* Override any GCC internal prototype to avoid an error.
+   Use char because int might match the return type of a GCC
+   builtin and then its argument prototype would still apply.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+char deflate ();
+int
+main ()
+{
+return deflate ();
+  ;
+  return 0;
+}
+_ACEOF
+for ac_lib in '' z; do
+  if test -z "$ac_lib"; then
+    ac_res="none required"
+  else
+    ac_res=-l$ac_lib
+    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
+  fi
+  if ac_fn_c_try_link "$LINENO"; then :
+  ac_cv_search_deflate=$ac_res
+fi
+rm -f core conftest.err conftest.$ac_objext \
+    conftest$ac_exeext
+  if ${ac_cv_search_deflate+:} false; then :
+  break
+fi
+done
+if ${ac_cv_search_deflate+:} false; then :
+
+else
+  ac_cv_search_deflate=no
+fi
+rm conftest.$ac_ext
+LIBS=$ac_func_search_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_deflate" >&5
+$as_echo "$ac_cv_search_deflate" >&6; }
+ac_res=$ac_cv_search_deflate
+if test "$ac_res" != no; then :
+  test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
+  HAVE_ZLIB="-DSQLITE_HAVE_ZLIB=1"
+else
+  HAVE_ZLIB=""
+fi
+
+
+
 #########
 # See whether we should allow loadable extensions
 # Check whether --enable-load-extension was given.
 if test "${enable_load_extension+set}" = set; then :
-  enableval=$enable_load_extension; use_loadextension=$enableval
+  enableval=$enable_load_extension;
 else
-  use_loadextension=yes
+  enable_load_extension=yes
 fi
 
-if test "${use_loadextension}" = "yes" ; then
+if test "${enable_load_extension}" = "yes" ; then
   OPT_FEATURE_FLAGS=""
   { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing dlopen" >&5
 $as_echo_n "checking for library containing dlopen... " >&6; }
@@ -11348,9 +11421,7 @@ fi
 #
 # Check whether --enable-memsys5 was given.
 if test "${enable_memsys5+set}" = set; then :
-  enableval=$enable_memsys5; enable_memsys5=yes
+  enableval=$enable_memsys5;
-else
-  enable_memsys5=no
 fi
 
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support MEMSYS5" >&5
@@ -11365,9 +11436,7 @@ $as_echo "no" >&6; }
 fi
 # Check whether --enable-memsys3 was given.
 if test "${enable_memsys3+set}" = set; then :
-  enableval=$enable_memsys3; enable_memsys3=yes
+  enableval=$enable_memsys3;
-else
-  enable_memsys3=no
 fi
 
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support MEMSYS3" >&5
@@ -11385,9 +11454,7 @@ fi
 # See whether we should enable Full Text Search extensions
 # Check whether --enable-fts3 was given.
 if test "${enable_fts3+set}" = set; then :
-  enableval=$enable_fts3; enable_fts3=yes
+  enableval=$enable_fts3;
-else
-  enable_fts3=no
 fi
 
 if test "${enable_fts3}" = "yes" ; then
@@ -11395,9 +11462,7 @@ if test "${enable_fts3}" = "yes" ; then
 fi
 # Check whether --enable-fts4 was given.
 if test "${enable_fts4+set}" = set; then :
-  enableval=$enable_fts4; enable_fts4=yes
+  enableval=$enable_fts4;
-else
-  enable_fts4=no
 fi
 
 if test "${enable_fts4}" = "yes" ; then
@@ -11461,9 +11526,7 @@ fi
 fi
 # Check whether --enable-fts5 was given.
 if test "${enable_fts5+set}" = set; then :
-  enableval=$enable_fts5; enable_fts5=yes
+  enableval=$enable_fts5;
-else
-  enable_fts5=no
 fi
 
 if test "${enable_fts5}" = "yes" ; then
@@ -11530,22 +11593,44 @@ fi
 # See whether we should enable JSON1
 # Check whether --enable-json1 was given.
 if test "${enable_json1+set}" = set; then :
-  enableval=$enable_json1; enable_json1=yes
+  enableval=$enable_json1;
-else
-  enable_json1=no
 fi
 
 if test "${enable_json1}" = "yes" ; then
   OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_JSON1"
 fi
 
+#########
+# See whether we should enable the LIMIT clause on UPDATE and DELETE
+# statements.
+# Check whether --enable-update-limit was given.
+if test "${enable_update_limit+set}" = set; then :
+  enableval=$enable_update_limit;
+fi
+
+if test "${enable_udlimit}" = "yes" ; then
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT"
+fi
+
+#########
+# See whether we should enable GEOPOLY
+# Check whether --enable-geopoly was given.
+if test "${enable_geopoly+set}" = set; then :
+  enableval=$enable_geopoly; enable_geopoly=yes
+else
+  enable_geopoly=no
+fi
+
+if test "${enable_geopoly}" = "yes" ; then
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_GEOPOLY"
+  enable_rtree=yes
+fi
+
 #########
 # See whether we should enable RTREE
 # Check whether --enable-rtree was given.
 if test "${enable_rtree+set}" = set; then :
-  enableval=$enable_rtree; enable_rtree=yes
+  enableval=$enable_rtree;
-else
-  enable_rtree=no
 fi
 
 if test "${enable_rtree}" = "yes" ; then
@@ -11556,9 +11641,7 @@ fi
 # See whether we should enable the SESSION extension
 # Check whether --enable-session was given.
 if test "${enable_session+set}" = set; then :
-  enableval=$enable_session; enable_session=yes
+  enableval=$enable_session;
-else
-  enable_session=no
 fi
 
 if test "${enable_session}" = "yes" ; then
@@ -11621,9 +11704,7 @@ BUILD_CFLAGS=$ac_temp_BUILD_CFLAGS
 # See whether we should use GCOV
 # Check whether --enable-gcov was given.
 if test "${enable_gcov+set}" = set; then :
-  enableval=$enable_gcov; use_gcov=$enableval
+  enableval=$enable_gcov;
-else
-  use_gcov=no
 fi
 
 if test "${use_gcov}" = "yes" ; then
@@ -12151,7 +12232,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by sqlite $as_me 3.21.0, which was
+This file was extended by sqlite $as_me 3.26.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -12217,7 +12298,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-sqlite config.status 3.21.0
+sqlite config.status 3.26.0
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 

configure.ac

@@ -120,7 +120,7 @@ USE_AMALGAMATION=1
 # if not, then we fall back to plain tclsh.
 # TODO: try other versions before falling back?
 # 
-AC_CHECK_PROGS(TCLSH_CMD, [tclsh8.6 tclsh8.5 tclsh], none)
+AC_CHECK_PROGS(TCLSH_CMD, [tclsh8.7 tclsh8.6 tclsh8.5 tclsh], none)
 if test "$TCLSH_CMD" = "none"; then
   # If we can't find a local tclsh, then building the amalgamation will fail.
   # We act as though --disable-amalgamation has been used.
@@ -182,7 +182,7 @@ AC_SUBST(BUILD_CC)
 # Do we want to support multithreaded use of sqlite
 #
 AC_ARG_ENABLE(threadsafe, 
-AC_HELP_STRING([--disable-threadsafe],[Disable mutexing]),,enable_threadsafe=yes)
+AC_HELP_STRING([--disable-threadsafe],[Disable mutexing]))
 AC_MSG_CHECKING([whether to support threadsafe operation])
 if test "$enable_threadsafe" = "no"; then
   SQLITE_THREADSAFE=0
@@ -557,9 +557,8 @@ AC_SEARCH_LIBS(fdatasync, [rt])
 
 #########
 # check for debug enabled
-AC_ARG_ENABLE(debug, AC_HELP_STRING([--enable-debug],[enable debugging & verbose explain]),
+AC_ARG_ENABLE(debug, AC_HELP_STRING([--enable-debug],[enable debugging & verbose explain]))
-      [use_debug=$enableval],[use_debug=no])
+if test "${enable_debug}" = "yes" ; then
-if test "${use_debug}" = "yes" ; then
   TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0"
 else
   TARGET_DEBUG="-DNDEBUG"
@@ -569,19 +568,23 @@ AC_SUBST(TARGET_DEBUG)
 #########
 # See whether we should use the amalgamation to build
 AC_ARG_ENABLE(amalgamation, AC_HELP_STRING([--disable-amalgamation],
-      [Disable the amalgamation and instead build all files separately]),
+      [Disable the amalgamation and instead build all files separately]))
-      [use_amalgamation=$enableval],[use_amalgamation=yes])
+if test "${enable_amalgamation}" == "no" ; then
-if test "${use_amalgamation}" != "yes" ; then
   USE_AMALGAMATION=0
 fi
 AC_SUBST(USE_AMALGAMATION)
 
+#########
+# Look for zlib.  Only needed by extensions and by the sqlite3.exe shell
+AC_CHECK_HEADERS(zlib.h)
+AC_SEARCH_LIBS(deflate, z, [HAVE_ZLIB="-DSQLITE_HAVE_ZLIB=1"], [HAVE_ZLIB=""])
+AC_SUBST(HAVE_ZLIB)
+
 #########
 # See whether we should allow loadable extensions
 AC_ARG_ENABLE(load-extension, AC_HELP_STRING([--disable-load-extension],
-      [Disable loading of external extensions]),
+      [Disable loading of external extensions]),,[enable_load_extension=yes])
-      [use_loadextension=$enableval],[use_loadextension=yes])
+if test "${enable_load_extension}" = "yes" ; then
-if test "${use_loadextension}" = "yes" ; then
   OPT_FEATURE_FLAGS=""
   AC_SEARCH_LIBS(dlopen, dl)
 else
@@ -592,8 +595,7 @@ fi
 # Do we want to support memsys3 and/or memsys5
 #
 AC_ARG_ENABLE(memsys5, 
-  AC_HELP_STRING([--enable-memsys5],[Enable MEMSYS5]),
+  AC_HELP_STRING([--enable-memsys5],[Enable MEMSYS5]))
-  [enable_memsys5=yes],[enable_memsys5=no])
 AC_MSG_CHECKING([whether to support MEMSYS5])
 if test "${enable_memsys5}" = "yes"; then
   OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS5"
@@ -602,8 +604,7 @@ else
   AC_MSG_RESULT([no])
 fi
 AC_ARG_ENABLE(memsys3, 
-  AC_HELP_STRING([--enable-memsys3],[Enable MEMSYS3]),
+  AC_HELP_STRING([--enable-memsys3],[Enable MEMSYS3]))
-  [enable_memsys3=yes],[enable_memsys3=no])
 AC_MSG_CHECKING([whether to support MEMSYS3])
 if test "${enable_memsys3}" = "yes" -a "${enable_memsys5}" = "no"; then
   OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS3"
@@ -615,21 +616,18 @@ fi
 #########
 # See whether we should enable Full Text Search extensions
 AC_ARG_ENABLE(fts3, AC_HELP_STRING([--enable-fts3],
-      [Enable the FTS3 extension]),
+      [Enable the FTS3 extension]))
-      [enable_fts3=yes],[enable_fts3=no])
 if test "${enable_fts3}" = "yes" ; then
   OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS3"
 fi
 AC_ARG_ENABLE(fts4, AC_HELP_STRING([--enable-fts4],
-      [Enable the FTS4 extension]),
+      [Enable the FTS4 extension]))
-      [enable_fts4=yes],[enable_fts4=no])
 if test "${enable_fts4}" = "yes" ; then
   OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS4"
   AC_SEARCH_LIBS([log],[m])
 fi
 AC_ARG_ENABLE(fts5, AC_HELP_STRING([--enable-fts5],
-      [Enable the FTS5 extension]),
+      [Enable the FTS5 extension]))
-      [enable_fts5=yes],[enable_fts5=no])
 if test "${enable_fts5}" = "yes" ; then
   OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS5"
   AC_SEARCH_LIBS([log],[m])
@@ -637,18 +635,34 @@ fi
 
 #########
 # See whether we should enable JSON1
-AC_ARG_ENABLE(json1, AC_HELP_STRING([--enable-json1],
+AC_ARG_ENABLE(json1, AC_HELP_STRING([--enable-json1],[Enable the JSON1 extension]))
-      [Enable the JSON1 extension]),
-      [enable_json1=yes],[enable_json1=no])
 if test "${enable_json1}" = "yes" ; then
   OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_JSON1"
 fi
 
+#########
+# See whether we should enable the LIMIT clause on UPDATE and DELETE
+# statements.
+AC_ARG_ENABLE(update-limit, AC_HELP_STRING([--enable-update-limit],
+      [Enable the UPDATE/DELETE LIMIT clause]))
+if test "${enable_udlimit}" = "yes" ; then
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT"
+fi
+
+#########
+# See whether we should enable GEOPOLY
+AC_ARG_ENABLE(geopoly, AC_HELP_STRING([--enable-geopoly],
+      [Enable the GEOPOLY extension]),
+      [enable_geopoly=yes],[enable_geopoly=no])
+if test "${enable_geopoly}" = "yes" ; then
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_GEOPOLY"
+  enable_rtree=yes
+fi
+
 #########
 # See whether we should enable RTREE
 AC_ARG_ENABLE(rtree, AC_HELP_STRING([--enable-rtree],
-      [Enable the RTREE extension]),
+      [Enable the RTREE extension]))
-      [enable_rtree=yes],[enable_rtree=no])
 if test "${enable_rtree}" = "yes" ; then
   OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_RTREE"
 fi
@@ -656,8 +670,7 @@ fi
 #########
 # See whether we should enable the SESSION extension
 AC_ARG_ENABLE(session, AC_HELP_STRING([--enable-session],
-      [Enable the SESSION extension]),
+      [Enable the SESSION extension]))
-      [enable_session=yes],[enable_session=no])
 if test "${enable_session}" = "yes" ; then
   OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_SESSION"
   OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_PREUPDATE_HOOK"
@@ -717,8 +730,7 @@ BUILD_CFLAGS=$ac_temp_BUILD_CFLAGS
 #########
 # See whether we should use GCOV
 AC_ARG_ENABLE(gcov, AC_HELP_STRING([--enable-gcov],
-      [Enable coverage testing using gcov]),
+      [Enable coverage testing using gcov]))
-      [use_gcov=$enableval],[use_gcov=no])
 if test "${use_gcov}" = "yes" ; then
   USE_GCOV=1
 else

doc/F2FS.txt

@@ -0,0 +1,87 @@
+
+SQLite's OS layer contains the following definitions used in F2FS related
+calls:
+
+#define F2FS_IOCTL_MAGIC        0xf5
+#define F2FS_IOC_START_ATOMIC_WRITE     _IO(F2FS_IOCTL_MAGIC, 1)
+#define F2FS_IOC_COMMIT_ATOMIC_WRITE    _IO(F2FS_IOCTL_MAGIC, 2)
+#define F2FS_IOC_START_VOLATILE_WRITE   _IO(F2FS_IOCTL_MAGIC, 3)
+#define F2FS_IOC_ABORT_VOLATILE_WRITE   _IO(F2FS_IOCTL_MAGIC, 5)
+#define F2FS_IOC_GET_FEATURES           _IOR(F2FS_IOCTL_MAGIC, 12, u32)
+#define F2FS_FEATURE_ATOMIC_WRITE       0x0004
+
+After opening a database file on Linux (including Android), SQLite determines
+whether or not a file supports F2FS atomic commits as follows:
+
+  u32 flags = 0;
+  rc = ioctl(fd, F2FS_IOC_GET_FEATURES, &flags);
+  if( rc==0 && (flags & F2FS_FEATURE_ATOMIC_WRITE) ){
+    /* File supports F2FS atomic commits */
+  }else{
+    /* File does NOT support F2FS atomic commits */
+  }
+
+where "fd" is the file-descriptor open on the database file.
+
+Usually, when writing to a database file that supports atomic commits, SQLite
+accumulates the entire transaction in heap memory, deferring all writes to the
+db file until the transaction is committed.
+
+When it is time to commit a transaction on a file that supports atomic
+commits, SQLite does:
+
+  /* Take an F_WRLCK lock on the database file. This prevents any other
+  ** SQLite clients from reading or writing the file until the lock
+  ** is released.  */
+  rc = fcntl(fd, F_SETLK, ...);
+  if( rc!=0 ) goto failed;
+
+  rc = ioctl(fd, F2FS_IOC_START_ATOMIC_WRITE);
+  if( rc!=0 ) goto fallback_to_legacy_journal_commit;
+
+  foreach (dirty page){
+    rc = write(fd, ...dirty page...);
+    if( rc!=0 ){
+      ioctl(fd, F2FS_IOC_ABORT_VOLATILE_WRITE);
+      goto fallback_to_legacy_journal_commit;
+    }
+  }
+
+  rc = ioctl(fd, F2FS_IOC_COMMIT_ATOMIC_WRITE);
+  if( rc!=0 ){
+    ioctl(fd, F2FS_IOC_ABORT_VOLATILE_WRITE);
+    goto fallback_to_legacy_journal_commit;
+  }
+
+  /* If we get there, the transaction has been successfully 
+  ** committed to persistent storage. The following call
+  ** relinquishes the F_WRLCK lock.  */
+  fcntl(fd, F_SETLK, ...);
+
+Assumptions:
+
+1. After either of the F2FS_IOC_ABORT_VOLATILE_WRITE calls return,
+   the database file is in the state that it was in before
+   F2FS_IOC_START_ATOMIC_WRITE was invoked. Even if the ioctl()
+   fails - we're ignoring the return code.
+
+   This is true regardless of the type of error that occurred in
+   ioctl() or write().
+
+2. If the system fails before the F2FS_IOC_COMMIT_ATOMIC_WRITE is
+   completed, then following a reboot the database file is in the
+   state that it was in before F2FS_IOC_START_ATOMIC_WRITE was invoked.
+   Or, if the write was commited right before the system failed, in a 
+   state indicating that all write() calls were successfully committed
+   to persistent storage before the failure occurred.
+
+3. If the process crashes before the F2FS_IOC_COMMIT_ATOMIC_WRITE is
+   completed then the file is automatically restored to the state that
+   it was in before F2FS_IOC_START_ATOMIC_WRITE was called. This occurs
+   before the posix advisory lock is automatically dropped - there is
+   no chance that another client will be able to read the file in a
+   half-committed state before the rollback operation occurs.
+
+
+
+

doc/lemon.html

@@ -2,12 +2,12 @@
 <head>
 <title>The Lemon Parser Generator</title>
 </head>
-<body bgcolor=white>
+<body bgcolor='white'>
-<h1 align=center>The Lemon Parser Generator</h1>
+<h1 align='center'>The Lemon Parser Generator</h1>
 
 <p>Lemon is an LALR(1) parser generator for C.
 It does the same job as "bison" and "yacc".
-But lemon is not a bison or yacc clone.  Lemon
+But Lemon is not a bison or yacc clone.  Lemon
 uses a different grammar syntax which is designed to
 reduce the number of coding errors.  Lemon also uses a
 parsing engine that is faster than yacc and
@@ -16,7 +16,7 @@ bison and which is both reentrant and threadsafe.
 has also been updated so that it too can generate a
 reentrant and threadsafe parser.)
 Lemon also implements features that can be used
-to eliminate resource leaks, making is suitable for use
+to eliminate resource leaks, making it suitable for use
 in long-running programs such as graphical user interfaces
 or embedded controllers.</p>
 
@@ -58,8 +58,8 @@ Lemon comes with a default parser template which works fine for most
 applications.  But the user is free to substitute a different parser
 template if desired.</p>
 
-<p>Depending on command-line options, Lemon will generate between
+<p>Depending on command-line options, Lemon will generate up to
-one and three files of outputs.
+three output files.
 <ul>
 <li>C code to implement the parser.
 <li>A header file defining an integer ID for each terminal symbol.
@@ -90,17 +90,23 @@ the states used by the parser automaton.</p>
 You can obtain a list of the available command-line options together
 with a brief explanation of what each does by typing
 <pre>
-   lemon -?
+   lemon "-?"
 </pre>
 As of this writing, the following command-line options are supported:
 <ul>
 <li><b>-b</b>
 Show only the basis for each parser state in the report file.
 <li><b>-c</b>
-Do not compress the generated action tables.
+Do not compress the generated action tables.  The parser will be a
+little larger and slower, but it will detect syntax errors sooner.
+<li><b>-d</b><i>directory</i>
+Write all output files into <i>directory</i>.  Normally, output files
+are written into the directory that contains the input grammar file.
 <li><b>-D<i>name</i></b>
-Define C preprocessor macro <i>name</i>.  This macro is useable by
+Define C preprocessor macro <i>name</i>.  This macro is usable by
-"%ifdef" lines in the grammar file.
+"<tt><a href='#pifdef'>%ifdef</a></tt>" and
+"<tt><a href='#pifdef'>%ifndef</a></tt>" lines
+in the grammar file.
 <li><b>-g</b>
 Do not generate a parser.  Instead write the input grammar to standard
 output with all comments, actions, and other extraneous text removed.
@@ -108,9 +114,9 @@ output with all comments, actions, and other extraneous text removed.
 Omit "#line" directives in the generated parser C code.
 <li><b>-m</b>
 Cause the output C source code to be compatible with the "makeheaders"
-program. 
+program.
 <li><b>-p</b>
-Display all conflicts that are resolved by 
+Display all conflicts that are resolved by
 <a href='#precrules'>precedence rules</a>.
 <li><b>-q</b>
 Suppress generation of the report file.
@@ -165,7 +171,7 @@ once for each token:
 </pre>
 The first argument to the Parse() routine is the pointer returned by
 ParseAlloc().
-The second argument is a small positive integer that tells the parse the
+The second argument is a small positive integer that tells the parser the
 type of the next token in the data stream.
 There is one token type for each terminal symbol in the grammar.
 The gram.h file generated by Lemon contains #define statements that
@@ -173,7 +179,7 @@ map symbolic terminal symbol names into appropriate integer values.
 A value of 0 for the second argument is a special flag to the
 parser to indicate that the end of input has been reached.
 The third argument is the value of the given token.  By default,
-the type of the third argument is integer, but the grammar will
+the type of the third argument is "void*", but the grammar will
 usually redefine this type to be some kind of structure.
 Typically the second argument will be a broad category of tokens
 such as "identifier" or "number" and the third argument will
@@ -181,7 +187,7 @@ be the name of the identifier or the value of the number.</p>
 
 <p>The Parse() function may have either three or four arguments,
 depending on the grammar.  If the grammar specification file requests
-it (via the <a href='#extraarg'><tt>extra_argument</tt> directive</a>),
+it (via the <tt><a href='#extraarg'>%extra_argument</a></tt> directive),
 the Parse() function will have a fourth parameter that can be
 of any type chosen by the programmer.  The parser doesn't do anything
 with this argument except to pass it through to action routines.
@@ -191,20 +197,20 @@ to the action routines without having to use global variables.</p>
 <p>A typical use of a Lemon parser might look something like the
 following:
 <pre>
-   01 ParseTree *ParseFile(const char *zFilename){
+    1 ParseTree *ParseFile(const char *zFilename){
-   02    Tokenizer *pTokenizer;
+    2    Tokenizer *pTokenizer;
-   03    void *pParser;
+    3    void *pParser;
-   04    Token sToken;
+    4    Token sToken;
-   05    int hTokenId;
+    5    int hTokenId;
-   06    ParserState sState;
+    6    ParserState sState;
-   07
+    7
-   08    pTokenizer = TokenizerCreate(zFilename);
+    8    pTokenizer = TokenizerCreate(zFilename);
-   09    pParser = ParseAlloc( malloc );
+    9    pParser = ParseAlloc( malloc );
-   10    InitParserState(&sState);
+   10    InitParserState(&amp;sState);
-   11    while( GetNextToken(pTokenizer, &hTokenId, &sToken) ){
+   11    while( GetNextToken(pTokenizer, &amp;hTokenId, &amp;sToken) ){
-   12       Parse(pParser, hTokenId, sToken, &sState);
+   12       Parse(pParser, hTokenId, sToken, &amp;sState);
    13    }
-   14    Parse(pParser, 0, sToken, &sState);
+   14    Parse(pParser, 0, sToken, &amp;sState);
    15    ParseFree(pParser, free );
    16    TokenizerFree(pTokenizer);
    17    return sState.treeRoot;
@@ -217,10 +223,10 @@ simple.)
 We assume the existence of some kind of tokenizer which is created
 using TokenizerCreate() on line 8 and deleted by TokenizerFree()
 on line 16.  The GetNextToken() function on line 11 retrieves the
-next token from the input file and puts its type in the 
+next token from the input file and puts its type in the
 integer variable hTokenId.  The sToken variable is assumed to be
 some kind of structure that contains details about each token,
-such as its complete text, what line it occurs on, etc. </p>
+such as its complete text, what line it occurs on, etc.</p>
 
 <p>This example also assumes the existence of structure of type
 ParserState that holds state information about a particular parse.
@@ -237,7 +243,7 @@ tree.</p>
 <pre>
    ParseFile(){
       pParser = ParseAlloc( malloc );
-      while( GetNextToken(pTokenizer,&hTokenId, &sToken) ){
+      while( GetNextToken(pTokenizer,&amp;hTokenId, &amp;sToken) ){
          Parse(pParser, hTokenId, sToken);
       }
       Parse(pParser, 0, sToken);
@@ -297,25 +303,25 @@ specifies additional information Lemon requires to do its job.
 Most of the work in using Lemon is in writing an appropriate
 grammar file.</p>
 
-<p>The grammar file for lemon is, for the most part, free format.
+<p>The grammar file for Lemon is, for the most part, free format.
 It does not have sections or divisions like yacc or bison.  Any
 declaration can occur at any point in the file.
 Lemon ignores whitespace (except where it is needed to separate
-tokens) and it honors the same commenting conventions as C and C++.</p>
+tokens), and it honors the same commenting conventions as C and C++.</p>
 
 <h3>Terminals and Nonterminals</h3>
 
 <p>A terminal symbol (token) is any string of alphanumeric
 and/or underscore characters
-that begins with an upper case letter.
+that begins with an uppercase letter.
 A terminal can contain lowercase letters after the first character,
-but the usual convention is to make terminals all upper case.
+but the usual convention is to make terminals all uppercase.
 A nonterminal, on the other hand, is any string of alphanumeric
-and underscore characters than begins with a lower case letter.
+and underscore characters than begins with a lowercase letter.
-Again, the usual convention is to make nonterminals use all lower
+Again, the usual convention is to make nonterminals use all lowercase
-case letters.</p>
+letters.</p>
 
-<p>In Lemon, terminal and nonterminal symbols do not need to 
+<p>In Lemon, terminal and nonterminal symbols do not need to
 be declared or identified in a separate section of the grammar file.
 Lemon is able to generate a list of all terminals and nonterminals
 by examining the grammar rules, and it can always distinguish a
@@ -339,7 +345,8 @@ The list of terminals and nonterminals on the right-hand side of the
 rule can be empty.
 Rules can occur in any order, except that the left-hand side of the
 first rule is assumed to be the start symbol for the grammar (unless
-specified otherwise using the <tt>%start</tt> directive described below.)
+specified otherwise using the <tt><a href='#start_symbol'>%start_symbol</a></tt>
+directive described below.)
 A typical sequence of grammar rules might look something like this:
 <pre>
   expr ::= expr PLUS expr.
@@ -382,7 +389,7 @@ names to each symbol in a grammar rule and then using those symbolic
 names in the action.
 In yacc or bison, one would write this:
 <pre>
-  expr -> expr PLUS expr  { $$ = $1 + $3; };
+  expr -&gt; expr PLUS expr  { $$ = $1 + $3; };
 </pre>
 But in Lemon, the same rule becomes the following:
 <pre>
@@ -422,14 +429,14 @@ of the shift, and a reduce-reduce conflict is resolved by reducing
 whichever rule comes first in the grammar file.</p>
 
 <p>Just like in
-yacc and bison, Lemon allows a measure of control 
+yacc and bison, Lemon allows a measure of control
-over the resolution of paring conflicts using precedence rules.
+over the resolution of parsing conflicts using precedence rules.
 A precedence value can be assigned to any terminal symbol
-using the 
+using the
-<a href='#pleft'>%left</a>,
+<tt><a href='#pleft'>%left</a></tt>,
-<a href='#pright'>%right</a> or
+<tt><a href='#pright'>%right</a></tt> or
-<a href='#pnonassoc'>%nonassoc</a> directives.  Terminal symbols
+<tt><a href='#pnonassoc'>%nonassoc</a></tt> directives.  Terminal symbols
-mentioned in earlier directives have a lower precedence that
+mentioned in earlier directives have a lower precedence than
 terminal symbols mentioned in later directives.  For example:</p>
 
 <p><pre>
@@ -505,29 +512,29 @@ as follows:
 <li> If the precedence of the token to be shifted is greater than
      the precedence of the rule to reduce, then resolve in favor
      of the shift.  No parsing conflict is reported.
-<li> If the precedence of the token it be shifted is less than the
+<li> If the precedence of the token to be shifted is less than the
      precedence of the rule to reduce, then resolve in favor of the
      reduce action.  No parsing conflict is reported.
 <li> If the precedences are the same and the shift token is
      right-associative, then resolve in favor of the shift.
      No parsing conflict is reported.
-<li> If the precedences are the same the shift token is
+<li> If the precedences are the same and the shift token is
      left-associative, then resolve in favor of the reduce.
      No parsing conflict is reported.
-<li> Otherwise, resolve the conflict by doing the shift and
+<li> Otherwise, resolve the conflict by doing the shift, and
-     report the parsing conflict.
+     report a parsing conflict.
 </ul>
 Reduce-reduce conflicts are resolved this way:
 <ul>
-<li> If either reduce rule 
+<li> If either reduce rule
      lacks precedence information, then resolve in favor of the
-     rule that appears first in the grammar and report a parsing
+     rule that appears first in the grammar, and report a parsing
      conflict.
-<li> If both rules have precedence and the precedence is different
+<li> If both rules have precedence and the precedence is different,
      then resolve the dispute in favor of the rule with the highest
-     precedence and do not report a conflict.
+     precedence, and do not report a conflict.
 <li> Otherwise, resolve the conflict by reducing by the rule that
-     appears first in the grammar and report a parsing conflict.
+     appears first in the grammar, and report a parsing conflict.
 </ul>
 
 <h3>Special Directives</h3>
@@ -536,40 +543,40 @@ Reduce-reduce conflicts are resolved this way:
 directives.  We've described all the grammar rules, so now we'll
 talk about the special directives.</p>
 
-<p>Directives in lemon can occur in any order.  You can put them before
+<p>Directives in Lemon can occur in any order.  You can put them before
-the grammar rules, or after the grammar rules, or in the mist of the
+the grammar rules, or after the grammar rules, or in the midst of the
 grammar rules.  It doesn't matter.  The relative order of
 directives used to assign precedence to terminals is important, but
 other than that, the order of directives in Lemon is arbitrary.</p>
 
 <p>Lemon supports the following special directives:
 <ul>
-<li><tt>%code</tt>
+<li><tt><a href='#pcode'>%code</a></tt>
-<li><tt>%default_destructor</tt>
+<li><tt><a href='#default_destructor'>%default_destructor</a></tt>
-<li><tt>%default_type</tt>
+<li><tt><a href='#default_type'>%default_type</a></tt>
-<li><tt>%destructor</tt>
+<li><tt><a href='#destructor'>%destructor</a></tt>
-<li><tt>%endif</tt>
+<li><tt><a href='#pifdef'>%endif</a></tt>
-<li><tt>%extra_argument</tt>
+<li><tt><a href='#extraarg'>%extra_argument</a></tt>
-<li><tt>%fallback</tt>
+<li><tt><a href='#pfallback'>%fallback</a></tt>
-<li><tt>%ifdef</tt>
+<li><tt><a href='#pifdef'>%ifdef</a></tt>
-<li><tt>%ifndef</tt>
+<li><tt><a href='#pifdef'>%ifndef</a></tt>
-<li><tt>%include</tt>
+<li><tt><a href='#pinclude'>%include</a></tt>
-<li><tt>%left</tt>
+<li><tt><a href='#pleft'>%left</a></tt>
-<li><tt>%name</tt>
+<li><tt><a href='#pname'>%name</a></tt>
-<li><tt>%nonassoc</tt>
+<li><tt><a href='#pnonassoc'>%nonassoc</a></tt>
-<li><tt>%parse_accept</tt>
+<li><tt><a href='#parse_accept'>%parse_accept</a></tt>
-<li><tt>%parse_failure </tt>
+<li><tt><a href='#parse_failure'>%parse_failure</a></tt>
-<li><tt>%right</tt>
+<li><tt><a href='#pright'>%right</a></tt>
-<li><tt>%stack_overflow</tt>
+<li><tt><a href='#stack_overflow'>%stack_overflow</a></tt>
-<li><tt>%stack_size</tt>
+<li><tt><a href='#stack_size'>%stack_size</a></tt>
-<li><tt>%start_symbol</tt>
+<li><tt><a href='#start_symbol'>%start_symbol</a></tt>
-<li><tt>%syntax_error</tt>
+<li><tt><a href='#syntax_error'>%syntax_error</a></tt>
-<li><tt>%token_class</tt>
+<li><tt><a href='#token_class'>%token_class</a></tt>
-<li><tt>%token_destructor</tt>
+<li><tt><a href='#token_destructor'>%token_destructor</a></tt>
-<li><tt>%token_prefix</tt>
+<li><tt><a href='#token_prefix'>%token_prefix</a></tt>
-<li><tt>%token_type</tt>
+<li><tt><a href='#token_type'>%token_type</a></tt>
-<li><tt>%type</tt>
+<li><tt><a href='#ptype'>%type</a></tt>
-<li><tt>%wildcard</tt>
+<li><tt><a href='#pwildcard'>%wildcard</a></tt>
 </ul>
 Each of these directives will be described separately in the
 following sections:</p>
@@ -577,43 +584,42 @@ following sections:</p>
 <a name='pcode'></a>
 <h4>The <tt>%code</tt> directive</h4>
 
-<p>The %code directive is used to specify addition C code that
+<p>The <tt>%code</tt> directive is used to specify additional C code that
 is added to the end of the main output file.  This is similar to
-the <a href='#pinclude'>%include</a> directive except that %include
+the <tt><a href='#pinclude'>%include</a></tt> directive except that
-is inserted at the beginning of the main output file.</p>
+<tt>%include</tt> is inserted at the beginning of the main output file.</p>
 
-<p>%code is typically used to include some action routines or perhaps
+<p><tt>%code</tt> is typically used to include some action routines or perhaps
-a tokenizer or even the "main()" function 
+a tokenizer or even the "main()" function
 as part of the output file.</p>
 
 <a name='default_destructor'></a>
 <h4>The <tt>%default_destructor</tt> directive</h4>
 
-<p>The %default_destructor directive specifies a destructor to 
+<p>The <tt>%default_destructor</tt> directive specifies a destructor to
 use for non-terminals that do not have their own destructor
-specified by a separate %destructor directive.  See the documentation
+specified by a separate <tt>%destructor</tt> directive.  See the documentation
-on the <a name='#destructor'>%destructor</a> directive below for
+on the <tt><a name='#destructor'>%destructor</a></tt> directive below for
 additional information.</p>
 
-<p>In some grammers, many different non-terminal symbols have the
+<p>In some grammars, many different non-terminal symbols have the
-same datatype and hence the same destructor.  This directive is
+same data type and hence the same destructor.  This directive is
-a convenience way to specify the same destructor for all those
+a convenient way to specify the same destructor for all those
 non-terminals using a single statement.</p>
 
 <a name='default_type'></a>
 <h4>The <tt>%default_type</tt> directive</h4>
 
-<p>The %default_type directive specifies the datatype of non-terminal
+<p>The <tt>%default_type</tt> directive specifies the data type of non-terminal
-symbols that do no have their own datatype defined using a separate
+symbols that do not have their own data type defined using a separate
-<a href='#ptype'>%type</a> directive.  
+<tt><a href='#ptype'>%type</a></tt> directive.</p>
-</p>
 
 <a name='destructor'></a>
 <h4>The <tt>%destructor</tt> directive</h4>
 
-<p>The %destructor directive is used to specify a destructor for
+<p>The <tt>%destructor</tt> directive is used to specify a destructor for
 a non-terminal symbol.
-(See also the <a href='#token_destructor'>%token_destructor</a>
+(See also the <tt><a href='#token_destructor'>%token_destructor</a></tt>
 directive which is used to specify a destructor for terminal symbols.)</p>
 
 <p>A non-terminal's destructor is called to dispose of the
@@ -635,7 +641,7 @@ or other resources held by that non-terminal.</p>
    %destructor nt { free($$); }
    nt(A) ::= ID NUM.   { A = malloc( 100 ); }
 </pre>
-This example is a bit contrived but it serves to illustrate how
+This example is a bit contrived, but it serves to illustrate how
 destructors work.  The example shows a non-terminal named
 "nt" that holds values of type "void*".  When the rule for
 an "nt" reduces, it sets the value of the non-terminal to
@@ -651,17 +657,17 @@ stack, unless the non-terminal is used in a C-code action.  If
 the non-terminal is used by C-code, then it is assumed that the
 C-code will take care of destroying it.
 More commonly, the value is used to build some
-larger structure and we don't want to destroy it, which is why
+larger structure, and we don't want to destroy it, which is why
 the destructor is not called in this circumstance.</p>
 
 <p>Destructors help avoid memory leaks by automatically freeing
 allocated objects when they go out of scope.
 To do the same using yacc or bison is much more difficult.</p>
 
-<a name="extraarg"></a>
+<a name='extraarg'></a>
 <h4>The <tt>%extra_argument</tt> directive</h4>
 
-The %extra_argument directive instructs Lemon to add a 4th parameter
+The <tt>%extra_argument</tt> directive instructs Lemon to add a 4th parameter
 to the parameter list of the Parse() function it generates.  Lemon
 doesn't do anything itself with this extra argument, but it does
 make the argument available to C-code action routines, destructors,
@@ -676,64 +682,91 @@ of type "MyStruct*" and all action routines will have access to
 a variable named "pAbc" that is the value of the 4th parameter
 in the most recent call to Parse().</p>
 
+<p>The <tt>%extra_context</tt> directive works the same except that it
+is passed in on the ParseAlloc() or ParseInit() routines instead of
+on Parse().
+
+<a name='extractx'></a>
+<h4>The <tt>%extra_context</tt> directive</h4>
+
+The <tt>%extra_context</tt> directive instructs Lemon to add a 2th parameter
+to the parameter list of the ParseAlloc() and ParseInif() functions.  Lemon
+doesn't do anything itself with these extra argument, but it does
+store the value make it available to C-code action routines, destructors,
+and so forth.  For example, if the grammar file contains:</p>
+
+<p><pre>
+    %extra_context { MyStruct *pAbc }
+</pre></p>
+
+<p>Then the ParseAlloc() and ParseInit() functions will have an 2th parameter
+of type "MyStruct*" and all action routines will have access to
+a variable named "pAbc" that is the value of that 2th parameter.</p>
+
+<p>The <tt>%extra_argument</tt> directive works the same except that it
+is passed in on the Parse() routine instead of on ParseAlloc()/ParseInit().
+
 <a name='pfallback'></a>
 <h4>The <tt>%fallback</tt> directive</h4>
 
-<p>The %fallback directive specifies an alternative meaning for one
+<p>The <tt>%fallback</tt> directive specifies an alternative meaning for one
 or more tokens.  The alternative meaning is tried if the original token
-would have generated a syntax error.
+would have generated a syntax error.</p>
 
-<p>The %fallback directive was added to support robust parsing of SQL
+<p>The <tt>%fallback</tt> directive was added to support robust parsing of SQL
-syntax in <a href="https://www.sqlite.org/">SQLite</a>.
+syntax in <a href='https://www.sqlite.org/'>SQLite</a>.
 The SQL language contains a large assortment of keywords, each of which
 appears as a different token to the language parser.  SQL contains so
-many keywords, that it can be difficult for programmers to keep up with
+many keywords that it can be difficult for programmers to keep up with
 them all.  Programmers will, therefore, sometimes mistakenly use an
-obscure language keyword for an identifier.  The %fallback directive
+obscure language keyword for an identifier.  The <tt>%fallback</tt> directive
 provides a mechanism to tell the parser:  "If you are unable to parse
-this keyword, try treating it as an identifier instead."
+this keyword, try treating it as an identifier instead."</p>
 
-<p>The syntax of %fallback is as follows:
+<p>The syntax of <tt>%fallback</tt> is as follows:
 
 <blockquote>
-<tt>%fallback</tt>  <i>ID</i> <i>TOKEN...</i> <b>.</b>
+<tt>%fallback</tt> <i>ID</i> <i>TOKEN...</i> <b>.</b>
-</blockquote>
+</blockquote></p>
 
-<p>In words, the %fallback directive is followed by a list of token names
+<p>In words, the <tt>%fallback</tt> directive is followed by a list of token
-terminated by a period.  The first token name is the fallback token - the
+names terminated by a period.
+The first token name is the fallback token &mdash; the
 token to which all the other tokens fall back to.  The second and subsequent
 arguments are tokens which fall back to the token identified by the first
-argument.
+argument.</p>
 
 <a name='pifdef'></a>
-<h4>The <tt>%ifdef</tt>, <tt>%ifndef</tt>, and <tt>%endif</tt> directives.</h4>
+<h4>The <tt>%ifdef</tt>, <tt>%ifndef</tt>, and <tt>%endif</tt> directives</h4>
 
-<p>The %ifdef, %ifndef, and %endif directives are similar to
+<p>The <tt>%ifdef</tt>, <tt>%ifndef</tt>, and <tt>%endif</tt> directives
-#ifdef, #ifndef, and #endif in the C-preprocessor, just not as general.
+are similar to #ifdef, #ifndef, and #endif in the C-preprocessor,
+just not as general.
 Each of these directives must begin at the left margin.  No whitespace
-is allowed between the "%" and the directive name.
+is allowed between the "%" and the directive name.</p>
 
-<p>Grammar text in between "%ifdef MACRO" and the next nested "%endif" is
+<p>Grammar text in between "<tt>%ifdef MACRO</tt>" and the next nested
+"<tt>%endif</tt>" is
 ignored unless the "-DMACRO" command-line option is used.  Grammar text
-betwen "%ifndef MACRO" and the next nested "%endif" is included except when
+betwen "<tt>%ifndef MACRO</tt>" and the next nested "<tt>%endif</tt>" is
-the "-DMACRO" command-line option is used.
+included except when the "-DMACRO" command-line option is used.</p>
 
-<p>Note that the argument to %ifdef and %ifndef must be a single 
+<p>Note that the argument to <tt>%ifdef</tt> and <tt>%ifndef</tt> must
-preprocessor symbol name, not a general expression.  There is no "%else"
+be a single preprocessor symbol name, not a general expression.
-directive.
+There is no "<tt>%else</tt>" directive.</p>
 
 
 <a name='pinclude'></a>
 <h4>The <tt>%include</tt> directive</h4>
 
-<p>The %include directive specifies C code that is included at the
+<p>The <tt>%include</tt> directive specifies C code that is included at the
-top of the generated parser.  You can include any text you want --
+top of the generated parser.  You can include any text you want &mdash;
 the Lemon parser generator copies it blindly.  If you have multiple
-%include directives in your grammar file, their values are concatenated
+<tt>%include</tt> directives in your grammar file, their values are concatenated
-so that all %include code ultimately appears near the top of the
+so that all <tt>%include</tt> code ultimately appears near the top of the
-generated parser, in the same order as it appeared in the grammer.</p>
+generated parser, in the same order as it appeared in the grammar.</p>
 
-<p>The %include directive is very handy for getting some extra #include
+<p>The <tt>%include</tt> directive is very handy for getting some extra #include
 preprocessor statements at the beginning of the generated parser.
 For example:</p>
 
@@ -742,17 +775,19 @@ For example:</p>
 </pre></p>
 
 <p>This might be needed, for example, if some of the C actions in the
-grammar call functions that are prototyed in unistd.h.</p>
+grammar call functions that are prototyped in unistd.h.</p>
 
 <a name='pleft'></a>
 <h4>The <tt>%left</tt> directive</h4>
 
-The %left directive is used (along with the <a href='#pright'>%right</a> and
+The <tt>%left</tt> directive is used (along with the
-<a href='#pnonassoc'>%nonassoc</a> directives) to declare precedences of 
+<tt><a href='#pright'>%right</a></tt> and
-terminal symbols.  Every terminal symbol whose name appears after
+<tt><a href='#pnonassoc'>%nonassoc</a></tt> directives) to declare
-a %left directive but before the next period (".") is
+precedences of terminal symbols.
+Every terminal symbol whose name appears after
+a <tt>%left</tt> directive but before the next period (".") is
 given the same left-associative precedence value.  Subsequent
-%left directives have higher precedence.  For example:</p>
+<tt>%left</tt> directives have higher precedence.  For example:</p>
 
 <p><pre>
    %left AND.
@@ -763,20 +798,21 @@ given the same left-associative precedence value.  Subsequent
    %right EXP NOT.
 </pre></p>
 
-<p>Note the period that terminates each %left, %right or %nonassoc
+<p>Note the period that terminates each <tt>%left</tt>,
+<tt>%right</tt> or <tt>%nonassoc</tt>
 directive.</p>
 
 <p>LALR(1) grammars can get into a situation where they require
 a large amount of stack space if you make heavy use or right-associative
-operators.  For this reason, it is recommended that you use %left
+operators.  For this reason, it is recommended that you use <tt>%left</tt>
-rather than %right whenever possible.</p>
+rather than <tt>%right</tt> whenever possible.</p>
 
 <a name='pname'></a>
 <h4>The <tt>%name</tt> directive</h4>
 
 <p>By default, the functions generated by Lemon all begin with the
 five-character string "Parse".  You can change this string to something
-different using the %name directive.  For instance:</p>
+different using the <tt>%name</tt> directive.  For instance:</p>
 
 <p><pre>
    %name Abcde
@@ -790,22 +826,22 @@ functions named
 <li> AbcdeTrace(), and
 <li> Abcde().
 </ul>
-The %name directive allows you to generator two or more different
+The <tt>%name</tt> directive allows you to generate two or more different
-parsers and link them all into the same executable.
+parsers and link them all into the same executable.</p>
-</p>
 
 <a name='pnonassoc'></a>
 <h4>The <tt>%nonassoc</tt> directive</h4>
 
 <p>This directive is used to assign non-associative precedence to
-one or more terminal symbols.  See the section on 
+one or more terminal symbols.  See the section on
 <a href='#precrules'>precedence rules</a>
-or on the <a href='#pleft'>%left</a> directive for additional information.</p>
+or on the <tt><a href='#pleft'>%left</a></tt> directive
+for additional information.</p>
 
 <a name='parse_accept'></a>
 <h4>The <tt>%parse_accept</tt> directive</h4>
 
-<p>The %parse_accept directive specifies a block of C code that is
+<p>The <tt>%parse_accept</tt> directive specifies a block of C code that is
 executed whenever the parser accepts its input string.  To "accept"
 an input string means that the parser was able to process all tokens
 without error.</p>
@@ -821,7 +857,7 @@ without error.</p>
 <a name='parse_failure'></a>
 <h4>The <tt>%parse_failure</tt> directive</h4>
 
-<p>The %parse_failure directive specifies a block of C code that
+<p>The <tt>%parse_failure</tt> directive specifies a block of C code that
 is executed whenever the parser fails complete.  This code is not
 executed until the parser has tried and failed to resolve an input
 error using is usual error recovery strategy.  The routine is
@@ -837,14 +873,14 @@ only invoked when parsing is unable to continue.</p>
 <h4>The <tt>%right</tt> directive</h4>
 
 <p>This directive is used to assign right-associative precedence to
-one or more terminal symbols.  See the section on 
+one or more terminal symbols.  See the section on
 <a href='#precrules'>precedence rules</a>
 or on the <a href='#pleft'>%left</a> directive for additional information.</p>
 
 <a name='stack_overflow'></a>
 <h4>The <tt>%stack_overflow</tt> directive</h4>
 
-<p>The %stack_overflow directive specifies a block of C code that
+<p>The <tt>%stack_overflow</tt> directive specifies a block of C code that
 is executed if the parser's internal stack ever overflows.  Typically
 this just prints an error message.  After a stack overflow, the parser
 will be unable to continue and must be reset.</p>
@@ -857,7 +893,7 @@ will be unable to continue and must be reset.</p>
 
 <p>You can help prevent parser stack overflows by avoiding the use
 of right recursion and right-precedence operators in your grammar.
-Use left recursion and and left-precedence operators instead, to
+Use left recursion and and left-precedence operators instead to
 encourage rules to reduce sooner and keep the stack size down.
 For example, do rules like this:
 <pre>
@@ -868,7 +904,7 @@ Not like this:
 <pre>
    list ::= element list.      // right-recursion.  Bad!
    list ::= .
-</pre>
+</pre></p>
 
 <a name='stack_size'></a>
 <h4>The <tt>%stack_size</tt> directive</h4>
@@ -876,7 +912,7 @@ Not like this:
 <p>If stack overflow is a problem and you can't resolve the trouble
 by using left-recursion, then you might want to increase the size
 of the parser's stack using this directive.  Put an positive integer
-after the %stack_size directive and Lemon will generate a parse
+after the <tt>%stack_size</tt> directive and Lemon will generate a parse
 with a stack of the requested size.  The default value is 100.</p>
 
 <p><pre>
@@ -886,25 +922,40 @@ with a stack of the requested size.  The default value is 100.</p>
 <a name='start_symbol'></a>
 <h4>The <tt>%start_symbol</tt> directive</h4>
 
-<p>By default, the start-symbol for the grammar that Lemon generates
+<p>By default, the start symbol for the grammar that Lemon generates
 is the first non-terminal that appears in the grammar file.  But you
-can choose a different start-symbol using the %start_symbol directive.</p>
+can choose a different start symbol using the
+<tt>%start_symbol</tt> directive.</p>
 
 <p><pre>
    %start_symbol  prog
 </pre></p>
 
+<a name='syntax_error'></a>
+<h4>The <tt>%syntax_error</tt> directive</h4>
+
+<p>See <a href='#error_processing'>Error Processing</a>.</p>
+
+<a name='token_class'></a>
+<h4>The <tt>%token_class</tt> directive</h4>
+
+<p>Undocumented.  Appears to be related to the MULTITERMINAL concept.
+<a href='http://sqlite.org/src/fdiff?v1=796930d5fc2036c7&v2=624b24c5dc048e09&sbs=0'>Implementation</a>.</p>
+
 <a name='token_destructor'></a>
 <h4>The <tt>%token_destructor</tt> directive</h4>
 
-<p>The %destructor directive assigns a destructor to a non-terminal
+<p>The <tt>%destructor</tt> directive assigns a destructor to a non-terminal
-symbol.  (See the description of the %destructor directive above.)
+symbol.  (See the description of the
-This directive does the same thing for all terminal symbols.</p>
+<tt><a href='%destructor'>%destructor</a></tt> directive above.)
+The <tt>%token_destructor</tt> directive does the same thing
+for all terminal symbols.</p>
 
 <p>Unlike non-terminal symbols which may each have a different data type
 for their values, terminals all use the same data type (defined by
-the %token_type directive) and so they use a common destructor.  Other
+the <tt><a href='#token_type'>%token_type</a></tt> directive)
-than that, the token destructor works just like the non-terminal
+and so they use a common destructor.
+Other than that, the token destructor works just like the non-terminal
 destructors.</p>
 
 <a name='token_prefix'></a>
@@ -913,8 +964,9 @@ destructors.</p>
 <p>Lemon generates #defines that assign small integer constants
 to each terminal symbol in the grammar.  If desired, Lemon will
 add a prefix specified by this directive
-to each of the #defines it generates.
+to each of the #defines it generates.</p>
-So if the default output of Lemon looked like this:
+
+<p>So if the default output of Lemon looked like this:
 <pre>
     #define AND              1
     #define MINUS            2
@@ -931,7 +983,7 @@ to cause Lemon to produce these symbols instead:
     #define TOKEN_MINUS      2
     #define TOKEN_OR         3
     #define TOKEN_PLUS       4
-</pre>
+</pre></p>
 
 <a name='token_type'></a><a name='ptype'></a>
 <h4>The <tt>%token_type</tt> and <tt>%type</tt> directives</h4>
@@ -952,7 +1004,7 @@ token structure.  Like this:</p>
 is "void*".</p>
 
 <p>Non-terminal symbols can each have their own data types.  Typically
-the data type  of a non-terminal is a pointer to the root of a parse-tree
+the data type of a non-terminal is a pointer to the root of a parse tree
 structure that contains all information about that non-terminal.
 For example:</p>
 
@@ -973,14 +1025,15 @@ and able to pay that price, fine.  You just need to know.</p>
 <a name='pwildcard'></a>
 <h4>The <tt>%wildcard</tt> directive</h4>
 
-<p>The %wildcard directive is followed by a single token name and a
+<p>The <tt>%wildcard</tt> directive is followed by a single token name and a
-period.  This directive specifies that the identified token should 
+period.  This directive specifies that the identified token should
-match any input token.
+match any input token.</p>
 
 <p>When the generated parser has the choice of matching an input against
 the wildcard token and some other token, the other token is always used.
-The wildcard token is only matched if there are no other alternatives.
+The wildcard token is only matched if there are no alternatives.</p>
 
+<a name='error_processing'></a>
 <h3>Error Processing</h3>
 
 <p>After extensive experimentation over several years, it has been
@@ -988,19 +1041,20 @@ discovered that the error recovery strategy used by yacc is about
 as good as it gets.  And so that is what Lemon uses.</p>
 
 <p>When a Lemon-generated parser encounters a syntax error, it
-first invokes the code specified by the %syntax_error directive, if
+first invokes the code specified by the <tt>%syntax_error</tt> directive, if
 any.  It then enters its error recovery strategy.  The error recovery
 strategy is to begin popping the parsers stack until it enters a
 state where it is permitted to shift a special non-terminal symbol
 named "error".  It then shifts this non-terminal and continues
-parsing.  But the %syntax_error routine will not be called again
+parsing.  The <tt>%syntax_error</tt> routine will not be called again
 until at least three new tokens have been successfully shifted.</p>
 
 <p>If the parser pops its stack until the stack is empty, and it still
-is unable to shift the error symbol, then the %parse_failed routine
+is unable to shift the error symbol, then the
+<tt><a href='#parse_failure'>%parse_failure</a></tt> routine
 is invoked and the parser resets itself to its start state, ready
 to begin parsing a new file.  This is what will happen at the very
-first syntax error, of course, if there are no instances of the 
+first syntax error, of course, if there are no instances of the
 "error" non-terminal in your grammar.</p>
 
 </body>

ext/expert/README.md

@@ -0,0 +1,83 @@
+## SQLite Expert Extension
+
+This folder contains code for a simple system to propose useful indexes
+given a database and a set of SQL queries. It works as follows:
+
+  1. The user database schema is copied to a temporary database.
+
+  1. All SQL queries are prepared against the temporary database.
+     Information regarding the WHERE and ORDER BY clauses, and other query
+     features that affect index selection are recorded.
+
+  1. The information gathered in step 2 is used to create candidate 
+     indexes - indexes that the planner might have made use of in the previous
+     step, had they been available.
+
+  1. A subset of the data in the user database is used to generate statistics
+     for all existing indexes and the candidate indexes generated in step 3
+     above.
+
+  1. The SQL queries are prepared a second time. If the planner uses any
+     of the indexes created in step 3, they are recommended to the user.
+
+# C API
+
+The SQLite expert C API is defined in sqlite3expert.h. Most uses will proceed
+as follows:
+
+  1. An sqlite3expert object is created by calling **sqlite3\_expert\_new()**.
+     A database handle opened by the user is passed as an argument.
+
+  1. The sqlite3expert object is configured with one or more SQL statements
+     by making one or more calls to **sqlite3\_expert\_sql()**. Each call may
+     specify a single SQL statement, or multiple statements separated by
+     semi-colons.
+  
+  1. Optionally, the **sqlite3\_expert\_config()** API may be used to 
+     configure the size of the data subset used to generate index statistics.
+     Using a smaller subset of the data can speed up the analysis.
+
+  1. **sqlite3\_expert\_analyze()** is called to run the analysis.
+
+  1. One or more calls are made to **sqlite3\_expert\_report()** to extract
+     components of the results of the analysis.
+
+  1. **sqlite3\_expert\_destroy()** is called to free all resources.
+
+Refer to comments in sqlite3expert.h for further details.
+
+# sqlite3_expert application
+
+The file "expert.c" contains the code for a command line application that
+uses the API described above. It can be compiled with (for example):
+
+<pre>
+  gcc -O2 sqlite3.c expert.c sqlite3expert.c -o sqlite3_expert
+</pre>
+
+Assuming the database is named "test.db", it can then be run to analyze a
+single query:
+
+<pre>
+  ./sqlite3_expert -sql &lt;sql-query&gt; test.db
+</pre>
+
+Or an entire text file worth of queries with:
+
+<pre>
+  ./sqlite3_expert -file &lt;text-file&gt; test.db
+</pre>
+
+By default, sqlite3\_expert generates index statistics using all the data in
+the user database. For a large database, this may be prohibitively time
+consuming. The "-sample" option may be used to configure sqlite3\_expert to
+generate statistics based on an integer percentage of the user database as
+follows:
+
+<pre>
+  # Generate statistics based on 25% of the user database rows:
+  ./sqlite3_expert -sample 25 -sql &lt;sql-query&gt; test.db
+
+  # Do not generate any statistics at all:
+  ./sqlite3_expert -sample 0 -sql &lt;sql-query&gt; test.db
+</pre>

ext/expert/expert.c

@@ -0,0 +1,156 @@
+/*
+** 2017 April 07
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+*/
+
+
+#include <sqlite3.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "sqlite3expert.h"
+
+
+static void option_requires_argument(const char *zOpt){
+  fprintf(stderr, "Option requires an argument: %s\n", zOpt);
+  exit(-3);
+}
+
+static int option_integer_arg(const char *zVal){
+  return atoi(zVal);
+}
+
+static void usage(char **argv){
+  fprintf(stderr, "\n");
+  fprintf(stderr, "Usage %s ?OPTIONS? DATABASE\n", argv[0]);
+  fprintf(stderr, "\n");
+  fprintf(stderr, "Options are:\n");
+  fprintf(stderr, "  -sql SQL   (analyze SQL statements passed as argument)\n");
+  fprintf(stderr, "  -file FILE (read SQL statements from file FILE)\n");
+  fprintf(stderr, "  -verbose LEVEL (integer verbosity level. default 1)\n");
+  fprintf(stderr, "  -sample PERCENT (percent of db to sample. default 100)\n");
+  exit(-1);
+}
+
+static int readSqlFromFile(sqlite3expert *p, const char *zFile, char **pzErr){
+  FILE *in = fopen(zFile, "rb");
+  long nIn;
+  size_t nRead;
+  char *pBuf;
+  int rc;
+  if( in==0 ){
+    *pzErr = sqlite3_mprintf("failed to open file %s\n", zFile);
+    return SQLITE_ERROR;
+  }
+  fseek(in, 0, SEEK_END);
+  nIn = ftell(in);
+  rewind(in);
+  pBuf = sqlite3_malloc64( nIn+1 );
+  nRead = fread(pBuf, nIn, 1, in);
+  fclose(in);
+  if( nRead!=1 ){
+    sqlite3_free(pBuf);
+    *pzErr = sqlite3_mprintf("failed to read file %s\n", zFile);
+    return SQLITE_ERROR;
+  }
+  pBuf[nIn] = 0;
+  rc = sqlite3_expert_sql(p, pBuf, pzErr);
+  sqlite3_free(pBuf);
+  return rc;
+}
+
+int main(int argc, char **argv){
+  const char *zDb;
+  int rc = 0;
+  char *zErr = 0;
+  int i;
+  int iVerbose = 1;               /* -verbose option */
+
+  sqlite3 *db = 0;
+  sqlite3expert *p = 0;
+
+  if( argc<2 ) usage(argv);
+  zDb = argv[argc-1];
+  if( zDb[0]=='-' ) usage(argv);
+  rc = sqlite3_open(zDb, &db);
+  if( rc!=SQLITE_OK ){
+    fprintf(stderr, "Cannot open db file: %s - %s\n", zDb, sqlite3_errmsg(db));
+    exit(-2);
+  }
+
+  p = sqlite3_expert_new(db, &zErr);
+  if( p==0 ){
+    fprintf(stderr, "Cannot run analysis: %s\n", zErr);
+    rc = 1;
+  }else{
+    for(i=1; i<(argc-1); i++){
+      char *zArg = argv[i];
+      int nArg;
+      if( zArg[0]=='-' && zArg[1]=='-' && zArg[2]!=0 ) zArg++;
+      nArg = (int)strlen(zArg);
+      if( nArg>=2 && 0==sqlite3_strnicmp(zArg, "-file", nArg) ){
+        if( ++i==(argc-1) ) option_requires_argument("-file");
+        rc = readSqlFromFile(p, argv[i], &zErr);
+      }
+
+      else if( nArg>=3 && 0==sqlite3_strnicmp(zArg, "-sql", nArg) ){
+        if( ++i==(argc-1) ) option_requires_argument("-sql");
+        rc = sqlite3_expert_sql(p, argv[i], &zErr);
+      }
+
+      else if( nArg>=3 && 0==sqlite3_strnicmp(zArg, "-sample", nArg) ){
+        int iSample;
+        if( ++i==(argc-1) ) option_requires_argument("-sample");
+        iSample = option_integer_arg(argv[i]);
+        sqlite3_expert_config(p, EXPERT_CONFIG_SAMPLE, iSample);
+      }
+
+      else if( nArg>=2 && 0==sqlite3_strnicmp(zArg, "-verbose", nArg) ){
+        if( ++i==(argc-1) ) option_requires_argument("-verbose");
+        iVerbose = option_integer_arg(argv[i]);
+      }
+
+      else{
+        usage(argv);
+      }
+    }
+  }
+
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_expert_analyze(p, &zErr);
+  }
+
+  if( rc==SQLITE_OK ){
+    int nQuery = sqlite3_expert_count(p);
+    if( iVerbose>0 ){
+      const char *zCand = sqlite3_expert_report(p,0,EXPERT_REPORT_CANDIDATES);
+      fprintf(stdout, "-- Candidates -------------------------------\n");
+      fprintf(stdout, "%s\n", zCand);
+    }
+    for(i=0; i<nQuery; i++){
+      const char *zSql = sqlite3_expert_report(p, i, EXPERT_REPORT_SQL);
+      const char *zIdx = sqlite3_expert_report(p, i, EXPERT_REPORT_INDEXES);
+      const char *zEQP = sqlite3_expert_report(p, i, EXPERT_REPORT_PLAN);
+      if( zIdx==0 ) zIdx = "(no new indexes)\n";
+      if( iVerbose>0 ){
+        fprintf(stdout, "-- Query %d ----------------------------------\n",i+1);
+        fprintf(stdout, "%s\n\n", zSql);
+      }
+      fprintf(stdout, "%s\n%s\n", zIdx, zEQP);
+    }
+  }else{
+    fprintf(stderr, "Error: %s\n", zErr ? zErr : "?");
+  }
+
+  sqlite3_expert_destroy(p);
+  sqlite3_free(zErr);
+  return rc;
+}

ext/expert/expert1.test

@@ -0,0 +1,382 @@
+# 2009 Nov 11
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# The focus of this file is testing the CLI shell tool. Specifically,
+# the ".recommend" command.
+#
+#
+
+# Test plan:
+#
+#
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+}
+source $testdir/tester.tcl
+set testprefix expert1
+
+if {[info commands sqlite3_expert_new]==""} {
+  finish_test
+  return
+}
+
+set CLI [test_binary_name sqlite3]
+set CMD [test_binary_name sqlite3_expert]
+
+proc squish {txt} {
+  regsub -all {[[:space:]]+} $txt { }
+}
+
+proc do_setup_rec_test {tn setup sql res} {
+  reset_db
+  db eval $setup
+  uplevel [list do_rec_test $tn $sql $res]
+}
+
+foreach {tn setup} {
+  1 {
+    if {![file executable $CMD]} { continue }
+
+    proc do_rec_test {tn sql res} {
+      set res [squish [string trim $res]]
+      set tst [subst -nocommands { 
+        squish [string trim [exec $::CMD -verbose 0 -sql {$sql;} test.db]]
+      }]
+      uplevel [list do_test $tn $tst $res]
+    }
+  }
+  2 {
+    if {[info commands sqlite3_expert_new]==""} { continue }
+
+    proc do_rec_test {tn sql res} {
+      set expert [sqlite3_expert_new db]
+      $expert sql $sql
+      $expert analyze
+
+      set result [list]
+      for {set i 0} {$i < [$expert count]} {incr i} {
+        set idx [string trim [$expert report $i indexes]]
+        if {$idx==""} {set idx "(no new indexes)"}
+        lappend result $idx
+        lappend result [string trim [$expert report $i plan]]
+      }
+
+      $expert destroy
+
+      set tst [subst -nocommands {set {} [squish [join {$result}]]}]
+      uplevel [list do_test $tn $tst [string trim [squish $res]]]
+    }
+  }
+  3 {
+    if {![file executable $CLI]} { continue }
+
+    proc do_rec_test {tn sql res} {
+      set res [squish [string trim $res]]
+      set tst [subst -nocommands { 
+        squish [string trim [exec $::CLI test.db ".expert" {$sql;}]]
+      }]
+      uplevel [list do_test $tn $tst $res]
+    }
+  }
+} {
+
+  eval $setup
+
+
+do_setup_rec_test $tn.1 { CREATE TABLE t1(a, b, c) } {
+  SELECT * FROM t1
+} {
+  (no new indexes)
+  SCAN TABLE t1
+}
+
+do_setup_rec_test $tn.2 {
+  CREATE TABLE t1(a, b, c);
+} {
+  SELECT * FROM t1 WHERE b>?;
+} {
+  CREATE INDEX t1_idx_00000062 ON t1(b);
+  SEARCH TABLE t1 USING INDEX t1_idx_00000062 (b>?)
+}
+
+do_setup_rec_test $tn.3 {
+  CREATE TABLE t1(a, b, c);
+} {
+  SELECT * FROM t1 WHERE b COLLATE nocase BETWEEN ? AND ?
+} {
+  CREATE INDEX t1_idx_3e094c27 ON t1(b COLLATE NOCASE);
+  SEARCH TABLE t1 USING INDEX t1_idx_3e094c27 (b>? AND b<?)
+}
+
+do_setup_rec_test $tn.4 {
+  CREATE TABLE t1(a, b, c);
+} {
+  SELECT a FROM t1 ORDER BY b;
+} {
+  CREATE INDEX t1_idx_00000062 ON t1(b);
+  SCAN TABLE t1 USING INDEX t1_idx_00000062
+}
+
+do_setup_rec_test $tn.5 {
+  CREATE TABLE t1(a, b, c);
+} {
+  SELECT a FROM t1 WHERE a=? ORDER BY b;
+} {
+  CREATE INDEX t1_idx_000123a7 ON t1(a, b);
+  SEARCH TABLE t1 USING COVERING INDEX t1_idx_000123a7 (a=?)
+}
+
+do_setup_rec_test $tn.6 {
+  CREATE TABLE t1(a, b, c);
+} {
+  SELECT min(a) FROM t1
+} {
+  CREATE INDEX t1_idx_00000061 ON t1(a);
+  SEARCH TABLE t1 USING COVERING INDEX t1_idx_00000061
+}
+
+do_setup_rec_test $tn.7 {
+  CREATE TABLE t1(a, b, c);
+} {
+  SELECT * FROM t1 ORDER BY a, b, c;
+} {
+  CREATE INDEX t1_idx_033e95fe ON t1(a, b, c);
+  SCAN TABLE t1 USING COVERING INDEX t1_idx_033e95fe
+}
+
+#do_setup_rec_test $tn.1.8 {
+#  CREATE TABLE t1(a, b, c);
+#} {
+#  SELECT * FROM t1 ORDER BY a ASC, b COLLATE nocase DESC, c ASC;
+#} {
+#  CREATE INDEX t1_idx_5be6e222 ON t1(a, b COLLATE NOCASE DESC, c);
+#  0|0|0|SCAN TABLE t1 USING COVERING INDEX t1_idx_5be6e222
+#}
+
+do_setup_rec_test $tn.8.1 {
+  CREATE TABLE t1(a COLLATE NOCase, b, c);
+} {
+  SELECT * FROM t1 WHERE a=?
+} {
+  CREATE INDEX t1_idx_00000061 ON t1(a);
+  SEARCH TABLE t1 USING INDEX t1_idx_00000061 (a=?)
+}
+do_setup_rec_test $tn.8.2 {
+  CREATE TABLE t1(a, b COLLATE nocase, c);
+} {
+  SELECT * FROM t1 ORDER BY a ASC, b DESC, c ASC;
+} {
+  CREATE INDEX t1_idx_5cb97285 ON t1(a, b DESC, c);
+  SCAN TABLE t1 USING COVERING INDEX t1_idx_5cb97285
+}
+
+
+# Tables with names that require quotes.
+#
+do_setup_rec_test $tn.9.1 {
+  CREATE TABLE "t t"(a, b, c);
+} {
+  SELECT * FROM "t t" WHERE a=?
+} {
+  CREATE INDEX 't t_idx_00000061' ON 't t'(a);
+  SEARCH TABLE t t USING INDEX t t_idx_00000061 (a=?) 
+}
+
+do_setup_rec_test $tn.9.2 {
+  CREATE TABLE "t t"(a, b, c);
+} {
+  SELECT * FROM "t t" WHERE b BETWEEN ? AND ?
+} {
+  CREATE INDEX 't t_idx_00000062' ON 't t'(b);
+  SEARCH TABLE t t USING INDEX t t_idx_00000062 (b>? AND b<?)
+}
+
+# Columns with names that require quotes.
+#
+do_setup_rec_test $tn.10.1 {
+  CREATE TABLE t3(a, "b b", c);
+} {
+  SELECT * FROM t3 WHERE "b b" = ?
+} {
+  CREATE INDEX t3_idx_00050c52 ON t3('b b');
+  SEARCH TABLE t3 USING INDEX t3_idx_00050c52 (b b=?)
+}
+
+do_setup_rec_test $tn.10.2 {
+  CREATE TABLE t3(a, "b b", c);
+} {
+  SELECT * FROM t3 ORDER BY "b b"
+} {
+  CREATE INDEX t3_idx_00050c52 ON t3('b b');
+  SCAN TABLE t3 USING INDEX t3_idx_00050c52
+}
+
+# Transitive constraints
+#
+do_setup_rec_test $tn.11.1 {
+  CREATE TABLE t5(a, b);
+  CREATE TABLE t6(c, d);
+} {
+  SELECT * FROM t5, t6 WHERE a=? AND b=c AND c=?
+} {
+  CREATE INDEX t5_idx_000123a7 ON t5(a, b);
+  CREATE INDEX t6_idx_00000063 ON t6(c);
+  SEARCH TABLE t6 USING INDEX t6_idx_00000063 (c=?) 
+  SEARCH TABLE t5 USING COVERING INDEX t5_idx_000123a7 (a=? AND b=?)
+}
+
+# OR terms.
+#
+do_setup_rec_test $tn.12.1 {
+  CREATE TABLE t7(a, b);
+} {
+  SELECT * FROM t7 WHERE a=? OR b=?
+} {
+  CREATE INDEX t7_idx_00000062 ON t7(b);
+  CREATE INDEX t7_idx_00000061 ON t7(a);
+  MULTI-INDEX OR
+    SEARCH TABLE t7 USING INDEX t7_idx_00000061 (a=?) 
+    SEARCH TABLE t7 USING INDEX t7_idx_00000062 (b=?)
+}
+
+# rowid terms.
+#
+do_setup_rec_test $tn.13.1 {
+  CREATE TABLE t8(a, b);
+} {
+  SELECT * FROM t8 WHERE rowid=?
+} {
+  (no new indexes)
+  SEARCH TABLE t8 USING INTEGER PRIMARY KEY (rowid=?)
+}
+do_setup_rec_test $tn.13.2 {
+  CREATE TABLE t8(a, b);
+} {
+  SELECT * FROM t8 ORDER BY rowid
+} {
+  (no new indexes)
+  SCAN TABLE t8
+}
+do_setup_rec_test $tn.13.3 {
+  CREATE TABLE t8(a, b);
+} {
+  SELECT * FROM t8 WHERE a=? ORDER BY rowid
+} {
+  CREATE INDEX t8_idx_00000061 ON t8(a); 
+  SEARCH TABLE t8 USING INDEX t8_idx_00000061 (a=?)
+}
+
+# Triggers
+#
+do_setup_rec_test $tn.14 {
+  CREATE TABLE t9(a, b, c);
+  CREATE TABLE t10(a, b, c);
+  CREATE TRIGGER t9t AFTER INSERT ON t9 BEGIN
+    UPDATE t10 SET a=new.a WHERE b = new.b;
+  END;
+} {
+  INSERT INTO t9 VALUES(?, ?, ?);
+} {
+  CREATE INDEX t10_idx_00000062 ON t10(b); 
+  SEARCH TABLE t10 USING INDEX t10_idx_00000062 (b=?)
+}
+
+do_setup_rec_test $tn.15 {
+  CREATE TABLE t1(a, b);
+  CREATE TABLE t2(c, d);
+
+  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
+  INSERT INTO t1 SELECT (i-1)/50, (i-1)/20 FROM s;
+
+  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
+  INSERT INTO t2 SELECT (i-1)/20, (i-1)/5 FROM s;
+} {
+  SELECT * FROM t2, t1 WHERE b=? AND d=? AND t2.rowid=t1.rowid
+} {
+  CREATE INDEX t2_idx_00000064 ON t2(d);
+  SEARCH TABLE t2 USING INDEX t2_idx_00000064 (d=?) 
+  SEARCH TABLE t1 USING INTEGER PRIMARY KEY (rowid=?)
+}
+
+do_setup_rec_test $tn.16 {
+  CREATE TABLE t1(a, b);
+} {
+  SELECT * FROM t1 WHERE b IS NOT NULL;
+} {
+  (no new indexes)
+  SCAN TABLE t1
+}
+
+}
+
+proc do_candidates_test {tn sql res} {
+  set res [squish [string trim $res]]
+
+  set expert [sqlite3_expert_new db]
+  $expert sql $sql
+  $expert analyze
+
+  set candidates [squish [string trim [$expert report 0 candidates]]]
+  $expert destroy
+
+  uplevel [list do_test $tn [list set {} $candidates] $res]
+}
+
+
+reset_db
+do_execsql_test 4.0 {
+  CREATE TABLE t1(a, b);
+  CREATE TABLE t2(c, d);
+
+  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
+  INSERT INTO t1 SELECT (i-1)/50, (i-1)/20 FROM s;
+
+  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
+  INSERT INTO t2 SELECT (i-1)/20, (i-1)/5 FROM s;
+}
+do_candidates_test 4.1 {
+  SELECT * FROM t1,t2 WHERE (b=? OR a=?) AND (c=? OR d=?)
+} {
+  CREATE INDEX t1_idx_00000062 ON t1(b); -- stat1: 100 20 
+  CREATE INDEX t1_idx_00000061 ON t1(a); -- stat1: 100 50 
+  CREATE INDEX t2_idx_00000063 ON t2(c); -- stat1: 100 20 
+  CREATE INDEX t2_idx_00000064 ON t2(d); -- stat1: 100 5
+}
+
+do_candidates_test 4.2 {
+  SELECT * FROM t1,t2 WHERE a=? AND b=? AND c=? AND d=?
+} {
+  CREATE INDEX t1_idx_000123a7 ON t1(a, b); -- stat1: 100 50 17
+  CREATE INDEX t2_idx_0001295b ON t2(c, d); -- stat1: 100 20 5
+}
+
+do_execsql_test 4.3 {
+  CREATE INDEX t1_idx_00000061 ON t1(a); -- stat1: 100 50 
+  CREATE INDEX t1_idx_00000062 ON t1(b); -- stat1: 100 20 
+  CREATE INDEX t1_idx_000123a7 ON t1(a, b); -- stat1: 100 50 16
+
+  CREATE INDEX t2_idx_00000063 ON t2(c); -- stat1: 100 20 
+  CREATE INDEX t2_idx_00000064 ON t2(d); -- stat1: 100 5
+  CREATE INDEX t2_idx_0001295b ON t2(c, d); -- stat1: 100 20 5
+
+  ANALYZE;
+  SELECT * FROM sqlite_stat1 ORDER BY 1, 2;
+} {
+  t1 t1_idx_00000061 {100 50} 
+  t1 t1_idx_00000062 {100 20}
+  t1 t1_idx_000123a7 {100 50 17}
+  t2 t2_idx_00000063 {100 20} 
+  t2 t2_idx_00000064 {100 5} 
+  t2 t2_idx_0001295b {100 20 5}
+}
+
+
+finish_test

ext/expert/sqlite3expert.h

@@ -0,0 +1,168 @@
+/*
+** 2017 April 07
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+*/
+
+
+#include "sqlite3.h"
+
+typedef struct sqlite3expert sqlite3expert;
+
+/*
+** Create a new sqlite3expert object.
+**
+** If successful, a pointer to the new object is returned and (*pzErr) set
+** to NULL. Or, if an error occurs, NULL is returned and (*pzErr) set to
+** an English-language error message. In this case it is the responsibility
+** of the caller to eventually free the error message buffer using
+** sqlite3_free().
+*/
+sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErr);
+
+/*
+** Configure an sqlite3expert object.
+**
+** EXPERT_CONFIG_SAMPLE:
+**   By default, sqlite3_expert_analyze() generates sqlite_stat1 data for
+**   each candidate index. This involves scanning and sorting the entire
+**   contents of each user database table once for each candidate index
+**   associated with the table. For large databases, this can be 
+**   prohibitively slow. This option allows the sqlite3expert object to
+**   be configured so that sqlite_stat1 data is instead generated based on a
+**   subset of each table, or so that no sqlite_stat1 data is used at all.
+**
+**   A single integer argument is passed to this option. If the value is less
+**   than or equal to zero, then no sqlite_stat1 data is generated or used by
+**   the analysis - indexes are recommended based on the database schema only.
+**   Or, if the value is 100 or greater, complete sqlite_stat1 data is
+**   generated for each candidate index (this is the default). Finally, if the
+**   value falls between 0 and 100, then it represents the percentage of user
+**   table rows that should be considered when generating sqlite_stat1 data.
+**
+**   Examples:
+**
+**     // Do not generate any sqlite_stat1 data
+**     sqlite3_expert_config(pExpert, EXPERT_CONFIG_SAMPLE, 0);
+**
+**     // Generate sqlite_stat1 data based on 10% of the rows in each table.
+**     sqlite3_expert_config(pExpert, EXPERT_CONFIG_SAMPLE, 10);
+*/
+int sqlite3_expert_config(sqlite3expert *p, int op, ...);
+
+#define EXPERT_CONFIG_SAMPLE 1    /* int */
+
+/*
+** Specify zero or more SQL statements to be included in the analysis.
+**
+** Buffer zSql must contain zero or more complete SQL statements. This
+** function parses all statements contained in the buffer and adds them
+** to the internal list of statements to analyze. If successful, SQLITE_OK
+** is returned and (*pzErr) set to NULL. Or, if an error occurs - for example
+** due to a error in the SQL - an SQLite error code is returned and (*pzErr)
+** may be set to point to an English language error message. In this case
+** the caller is responsible for eventually freeing the error message buffer
+** using sqlite3_free().
+**
+** If an error does occur while processing one of the statements in the
+** buffer passed as the second argument, none of the statements in the
+** buffer are added to the analysis.
+**
+** This function must be called before sqlite3_expert_analyze(). If a call
+** to this function is made on an sqlite3expert object that has already
+** been passed to sqlite3_expert_analyze() SQLITE_MISUSE is returned
+** immediately and no statements are added to the analysis.
+*/
+int sqlite3_expert_sql(
+  sqlite3expert *p,               /* From a successful sqlite3_expert_new() */
+  const char *zSql,               /* SQL statement(s) to add */
+  char **pzErr                    /* OUT: Error message (if any) */
+);
+
+
+/*
+** This function is called after the sqlite3expert object has been configured
+** with all SQL statements using sqlite3_expert_sql() to actually perform
+** the analysis. Once this function has been called, it is not possible to
+** add further SQL statements to the analysis.
+**
+** If successful, SQLITE_OK is returned and (*pzErr) is set to NULL. Or, if
+** an error occurs, an SQLite error code is returned and (*pzErr) set to 
+** point to a buffer containing an English language error message. In this
+** case it is the responsibility of the caller to eventually free the buffer
+** using sqlite3_free().
+**
+** If an error does occur within this function, the sqlite3expert object
+** is no longer useful for any purpose. At that point it is no longer
+** possible to add further SQL statements to the object or to re-attempt
+** the analysis. The sqlite3expert object must still be freed using a call
+** sqlite3_expert_destroy().
+*/
+int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr);
+
+/*
+** Return the total number of statements loaded using sqlite3_expert_sql().
+** The total number of SQL statements may be different from the total number
+** to calls to sqlite3_expert_sql().
+*/
+int sqlite3_expert_count(sqlite3expert*);
+
+/*
+** Return a component of the report.
+**
+** This function is called after sqlite3_expert_analyze() to extract the
+** results of the analysis. Each call to this function returns either a
+** NULL pointer or a pointer to a buffer containing a nul-terminated string.
+** The value passed as the third argument must be one of the EXPERT_REPORT_*
+** #define constants defined below.
+**
+** For some EXPERT_REPORT_* parameters, the buffer returned contains 
+** information relating to a specific SQL statement. In these cases that
+** SQL statement is identified by the value passed as the second argument.
+** SQL statements are numbered from 0 in the order in which they are parsed.
+** If an out-of-range value (less than zero or equal to or greater than the
+** value returned by sqlite3_expert_count()) is passed as the second argument
+** along with such an EXPERT_REPORT_* parameter, NULL is always returned.
+**
+** EXPERT_REPORT_SQL:
+**   Return the text of SQL statement iStmt.
+**
+** EXPERT_REPORT_INDEXES:
+**   Return a buffer containing the CREATE INDEX statements for all recommended
+**   indexes for statement iStmt. If there are no new recommeded indexes, NULL 
+**   is returned.
+**
+** EXPERT_REPORT_PLAN:
+**   Return a buffer containing the EXPLAIN QUERY PLAN output for SQL query
+**   iStmt after the proposed indexes have been added to the database schema.
+**
+** EXPERT_REPORT_CANDIDATES:
+**   Return a pointer to a buffer containing the CREATE INDEX statements 
+**   for all indexes that were tested (for all SQL statements). The iStmt
+**   parameter is ignored for EXPERT_REPORT_CANDIDATES calls.
+*/
+const char *sqlite3_expert_report(sqlite3expert*, int iStmt, int eReport);
+
+/*
+** Values for the third argument passed to sqlite3_expert_report().
+*/
+#define EXPERT_REPORT_SQL        1
+#define EXPERT_REPORT_INDEXES    2
+#define EXPERT_REPORT_PLAN       3
+#define EXPERT_REPORT_CANDIDATES 4
+
+/*
+** Free an (sqlite3expert*) handle and all associated resources. There 
+** should be one call to this function for each successful call to 
+** sqlite3-expert_new().
+*/
+void sqlite3_expert_destroy(sqlite3expert*);
+
+

ext/expert/test_expert.c

@@ -0,0 +1,220 @@
+/*
+** 2017 April 07
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+*/
+
+#if defined(SQLITE_TEST)
+
+#include "sqlite3expert.h"
+#include <assert.h>
+#include <string.h>
+
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#  ifndef SQLITE_TCLAPI
+#    define SQLITE_TCLAPI
+#  endif
+#endif
+
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+
+/*
+** Extract an sqlite3* db handle from the object passed as the second
+** argument. If successful, set *pDb to point to the db handle and return
+** TCL_OK. Otherwise, return TCL_ERROR.
+*/
+static int dbHandleFromObj(Tcl_Interp *interp, Tcl_Obj *pObj, sqlite3 **pDb){
+  Tcl_CmdInfo info;
+  if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(pObj), &info) ){
+    Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(pObj), 0);
+    return TCL_ERROR;
+  }
+
+  *pDb = *(sqlite3 **)info.objClientData;
+  return TCL_OK;
+}
+
+
+/*
+** Tclcmd:  $expert sql SQL
+**          $expert analyze
+**          $expert count
+**          $expert report STMT EREPORT
+**          $expert destroy
+*/
+static int SQLITE_TCLAPI testExpertCmd(
+  void *clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  sqlite3expert *pExpert = (sqlite3expert*)clientData;
+  struct Subcmd {
+    const char *zSub;
+    int nArg;
+    const char *zMsg;
+  } aSub[] = {
+    { "sql",       1, "TABLE",        }, /* 0 */
+    { "analyze",   0, "",             }, /* 1 */
+    { "count",     0, "",             }, /* 2 */
+    { "report",    2, "STMT EREPORT", }, /* 3 */
+    { "destroy",   0, "",             }, /* 4 */
+    { 0 }
+  };
+  int iSub;
+  int rc = TCL_OK;
+  char *zErr = 0;
+
+  if( objc<2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "SUBCOMMAND ...");
+    return TCL_ERROR;
+  }
+  rc = Tcl_GetIndexFromObjStruct(interp, 
+      objv[1], aSub, sizeof(aSub[0]), "sub-command", 0, &iSub
+  );
+  if( rc!=TCL_OK ) return rc;
+  if( objc!=2+aSub[iSub].nArg ){
+    Tcl_WrongNumArgs(interp, 2, objv, aSub[iSub].zMsg);
+    return TCL_ERROR;
+  }
+
+  switch( iSub ){
+    case 0: {      /* sql */
+      char *zArg = Tcl_GetString(objv[2]);
+      rc = sqlite3_expert_sql(pExpert, zArg, &zErr);
+      break;
+    }
+
+    case 1: {      /* analyze */
+      rc = sqlite3_expert_analyze(pExpert, &zErr);
+      break;
+    }
+
+    case 2: {      /* count */
+      int n = sqlite3_expert_count(pExpert);
+      Tcl_SetObjResult(interp, Tcl_NewIntObj(n));
+      break;
+    }
+
+    case 3: {      /* report */
+      const char *aEnum[] = {
+        "sql", "indexes", "plan", "candidates", 0
+      };
+      int iEnum;
+      int iStmt;
+      const char *zReport;
+
+      if( Tcl_GetIntFromObj(interp, objv[2], &iStmt) 
+       || Tcl_GetIndexFromObj(interp, objv[3], aEnum, "report", 0, &iEnum)
+      ){
+        return TCL_ERROR;
+      }
+
+      assert( EXPERT_REPORT_SQL==1 );
+      assert( EXPERT_REPORT_INDEXES==2 );
+      assert( EXPERT_REPORT_PLAN==3 );
+      assert( EXPERT_REPORT_CANDIDATES==4 );
+      zReport = sqlite3_expert_report(pExpert, iStmt, 1+iEnum);
+      Tcl_SetObjResult(interp, Tcl_NewStringObj(zReport, -1));
+      break;
+    }
+
+    default:       /* destroy */
+      assert( iSub==4 );     
+      Tcl_DeleteCommand(interp, Tcl_GetString(objv[0]));
+      break;
+  }
+
+  if( rc!=TCL_OK ){
+    if( zErr ){
+      Tcl_SetObjResult(interp, Tcl_NewStringObj(zErr, -1));
+    }else{
+      extern const char *sqlite3ErrName(int);
+      Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
+    }
+  }
+  sqlite3_free(zErr);
+  return rc;
+}
+
+static void SQLITE_TCLAPI testExpertDel(void *clientData){
+  sqlite3expert *pExpert = (sqlite3expert*)clientData;
+  sqlite3_expert_destroy(pExpert);
+}
+
+/*
+** sqlite3_expert_new DB
+*/
+static int SQLITE_TCLAPI test_sqlite3_expert_new(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  static int iCmd = 0;
+  sqlite3 *db;
+  char *zCmd = 0;
+  char *zErr = 0;
+  sqlite3expert *pExpert;
+  int rc = TCL_OK;
+
+  if( objc!=2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "DB");
+    return TCL_ERROR;
+  }
+  if( dbHandleFromObj(interp, objv[1], &db) ){
+    return TCL_ERROR;
+  }
+
+  zCmd = sqlite3_mprintf("sqlite3expert%d", ++iCmd);
+  if( zCmd==0 ){
+    Tcl_AppendResult(interp, "out of memory", (char*)0);
+    return TCL_ERROR;
+  }
+
+  pExpert = sqlite3_expert_new(db, &zErr);
+  if( pExpert==0 ){
+    Tcl_AppendResult(interp, zErr, (char*)0);
+    rc = TCL_ERROR;
+  }else{
+    void *p = (void*)pExpert;
+    Tcl_CreateObjCommand(interp, zCmd, testExpertCmd, p, testExpertDel);
+    Tcl_SetObjResult(interp, Tcl_NewStringObj(zCmd, -1));
+  }
+
+  sqlite3_free(zCmd);
+  sqlite3_free(zErr);
+  return rc;
+}
+
+#endif  /* ifndef SQLITE_OMIT_VIRTUALTABLE */
+
+int TestExpert_Init(Tcl_Interp *interp){
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+  struct Cmd {
+    const char *zCmd;
+    Tcl_ObjCmdProc *xProc;
+  } aCmd[] = {
+    { "sqlite3_expert_new", test_sqlite3_expert_new },
+  };
+  int i;
+
+  for(i=0; i<sizeof(aCmd)/sizeof(struct Cmd); i++){
+    struct Cmd *p = &aCmd[i];
+    Tcl_CreateObjCommand(interp, p->zCmd, p->xProc, 0, 0);
+  }
+#endif
+  return TCL_OK;
+}
+
+#endif

ext/fts3/fts3.c

@@ -3808,7 +3808,7 @@ static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
   int rc = SQLITE_OK;
   UNUSED_PARAMETER(iSavepoint);
   assert( ((Fts3Table *)pVtab)->inTransaction );
-  assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint );
+  assert( ((Fts3Table *)pVtab)->mxSavepoint <= iSavepoint );
   TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint );
   if( ((Fts3Table *)pVtab)->bIgnoreSavepoint==0 ){
     rc = fts3SyncMethod(pVtab);
@@ -3963,7 +3963,7 @@ int sqlite3Fts3Init(sqlite3 *db){
 
 #ifdef SQLITE_TEST
   if( rc==SQLITE_OK ){
-    rc = sqlite3Fts3ExprInitTestInterface(db);
+    rc = sqlite3Fts3ExprInitTestInterface(db, pHash);
   }
 #endif
 

ext/fts3/fts3Int.h

@@ -584,7 +584,7 @@ int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int,
 );
 void sqlite3Fts3ExprFree(Fts3Expr *);
 #ifdef SQLITE_TEST
-int sqlite3Fts3ExprInitTestInterface(sqlite3 *db);
+int sqlite3Fts3ExprInitTestInterface(sqlite3 *db, Fts3Hash*);
 int sqlite3Fts3InitTerm(sqlite3 *db);
 #endif
 

ext/fts3/fts3_expr.c

@@ -1108,34 +1108,6 @@ void sqlite3Fts3ExprFree(Fts3Expr *pDel){
 
 #include <stdio.h>
 
-/*
-** Function to query the hash-table of tokenizers (see README.tokenizers).
-*/
-static int queryTestTokenizer(
-  sqlite3 *db, 
-  const char *zName,  
-  const sqlite3_tokenizer_module **pp
-){
-  int rc;
-  sqlite3_stmt *pStmt;
-  const char zSql[] = "SELECT fts3_tokenizer(?)";
-
-  *pp = 0;
-  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
-  if( rc!=SQLITE_OK ){
-    return rc;
-  }
-
-  sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
-  if( SQLITE_ROW==sqlite3_step(pStmt) ){
-    if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
-      memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
-    }
-  }
-
-  return sqlite3_finalize(pStmt);
-}
-
 /*
 ** Return a pointer to a buffer containing a text representation of the
 ** expression passed as the first argument. The buffer is obtained from
@@ -1203,12 +1175,12 @@ static char *exprToString(Fts3Expr *pExpr, char *zBuf){
 **
 **   SELECT fts3_exprtest('simple', 'Bill col2:Bloggs', 'col1', 'col2');
 */
-static void fts3ExprTest(
+static void fts3ExprTestCommon(
+  int bRebalance,
   sqlite3_context *context,
   int argc,
   sqlite3_value **argv
 ){
-  sqlite3_tokenizer_module const *pModule = 0;
   sqlite3_tokenizer *pTokenizer = 0;
   int rc;
   char **azCol = 0;
@@ -1218,7 +1190,9 @@ static void fts3ExprTest(
   int ii;
   Fts3Expr *pExpr;
   char *zBuf = 0;
-  sqlite3 *db = sqlite3_context_db_handle(context);
+  Fts3Hash *pHash = (Fts3Hash*)sqlite3_user_data(context);
+  const char *zTokenizer = 0;
+  char *zErr = 0;
 
   if( argc<3 ){
     sqlite3_result_error(context, 
@@ -1227,24 +1201,18 @@ static void fts3ExprTest(
     return;
   }
 
-  rc = queryTestTokenizer(db,
+  zTokenizer = (const char*)sqlite3_value_text(argv[0]);
-                          (const char *)sqlite3_value_text(argv[0]), &pModule);
+  rc = sqlite3Fts3InitTokenizer(pHash, zTokenizer, &pTokenizer, &zErr);
-  if( rc==SQLITE_NOMEM ){
+  if( rc!=SQLITE_OK ){
-    sqlite3_result_error_nomem(context);
+    if( rc==SQLITE_NOMEM ){
-    goto exprtest_out;
+      sqlite3_result_error_nomem(context);
-  }else if( !pModule ){
+    }else{
-    sqlite3_result_error(context, "No such tokenizer module", -1);
+      sqlite3_result_error(context, zErr, -1);
-    goto exprtest_out;
+    }
+    sqlite3_free(zErr);
+    return;
   }
 
-  rc = pModule->xCreate(0, 0, &pTokenizer);
-  assert( rc==SQLITE_NOMEM || rc==SQLITE_OK );
-  if( rc==SQLITE_NOMEM ){
-    sqlite3_result_error_nomem(context);
-    goto exprtest_out;
-  }
-  pTokenizer->pModule = pModule;
-
   zExpr = (const char *)sqlite3_value_text(argv[1]);
   nExpr = sqlite3_value_bytes(argv[1]);
   nCol = argc-2;
@@ -1257,7 +1225,7 @@ static void fts3ExprTest(
     azCol[ii] = (char *)sqlite3_value_text(argv[ii+2]);
   }
 
-  if( sqlite3_user_data(context) ){
+  if( bRebalance ){
     char *zDummy = 0;
     rc = sqlite3Fts3ExprParse(
         pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr, &zDummy
@@ -1283,23 +1251,38 @@ static void fts3ExprTest(
   sqlite3Fts3ExprFree(pExpr);
 
 exprtest_out:
-  if( pModule && pTokenizer ){
+  if( pTokenizer ){
-    rc = pModule->xDestroy(pTokenizer);
+    rc = pTokenizer->pModule->xDestroy(pTokenizer);
   }
   sqlite3_free(azCol);
 }
 
+static void fts3ExprTest(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  fts3ExprTestCommon(0, context, argc, argv);
+}
+static void fts3ExprTestRebalance(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  fts3ExprTestCommon(1, context, argc, argv);
+}
+
 /*
 ** Register the query expression parser test function fts3_exprtest() 
 ** with database connection db. 
 */
-int sqlite3Fts3ExprInitTestInterface(sqlite3* db){
+int sqlite3Fts3ExprInitTestInterface(sqlite3 *db, Fts3Hash *pHash){
   int rc = sqlite3_create_function(
-      db, "fts3_exprtest", -1, SQLITE_UTF8, 0, fts3ExprTest, 0, 0
+      db, "fts3_exprtest", -1, SQLITE_UTF8, (void*)pHash, fts3ExprTest, 0, 0
   );
   if( rc==SQLITE_OK ){
     rc = sqlite3_create_function(db, "fts3_exprtest_rebalance", 
-        -1, SQLITE_UTF8, (void *)1, fts3ExprTest, 0, 0
+        -1, SQLITE_UTF8, (void*)pHash, fts3ExprTestRebalance, 0, 0
     );
   }
   return rc;

ext/fts3/fts3_write.c

@@ -1908,6 +1908,7 @@ static int fts3WriteSegment(
     sqlite3_bind_blob(pStmt, 2, z, n, SQLITE_STATIC);
     sqlite3_step(pStmt);
     rc = sqlite3_reset(pStmt);
+    sqlite3_bind_null(pStmt, 2);
   }
   return rc;
 }
@@ -1964,6 +1965,7 @@ static int fts3WriteSegdir(
     sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC);
     sqlite3_step(pStmt);
     rc = sqlite3_reset(pStmt);
+    sqlite3_bind_null(pStmt, 6);
   }
   return rc;
 }
@@ -3443,6 +3445,7 @@ static void fts3UpdateDocTotals(
   sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, SQLITE_STATIC);
   sqlite3_step(pStmt);
   *pRC = sqlite3_reset(pStmt);
+  sqlite3_bind_null(pStmt, 2);
   sqlite3_free(a);
 }
 
@@ -4631,6 +4634,7 @@ static int fts3TruncateSegment(
       sqlite3_bind_int(pChomp, 4, iIdx);
       sqlite3_step(pChomp);
       rc = sqlite3_reset(pChomp);
+      sqlite3_bind_null(pChomp, 2);
     }
   }
 
@@ -4710,6 +4714,7 @@ static int fts3IncrmergeHintStore(Fts3Table *p, Blob *pHint){
     sqlite3_bind_blob(pReplace, 2, pHint->a, pHint->n, SQLITE_STATIC);
     sqlite3_step(pReplace);
     rc = sqlite3_reset(pReplace);
+    sqlite3_bind_null(pReplace, 2);
   }
 
   return rc;
@@ -5524,7 +5529,6 @@ int sqlite3Fts3UpdateMethod(
 ){
   Fts3Table *p = (Fts3Table *)pVtab;
   int rc = SQLITE_OK;             /* Return Code */
-  int isRemove = 0;               /* True for an UPDATE or DELETE */
   u32 *aSzIns = 0;                /* Sizes of inserted documents */
   u32 *aSzDel = 0;                /* Sizes of deleted documents */
   int nChng = 0;                  /* Net change in number of documents */
@@ -5622,7 +5626,6 @@ int sqlite3Fts3UpdateMethod(
   if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
     assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER );
     rc = fts3DeleteByRowid(p, apVal[0], &nChng, aSzDel);
-    isRemove = 1;
   }
   
   /* If this is an INSERT or UPDATE operation, insert the new record. */
@@ -5634,7 +5637,7 @@ int sqlite3Fts3UpdateMethod(
         rc = FTS_CORRUPT_VTAB;
       }
     }
-    if( rc==SQLITE_OK && (!isRemove || *pRowid!=p->iPrevDocid ) ){
+    if( rc==SQLITE_OK ){
       rc = fts3PendingTermsDocid(p, 0, iLangid, *pRowid);
     }
     if( rc==SQLITE_OK ){

ext/fts3/unicode/mkunicode.tcl

@@ -529,6 +529,263 @@ proc print_fold {zFunc} {
   puts "\}"
 }
 
+proc code {txt} {
+  set txt [string trimright $txt]
+  set txt [string trimleft $txt "\n"]
+  set n [expr {[string length $txt] - [string length [string trim $txt]]}]
+  set ret ""
+  foreach L [split $txt "\n"] {
+    append ret "[string range $L $n end]\n"
+  }
+  return [uplevel "subst -nocommands {$ret}"]
+}
+
+proc intarray {lInt} {
+  set ret ""
+  set n [llength $lInt]
+  for {set i 0} {$i < $n} {incr i 10} {
+    append ret "\n    "
+    foreach int [lrange $lInt $i [expr $i+9]] {
+      append ret [format "%-7s" "$int, "]
+    }
+  }
+  append ret "\n  "
+  set ret
+}
+
+proc categories_switch {Cvar first lSecond} {
+  upvar $Cvar C
+  set ret ""
+  append ret "case '$first':\n"
+  append ret "          switch( zCat\[1\] ){\n"
+  foreach s $lSecond {
+    append ret "            case '$s': aArray\[$C($first$s)\] = 1; break;\n"
+  }
+  append ret "            case '*': \n"
+  foreach s $lSecond {
+    append ret "              aArray\[$C($first$s)\] = 1;\n"
+  }
+  append ret "              break;\n"
+  append ret "            default: return 1;"
+  append ret "          }\n"
+  append ret "          break;\n"
+}
+
+# Argument is a list. Each element of which is itself a list of two elements:
+#
+#   * the codepoint
+#   * the category
+#
+# List elements are sorted in order of codepoint.
+#
+proc print_categories {lMap} {
+  set categories {
+    Cc Cf Cn Cs
+    Ll Lm Lo Lt Lu
+    Mc Me Mn
+    Nd Nl No
+    Pc Pd Pe Pf Pi Po Ps
+    Sc Sk Sm So
+    Zl Zp Zs
+
+    LC Co
+  }
+
+  for {set i 0} {$i < [llength $categories]} {incr i} {
+    set C([lindex $categories $i]) [expr 1+$i]
+  }
+
+  set caseC [categories_switch C C {c f n s o}]
+  set caseL [categories_switch C L {l m o t u C}]
+  set caseM [categories_switch C M {c e n}]
+  set caseN [categories_switch C N {d l o}]
+  set caseP [categories_switch C P {c d e f i o s}]
+  set caseS [categories_switch C S {c k m o}]
+  set caseZ [categories_switch C Z {l p s}]
+
+  set nCat [expr [llength [array names C]] + 1]
+  puts [code {
+    int sqlite3Fts5UnicodeNCat(void) { 
+      return $nCat;
+    }
+
+    int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ 
+      aArray[0] = 1;
+      switch( zCat[0] ){
+        $caseC
+        $caseL
+        $caseM
+        $caseN
+        $caseP
+        $caseS
+        $caseZ
+      }
+      return 0;
+    }
+  }]
+
+  set nRepeat 0
+  set first   [lindex $lMap 0 0]
+  set class   [lindex $lMap 0 1]
+  set prev -1
+
+  set CASE(0) "Lu"
+  set CASE(1) "Ll"
+
+  foreach m $lMap {
+    foreach {codepoint cl} $m {}
+    set codepoint [expr "0x$codepoint"]
+    if {$codepoint>=(1<<20)} continue
+
+    set bNew 0
+    if {$codepoint!=($prev+1)} {
+      set bNew 1
+    } elseif {
+      $cl==$class || ($class=="LC" && $cl==$CASE([expr $nRepeat & 0x01]))
+    } {
+      incr nRepeat
+    } elseif {$class=="Lu" && $nRepeat==1 && $cl=="Ll"} {
+      set class LC
+      incr nRepeat
+    } else {
+      set bNew 1
+    }
+    if {$bNew} {
+      lappend lEntries [list $first $class $nRepeat]
+      set nRepeat 1
+      set first $codepoint
+      set class $cl
+    }
+    set prev $codepoint
+  }
+  if {$nRepeat>0} {
+    lappend lEntries [list $first $class $nRepeat]
+  }
+
+  set aBlock [list 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
+  set aMap [list]
+  foreach e $lEntries {
+    foreach {cp class nRepeat} $e {}
+    set block [expr ($cp>>16)]
+    if {$block>0 && [lindex $aBlock $block]==0} {
+      for {set i 1} {$i<=$block} {incr i} {
+        if {[lindex $aBlock $i]==0} {
+          lset aBlock $i [llength $aMap]
+        }
+      }
+    }
+    lappend aMap [expr {$cp & 0xFFFF}]
+    lappend aData [expr {($nRepeat << 5) + $C($class)}]
+  }
+  for {set i 1} {$i<[llength $aBlock]} {incr i} {
+    if {[lindex $aBlock $i]==0} {
+      lset aBlock $i [llength $aMap]
+    }
+  }
+
+  set aBlockArray [intarray $aBlock]
+  set aMapArray [intarray $aMap]
+  set aDataArray [intarray $aData]
+  puts [code {
+    static u16 aFts5UnicodeBlock[] = {$aBlockArray};
+    static u16 aFts5UnicodeMap[] = {$aMapArray};
+    static u16 aFts5UnicodeData[] = {$aDataArray};
+
+    int sqlite3Fts5UnicodeCategory(int iCode) { 
+      int iRes = -1;
+      int iHi;
+      int iLo;
+      int ret;
+      u16 iKey;
+
+      if( iCode>=(1<<20) ){
+        return 0;
+      }
+      iLo = aFts5UnicodeBlock[(iCode>>16)];
+      iHi = aFts5UnicodeBlock[1+(iCode>>16)];
+      iKey = (iCode & 0xFFFF);
+      while( iHi>iLo ){
+        int iTest = (iHi + iLo) / 2;
+        assert( iTest>=iLo && iTest<iHi );
+        if( iKey>=aFts5UnicodeMap[iTest] ){
+          iRes = iTest;
+          iLo = iTest+1;
+        }else{
+          iHi = iTest;
+        }
+      }
+
+      if( iRes<0 ) return 0;
+      if( iKey>=(aFts5UnicodeMap[iRes]+(aFts5UnicodeData[iRes]>>5)) ) return 0;
+      ret = aFts5UnicodeData[iRes] & 0x1F;
+      if( ret!=$C(LC) ) return ret;
+      return ((iKey - aFts5UnicodeMap[iRes]) & 0x01) ? $C(Ll) : $C(Lu);
+    }
+
+    void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){
+      int i = 0;
+      int iTbl = 0;
+      while( i<128 ){
+        int bToken = aArray[ aFts5UnicodeData[iTbl] & 0x1F ];
+        int n = (aFts5UnicodeData[iTbl] >> 5) + i;
+        for(; i<128 && i<n; i++){
+          aAscii[i] = bToken;
+        }
+        iTbl++;
+      }
+    }
+  }]
+}
+
+proc print_test_categories {lMap} {
+
+  set lCP [list]
+  foreach e $lMap {
+    foreach {cp cat} $e {}
+    if {[expr 0x$cp] < (1<<20)} {
+      lappend lCP "{0x$cp, \"$cat\"}, "
+    }
+  }
+
+  set aCP "\n"
+  for {set i 0} {$i < [llength $lCP]} {incr i 4} {
+    append aCP "    [join [lrange $lCP $i $i+3]]\n"
+  }
+
+
+  puts [code {
+    static int categories_test (int *piCode){
+      struct Codepoint {
+        int iCode;
+        const char *zCat;
+      } aCP[] = {$aCP};
+      int i;
+      int iCP = 0;
+
+      for(i=0; i<1000000; i++){
+        u8 aArray[40];
+        int cat = 0;
+        int c = 0;
+        memset(aArray, 0, sizeof(aArray));
+        if( aCP[iCP].iCode==i ){
+          sqlite3Fts5UnicodeCatParse(aCP[iCP].zCat, aArray);
+          iCP++;
+        }else{
+          aArray[0] = 1;
+        }
+
+        c = sqlite3Fts5UnicodeCategory(i);
+        if( aArray[c]==0 ){
+          *piCode = i;
+          return 1;
+        }
+      }
+
+      return 0;
+    }
+  }]
+}
+
 proc print_fold_test {zFunc mappings} {
   global tl_lookup_table
 
@@ -605,15 +862,21 @@ proc print_test_main {} {
   puts "#include <stdio.h>"
   puts ""
   puts "int main(int argc, char **argv)\{"
-  puts "  int r1, r2;"
+  puts "  int r1, r2, r3;"
   puts "  int code;"
+  puts "  r3 = 0;"
   puts "  r1 = isalnum_test(&code);"
   puts "  if( r1 ) printf(\"isalnum(): Problem with code %d\\n\",code);"
   puts "  else printf(\"isalnum(): test passed\\n\");"
   puts "  r2 = fold_test(&code);"
   puts "  if( r2 ) printf(\"fold(): Problem with code %d\\n\",code);"
   puts "  else printf(\"fold(): test passed\\n\");"
-  puts "  return (r1 || r2);"
+  if {$::generate_fts5_code} {
+    puts "  r3 = categories_test(&code);"
+    puts "  if( r3 ) printf(\"categories(): Problem with code %d\\n\",code);"
+    puts "  else printf(\"categories(): test passed\\n\");"
+  }
+  puts "  return (r1 || r2 || r3);"
   puts "\}"
 }
 
@@ -651,10 +914,18 @@ for {set i 0} {$i < [llength $argv]-2} {incr i} {
 
 print_fileheader
 
+if {$::generate_test_code} {
+  puts "typedef unsigned short int u16;"
+  puts "typedef unsigned char u8;"
+  puts "#include <string.h>"
+}
+
 # Print the isalnum() function to stdout.
 #
 set lRange [an_load_separator_ranges]
-print_isalnum ${function_prefix}UnicodeIsalnum $lRange
+if {$generate_fts5_code==0} {
+  print_isalnum ${function_prefix}UnicodeIsalnum $lRange
+}
 
 # Leave a gap between the two generated C functions.
 #
@@ -677,12 +948,21 @@ puts ""
 #
 print_fold ${function_prefix}UnicodeFold
 
+if {$generate_fts5_code} {
+  puts ""
+  puts ""
+  print_categories [cc_load_unicodedata_text ${unicodedata.txt}]
+}
+
 # Print the test routines and main() function to stdout, if -test 
 # was specified.
 #
 if {$::generate_test_code} {
-  print_test_isalnum ${function_prefix}UnicodeIsalnum $lRange
+  if {$generate_fts5_code==0} {
+    print_test_isalnum ${function_prefix}UnicodeIsalnum $lRange
+  }
   print_fold_test ${function_prefix}UnicodeFold $mappings
+  print_test_categories [cc_load_unicodedata_text ${unicodedata.txt}]
   print_test_main 
 }
 

ext/fts3/unicode/parseunicode.tcl

@@ -143,4 +143,40 @@ proc tl_load_casefolding_txt {zName} {
   }
 }
 
+proc cc_load_unicodedata_text {zName} {
+  set fd [open $zName]
+  set lField {
+    code
+    character_name
+    general_category
+    canonical_combining_classes
+    bidirectional_category
+    character_decomposition_mapping
+    decimal_digit_value
+    digit_value
+    numeric_value
+    mirrored
+    unicode_1_name
+    iso10646_comment_field
+    uppercase_mapping
+    lowercase_mapping
+    titlecase_mapping
+  }
+  set lRet [list]
+
+  while { ![eof $fd] } {
+    set line [gets $fd]
+    if {$line == ""} continue
+
+    set fields [split $line ";"]
+    if {[llength $fields] != [llength $lField]} { error "parse error: $line" }
+    foreach $lField $fields {}
+
+    lappend lRet [list $code $general_category]
+  }
+
+  close $fd
+  set lRet
+}
+
 

ext/fts5/fts5.h

@@ -444,7 +444,7 @@ struct Fts5ExtensionApi {
 **            This way, even if the tokenizer does not provide synonyms
 **            when tokenizing query text (it should not - to do would be
 **            inefficient), it doesn't matter if the user queries for 
-**            'first + place' or '1st + place', as there are entires in the
+**            'first + place' or '1st + place', as there are entries in the
 **            FTS index corresponding to both forms of the first token.
 **   </ol>
 **
@@ -472,7 +472,7 @@ struct Fts5ExtensionApi {
 **   extra data to the FTS index or require FTS5 to query for multiple terms,
 **   so it is efficient in terms of disk space and query speed. However, it
 **   does not support prefix queries very well. If, as suggested above, the
-**   token "first" is subsituted for "1st" by the tokenizer, then the query:
+**   token "first" is substituted for "1st" by the tokenizer, then the query:
 **
 **   <codeblock>
 **     ... MATCH '1s*'</codeblock>

ext/fts5/fts5Int.h

@@ -722,6 +722,8 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm(
   int bPrefix
 );
 
+void sqlite3Fts5ParseSetCaret(Fts5ExprPhrase*);
+
 Fts5ExprNearset *sqlite3Fts5ParseNearset(
   Fts5Parse*, 
   Fts5ExprNearset*,
@@ -782,9 +784,12 @@ int sqlite3Fts5VocabInit(Fts5Global*, sqlite3*);
 /**************************************************************************
 ** Interface to automatically generated code in fts5_unicode2.c. 
 */
-int sqlite3Fts5UnicodeIsalnum(int c);
 int sqlite3Fts5UnicodeIsdiacritic(int c);
 int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic);
+
+int sqlite3Fts5UnicodeCatParse(const char*, u8*);
+int sqlite3Fts5UnicodeCategory(int iCode);
+void sqlite3Fts5UnicodeAscii(u8*, u8*);
 /*
 ** End of interface to code in fts5_unicode2.c.
 **************************************************************************/

ext/fts5/fts5_aux.c

@@ -358,6 +358,16 @@ static int fts5SnippetScore(
   return rc;
 }
 
+/*
+** Return the value in pVal interpreted as utf-8 text. Except, if pVal 
+** contains a NULL value, return a pointer to a static string zero
+** bytes in length instead of a NULL pointer.
+*/
+static const char *fts5ValueToText(sqlite3_value *pVal){
+  const char *zRet = (const char*)sqlite3_value_text(pVal);
+  return zRet ? zRet : "";
+}
+
 /*
 ** Implementation of snippet() function.
 */
@@ -393,9 +403,9 @@ static void fts5SnippetFunction(
   nCol = pApi->xColumnCount(pFts);
   memset(&ctx, 0, sizeof(HighlightContext));
   iCol = sqlite3_value_int(apVal[0]);
-  ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]);
+  ctx.zOpen = fts5ValueToText(apVal[1]);
-  ctx.zClose = (const char*)sqlite3_value_text(apVal[2]);
+  ctx.zClose = fts5ValueToText(apVal[2]);
-  zEllips = (const char*)sqlite3_value_text(apVal[3]);
+  zEllips = fts5ValueToText(apVal[3]);
   nToken = sqlite3_value_int(apVal[4]);
 
   iBestCol = (iCol>=0 ? iCol : 0);

ext/fts5/fts5_expr.c

@@ -36,6 +36,7 @@ void sqlite3Fts5Parser(void*, int, Fts5Token, Fts5Parse*);
 #include <stdio.h>
 void sqlite3Fts5ParserTrace(FILE*, char*);
 #endif
+int sqlite3Fts5ParserFallback(int);
 
 
 struct Fts5Expr {
@@ -87,7 +88,8 @@ struct Fts5ExprNode {
 ** or term prefix.
 */
 struct Fts5ExprTerm {
-  int bPrefix;                    /* True for a prefix term */
+  u8 bPrefix;                     /* True for a prefix term */
+  u8 bFirst;                      /* True if token must be first in column */
   char *zTerm;                    /* nul-terminated term */
   Fts5IndexIter *pIter;           /* Iterator for this term */
   Fts5ExprTerm *pSynonym;         /* Pointer to first in list of synonyms */
@@ -168,6 +170,7 @@ static int fts5ExprGetToken(
     case '+':  tok = FTS5_PLUS;  break;
     case '*':  tok = FTS5_STAR;  break;
     case '-':  tok = FTS5_MINUS; break;
+    case '^':  tok = FTS5_CARET; break;
     case '\0': tok = FTS5_EOF;   break;
 
     case '"': {
@@ -427,6 +430,7 @@ static int fts5ExprPhraseIsMatch(
   Fts5PoslistReader *aIter = aStatic;
   int i;
   int rc = SQLITE_OK;
+  int bFirst = pPhrase->aTerm[0].bFirst;
   
   fts5BufferZero(&pPhrase->poslist);
 
@@ -481,8 +485,10 @@ static int fts5ExprPhraseIsMatch(
     }while( bMatch==0 );
 
     /* Append position iPos to the output */
-    rc = sqlite3Fts5PoslistWriterAppend(&pPhrase->poslist, &writer, iPos);
+    if( bFirst==0 || FTS5_POS2OFFSET(iPos)==0 ){
-    if( rc!=SQLITE_OK ) goto ismatch_out;
+      rc = sqlite3Fts5PoslistWriterAppend(&pPhrase->poslist, &writer, iPos);
+      if( rc!=SQLITE_OK ) goto ismatch_out;
+    }
 
     for(i=0; i<pPhrase->nTerm; i++){
       if( sqlite3Fts5PoslistReaderNext(&aIter[i]) ) goto ismatch_out;
@@ -736,7 +742,9 @@ static int fts5ExprNearTest(
     ** phrase is not a match, break out of the loop early.  */
     for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
       Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
-      if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){
+      if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym 
+       || pNear->pColset || pPhrase->aTerm[0].bFirst
+      ){
         int bMatch = 0;
         rc = fts5ExprPhraseIsMatch(pNode, pPhrase, &bMatch);
         if( bMatch==0 ) break;
@@ -917,6 +925,7 @@ static int fts5ExprNodeTest_STRING(
   assert( pNear->nPhrase>1 
        || pNear->apPhrase[0]->nTerm>1 
        || pNear->apPhrase[0]->aTerm[0].pSynonym
+       || pNear->apPhrase[0]->aTerm[0].bFirst
   );
 
   /* Initialize iLast, the "lastest" rowid any iterator points to. If the
@@ -1441,6 +1450,16 @@ static void fts5ExprPhraseFree(Fts5ExprPhrase *pPhrase){
   }
 }
 
+/*
+** Set the "bFirst" flag on the first token of the phrase passed as the
+** only argument.
+*/
+void sqlite3Fts5ParseSetCaret(Fts5ExprPhrase *pPhrase){
+  if( pPhrase && pPhrase->nTerm ){
+    pPhrase->aTerm[0].bFirst = 1;
+  }
+}
+
 /*
 ** If argument pNear is NULL, then a new Fts5ExprNearset object is allocated
 ** and populated with pPhrase. Or, if pNear is not NULL, phrase pPhrase is
@@ -1658,7 +1677,7 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm(
       ** no token characters at all. (e.g ... MATCH '""'). */
       sCtx.pPhrase = sqlite3Fts5MallocZero(&pParse->rc, sizeof(Fts5ExprPhrase));
     }else if( sCtx.pPhrase->nTerm ){
-      sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = bPrefix;
+      sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = (u8)bPrefix;
     }
     pParse->apPhrase[pParse->nPhrase-1] = sCtx.pPhrase;
   }
@@ -1719,6 +1738,7 @@ int sqlite3Fts5ExprClonePhrase(
       }
       if( rc==SQLITE_OK ){
         sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix;
+        sCtx.pPhrase->aTerm[i].bFirst = pOrig->aTerm[i].bFirst;
       }
     }
   }else{
@@ -1737,7 +1757,10 @@ int sqlite3Fts5ExprClonePhrase(
     pNew->pRoot->pNear->nPhrase = 1;
     sCtx.pPhrase->pNode = pNew->pRoot;
 
-    if( pOrig->nTerm==1 && pOrig->aTerm[0].pSynonym==0 ){
+    if( pOrig->nTerm==1 
+     && pOrig->aTerm[0].pSynonym==0 
+     && pOrig->aTerm[0].bFirst==0 
+    ){
       pNew->pRoot->eType = FTS5_TERM;
       pNew->pRoot->xNext = fts5ExprNodeNext_TERM;
     }else{
@@ -2011,6 +2034,7 @@ static void fts5ExprAssignXNext(Fts5ExprNode *pNode){
       Fts5ExprNearset *pNear = pNode->pNear;
       if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 
        && pNear->apPhrase[0]->aTerm[0].pSynonym==0
+       && pNear->apPhrase[0]->aTerm[0].bFirst==0
       ){
         pNode->eType = FTS5_TERM;
         pNode->xNext = fts5ExprNodeNext_TERM;
@@ -2097,20 +2121,23 @@ Fts5ExprNode *sqlite3Fts5ParseNode(
           }
         }
 
-        if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL 
+        if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL ){
-         && (pNear->nPhrase!=1 || pNear->apPhrase[0]->nTerm>1)
+          Fts5ExprPhrase *pPhrase = pNear->apPhrase[0];
-        ){
+          if( pNear->nPhrase!=1 
-          assert( pParse->rc==SQLITE_OK );
+           || pPhrase->nTerm>1
-          pParse->rc = SQLITE_ERROR;
+           || (pPhrase->nTerm>0 && pPhrase->aTerm[0].bFirst)
-          assert( pParse->zErr==0 );
+          ){
-          pParse->zErr = sqlite3_mprintf(
+            assert( pParse->rc==SQLITE_OK );
-              "fts5: %s queries are not supported (detail!=full)", 
+            pParse->rc = SQLITE_ERROR;
-              pNear->nPhrase==1 ? "phrase": "NEAR"
+            assert( pParse->zErr==0 );
-          );
+            pParse->zErr = sqlite3_mprintf(
-          sqlite3_free(pRet);
+                "fts5: %s queries are not supported (detail!=full)", 
-          pRet = 0;
+                pNear->nPhrase==1 ? "phrase": "NEAR"
+                );
+            sqlite3_free(pRet);
+            pRet = 0;
+          }
         }
-
       }else{
         fts5ExprAddChildren(pRet, pLeft);
         fts5ExprAddChildren(pRet, pRight);
@@ -2514,14 +2541,19 @@ static void fts5ExprIsAlnum(
   sqlite3_value **apVal           /* Function arguments */
 ){
   int iCode;
+  u8 aArr[32];
   if( nArg!=1 ){
     sqlite3_result_error(pCtx, 
         "wrong number of arguments to function fts5_isalnum", -1
     );
     return;
   }
+  memset(aArr, 0, sizeof(aArr));
+  sqlite3Fts5UnicodeCatParse("L*", aArr);
+  sqlite3Fts5UnicodeCatParse("N*", aArr);
+  sqlite3Fts5UnicodeCatParse("Co", aArr);
   iCode = sqlite3_value_int(apVal[0]);
-  sqlite3_result_int(pCtx, sqlite3Fts5UnicodeIsalnum(iCode));
+  sqlite3_result_int(pCtx, aArr[sqlite3Fts5UnicodeCategory(iCode)]);
 }
 
 static void fts5ExprFold(
@@ -2565,10 +2597,12 @@ int sqlite3Fts5ExprInit(Fts5Global *pGlobal, sqlite3 *db){
     rc = sqlite3_create_function(db, p->z, -1, SQLITE_UTF8, pCtx, p->x, 0, 0);
   }
 
-  /* Avoid a warning indicating that sqlite3Fts5ParserTrace() is unused */
+  /* Avoid warnings indicating that sqlite3Fts5ParserTrace() and
+  ** sqlite3Fts5ParserFallback() are unused */
 #ifndef NDEBUG
   (void)sqlite3Fts5ParserTrace;
 #endif
+  (void)sqlite3Fts5ParserFallback;
 
   return rc;
 }

ext/fts5/fts5_index.c

@@ -758,6 +758,7 @@ static void fts5DataWrite(Fts5Index *p, i64 iRowid, const u8 *pData, int nData){
   sqlite3_bind_blob(p->pWriter, 2, pData, nData, SQLITE_STATIC);
   sqlite3_step(p->pWriter);
   p->rc = sqlite3_reset(p->pWriter);
+  sqlite3_bind_null(p->pWriter, 2);
 }
 
 /*
@@ -2386,6 +2387,7 @@ static void fts5SegIterSeekInit(
     bDlidx = (val & 0x0001);
   }
   p->rc = sqlite3_reset(pIdxSelect);
+  sqlite3_bind_null(pIdxSelect, 2);
 
   if( iPg<pSeg->pgnoFirst ){
     iPg = pSeg->pgnoFirst;
@@ -3598,6 +3600,7 @@ static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){
           sqlite3_bind_blob(pIdxSelect, 2, aBlob, 2, SQLITE_STATIC);
           assert( sqlite3_step(pIdxSelect)!=SQLITE_ROW );
           p->rc = sqlite3_reset(pIdxSelect);
+          sqlite3_bind_null(pIdxSelect, 2);
         }
       }
 #endif
@@ -3724,6 +3727,7 @@ static void fts5WriteFlushBtree(Fts5Index *p, Fts5SegWriter *pWriter){
     sqlite3_bind_int64(p->pIdxWriter, 3, bFlag + ((i64)pWriter->iBtPage<<1));
     sqlite3_step(p->pIdxWriter);
     p->rc = sqlite3_reset(p->pIdxWriter);
+    sqlite3_bind_null(p->pIdxWriter, 2);
   }
   pWriter->iBtPage = 0;
 }
@@ -4909,7 +4913,13 @@ static void fts5MergePrefixLists(
     Fts5Buffer out = {0, 0, 0};
     Fts5Buffer tmp = {0, 0, 0};
 
-    if( sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n) ) return;
+    /* The maximum size of the output is equal to the sum of the two 
+    ** input sizes + 1 varint (9 bytes). The extra varint is because if the
+    ** first rowid in one input is a large negative number, and the first in
+    ** the other a non-negative number, the delta for the non-negative
+    ** number will be larger on disk than the literal integer value
+    ** was.  */
+    if( sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n + 9) ) return;
     fts5DoclistIterInit(p1, &i1);
     fts5DoclistIterInit(p2, &i2);
 
@@ -5003,6 +5013,7 @@ static void fts5MergePrefixLists(
       fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid);
       fts5BufferSafeAppendBlob(&out, i2.aPoslist, i2.aEof - i2.aPoslist);
     }
+    assert( out.n<=(p1->n+p2->n+9) );
 
     fts5BufferSet(&p->rc, p1, out.n, out.p);
     fts5BufferFree(&tmp);
@@ -5250,7 +5261,10 @@ int sqlite3Fts5IndexCharlenToBytelen(
   for(i=0; i<nChar; i++){
     if( n>=nByte ) return 0;      /* Input contains fewer than nChar chars */
     if( (unsigned char)p[n++]>=0xc0 ){
-      while( (p[n] & 0xc0)==0x80 ) n++;
+      while( (p[n] & 0xc0)==0x80 ){
+        n++;
+        if( n>=nByte ) break;
+      }
     }
   }
   return n;

ext/fts5/fts5_main.c

@@ -280,7 +280,7 @@ static void fts5CheckTransactionState(Fts5Table *p, int op, int iSavepoint){
     case FTS5_SAVEPOINT:
       assert( p->ts.eState==1 );
       assert( iSavepoint>=0 );
-      assert( iSavepoint>p->ts.iSavepoint );
+      assert( iSavepoint>=p->ts.iSavepoint );
       p->ts.iSavepoint = iSavepoint;
       break;
       
@@ -535,6 +535,12 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
   aColMap[1] = nCol;
   aColMap[2] = nCol+1;
 
+  assert( SQLITE_INDEX_CONSTRAINT_EQ<SQLITE_INDEX_CONSTRAINT_MATCH );
+  assert( SQLITE_INDEX_CONSTRAINT_GT<SQLITE_INDEX_CONSTRAINT_MATCH );
+  assert( SQLITE_INDEX_CONSTRAINT_LE<SQLITE_INDEX_CONSTRAINT_MATCH );
+  assert( SQLITE_INDEX_CONSTRAINT_GE<SQLITE_INDEX_CONSTRAINT_MATCH );
+  assert( SQLITE_INDEX_CONSTRAINT_LE<SQLITE_INDEX_CONSTRAINT_MATCH );
+
   /* Set idxFlags flags for all WHERE clause terms that will be used. */
   for(i=0; i<pInfo->nConstraint; i++){
     struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
@@ -553,11 +559,11 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
         pInfo->estimatedCost = 1e50;
         return SQLITE_OK;
       }
-    }else{
+    }else if( p->op<=SQLITE_INDEX_CONSTRAINT_MATCH ){
       int j;
       for(j=1; j<ArraySize(aConstraint); j++){
         struct Constraint *pC = &aConstraint[j];
-        if( iCol==aColMap[pC->iCol] && p->op & pC->op && p->usable ){
+        if( iCol==aColMap[pC->iCol] && (p->op & pC->op) && p->usable ){
           pC->iConsIndex = i;
           idxFlags |= pC->fts5op;
         }
@@ -1199,6 +1205,13 @@ static int fts5FilterMethod(
     assert( nVal==0 && pMatch==0 && bOrderByRank==0 && bDesc==0 );
     assert( pCsr->iLastRowid==LARGEST_INT64 );
     assert( pCsr->iFirstRowid==SMALLEST_INT64 );
+    if( pTab->pSortCsr->bDesc ){
+      pCsr->iLastRowid = pTab->pSortCsr->iFirstRowid;
+      pCsr->iFirstRowid = pTab->pSortCsr->iLastRowid;
+    }else{
+      pCsr->iLastRowid = pTab->pSortCsr->iLastRowid;
+      pCsr->iFirstRowid = pTab->pSortCsr->iFirstRowid;
+    }
     pCsr->ePlan = FTS5_PLAN_SOURCE;
     pCsr->pExpr = pTab->pSortCsr->pExpr;
     rc = fts5CursorFirst(pTab, pCsr, bDesc);

ext/fts5/fts5_storage.c

@@ -458,6 +458,7 @@ static int fts5StorageInsertDocsize(
       sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
       sqlite3_step(pReplace);
       rc = sqlite3_reset(pReplace);
+      sqlite3_bind_null(pReplace, 2);
     }
   }
   return rc;
@@ -1118,6 +1119,7 @@ int sqlite3Fts5StorageConfigValue(
     }
     sqlite3_step(pReplace);
     rc = sqlite3_reset(pReplace);
+    sqlite3_bind_null(pReplace, 1);
   }
   if( rc==SQLITE_OK && pVal ){
     int iNew = p->pConfig->iCookie + 1;

ext/fts5/fts5_tcl.c

@@ -433,7 +433,7 @@ static int SQLITE_TCLAPI xF5tApi(
       int iVal;
       int bClear;
       if( Tcl_GetBooleanFromObj(interp, objv[2], &bClear) ) return TCL_ERROR;
-      iVal = ((char*)p->pApi->xGetAuxdata(p->pFts, bClear) - (char*)0);
+      iVal = (int)((char*)p->pApi->xGetAuxdata(p->pFts, bClear) - (char*)0);
       Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal));
       break;
     }
@@ -482,7 +482,7 @@ static int SQLITE_TCLAPI xF5tApi(
 
       rc = p->pApi->xPhraseFirstColumn(p->pFts, iPhrase, &iter, &iCol);
       if( rc!=SQLITE_OK ){
-        Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
+        Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
         return TCL_ERROR;
       }
       for( ; iCol>=0; p->pApi->xPhraseNextColumn(p->pFts, &iter, &iCol)){
@@ -924,7 +924,7 @@ static int SQLITE_TCLAPI f5tTokenizerReturn(
 
   rc = p->xToken(p->pCtx, tflags, zToken, nToken, iStart, iEnd);
   Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
-  return TCL_OK;
+  return rc==SQLITE_OK ? TCL_OK : TCL_ERROR;
 
  usage:
   Tcl_WrongNumArgs(interp, 1, objv, "?-colocated? TEXT START END");

ext/fts5/fts5_tokenize.c

@@ -237,6 +237,8 @@ struct Unicode61Tokenizer {
   int bRemoveDiacritic;           /* True if remove_diacritics=1 is set */
   int nException;
   int *aiException;
+
+  unsigned char aCategory[32];    /* True for token char categories */
 };
 
 static int fts5UnicodeAddExceptions(
@@ -261,7 +263,7 @@ static int fts5UnicodeAddExceptions(
         if( iCode<128 ){
           p->aTokenChar[iCode] = (unsigned char)bTokenChars;
         }else{
-          bToken = sqlite3Fts5UnicodeIsalnum(iCode);
+          bToken = p->aCategory[sqlite3Fts5UnicodeCategory(iCode)];
           assert( (bToken==0 || bToken==1) ); 
           assert( (bTokenChars==0 || bTokenChars==1) );
           if( bToken!=bTokenChars && sqlite3Fts5UnicodeIsdiacritic(iCode)==0 ){
@@ -322,6 +324,21 @@ static void fts5UnicodeDelete(Fts5Tokenizer *pTok){
   return;
 }
 
+static int unicodeSetCategories(Unicode61Tokenizer *p, const char *zCat){
+  const char *z = zCat;
+
+  while( *z ){
+    while( *z==' ' || *z=='\t' ) z++;
+    if( *z && sqlite3Fts5UnicodeCatParse(z, p->aCategory) ){
+      return SQLITE_ERROR;
+    }
+    while( *z!=' ' && *z!='\t' && *z!='\0' ) z++;
+  }
+
+  sqlite3Fts5UnicodeAscii(p->aCategory, p->aTokenChar);
+  return SQLITE_OK;
+}
+
 /*
 ** Create a "unicode61" tokenizer.
 */
@@ -340,15 +357,28 @@ static int fts5UnicodeCreate(
   }else{
     p = (Unicode61Tokenizer*)sqlite3_malloc(sizeof(Unicode61Tokenizer));
     if( p ){
+      const char *zCat = "L* N* Co";
       int i;
       memset(p, 0, sizeof(Unicode61Tokenizer));
-      memcpy(p->aTokenChar, aAsciiTokenChar, sizeof(aAsciiTokenChar));
+
       p->bRemoveDiacritic = 1;
       p->nFold = 64;
       p->aFold = sqlite3_malloc(p->nFold * sizeof(char));
       if( p->aFold==0 ){
         rc = SQLITE_NOMEM;
       }
+
+      /* Search for a "categories" argument */
+      for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
+        if( 0==sqlite3_stricmp(azArg[i], "categories") ){
+          zCat = azArg[i+1];
+        }
+      }
+
+      if( rc==SQLITE_OK ){
+        rc = unicodeSetCategories(p, zCat);
+      }
+
       for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
         const char *zArg = azArg[i+1];
         if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
@@ -362,10 +392,14 @@ static int fts5UnicodeCreate(
         }else
         if( 0==sqlite3_stricmp(azArg[i], "separators") ){
           rc = fts5UnicodeAddExceptions(p, zArg, 0);
+        }else
+        if( 0==sqlite3_stricmp(azArg[i], "categories") ){
+          /* no-op */
         }else{
           rc = SQLITE_ERROR;
         }
       }
+
     }else{
       rc = SQLITE_NOMEM;
     }
@@ -384,8 +418,10 @@ static int fts5UnicodeCreate(
 ** character (not a separator).
 */
 static int fts5UnicodeIsAlnum(Unicode61Tokenizer *p, int iCode){
-  assert( (sqlite3Fts5UnicodeIsalnum(iCode) & 0xFFFFFFFE)==0 );
+  return (
-  return sqlite3Fts5UnicodeIsalnum(iCode) ^ fts5UnicodeIsException(p, iCode);
+    p->aCategory[sqlite3Fts5UnicodeCategory(iCode)]
+    ^ fts5UnicodeIsException(p, iCode)
+  );
 }
 
 static int fts5UnicodeTokenize(

ext/fts5/fts5_unicode2.c

@@ -18,135 +18,6 @@
 
 #include <assert.h>
 
-/*
-** Return true if the argument corresponds to a unicode codepoint
-** classified as either a letter or a number. Otherwise false.
-**
-** The results are undefined if the value passed to this function
-** is less than zero.
-*/
-int sqlite3Fts5UnicodeIsalnum(int c){
-  /* Each unsigned integer in the following array corresponds to a contiguous
-  ** range of unicode codepoints that are not either letters or numbers (i.e.
-  ** codepoints for which this function should return 0).
-  **
-  ** The most significant 22 bits in each 32-bit value contain the first 
-  ** codepoint in the range. The least significant 10 bits are used to store
-  ** the size of the range (always at least 1). In other words, the value 
-  ** ((C<<22) + N) represents a range of N codepoints starting with codepoint 
-  ** C. It is not possible to represent a range larger than 1023 codepoints 
-  ** using this format.
-  */
-  static const unsigned int aEntry[] = {
-    0x00000030, 0x0000E807, 0x00016C06, 0x0001EC2F, 0x0002AC07,
-    0x0002D001, 0x0002D803, 0x0002EC01, 0x0002FC01, 0x00035C01,
-    0x0003DC01, 0x000B0804, 0x000B480E, 0x000B9407, 0x000BB401,
-    0x000BBC81, 0x000DD401, 0x000DF801, 0x000E1002, 0x000E1C01,
-    0x000FD801, 0x00120808, 0x00156806, 0x00162402, 0x00163C01,
-    0x00164437, 0x0017CC02, 0x00180005, 0x00181816, 0x00187802,
-    0x00192C15, 0x0019A804, 0x0019C001, 0x001B5001, 0x001B580F,
-    0x001B9C07, 0x001BF402, 0x001C000E, 0x001C3C01, 0x001C4401,
-    0x001CC01B, 0x001E980B, 0x001FAC09, 0x001FD804, 0x00205804,
-    0x00206C09, 0x00209403, 0x0020A405, 0x0020C00F, 0x00216403,
-    0x00217801, 0x0023901B, 0x00240004, 0x0024E803, 0x0024F812,
-    0x00254407, 0x00258804, 0x0025C001, 0x00260403, 0x0026F001,
-    0x0026F807, 0x00271C02, 0x00272C03, 0x00275C01, 0x00278802,
-    0x0027C802, 0x0027E802, 0x00280403, 0x0028F001, 0x0028F805,
-    0x00291C02, 0x00292C03, 0x00294401, 0x0029C002, 0x0029D401,
-    0x002A0403, 0x002AF001, 0x002AF808, 0x002B1C03, 0x002B2C03,
-    0x002B8802, 0x002BC002, 0x002C0403, 0x002CF001, 0x002CF807,
-    0x002D1C02, 0x002D2C03, 0x002D5802, 0x002D8802, 0x002DC001,
-    0x002E0801, 0x002EF805, 0x002F1803, 0x002F2804, 0x002F5C01,
-    0x002FCC08, 0x00300403, 0x0030F807, 0x00311803, 0x00312804,
-    0x00315402, 0x00318802, 0x0031FC01, 0x00320802, 0x0032F001,
-    0x0032F807, 0x00331803, 0x00332804, 0x00335402, 0x00338802,
-    0x00340802, 0x0034F807, 0x00351803, 0x00352804, 0x00355C01,
-    0x00358802, 0x0035E401, 0x00360802, 0x00372801, 0x00373C06,
-    0x00375801, 0x00376008, 0x0037C803, 0x0038C401, 0x0038D007,
-    0x0038FC01, 0x00391C09, 0x00396802, 0x003AC401, 0x003AD006,
-    0x003AEC02, 0x003B2006, 0x003C041F, 0x003CD00C, 0x003DC417,
-    0x003E340B, 0x003E6424, 0x003EF80F, 0x003F380D, 0x0040AC14,
-    0x00412806, 0x00415804, 0x00417803, 0x00418803, 0x00419C07,
-    0x0041C404, 0x0042080C, 0x00423C01, 0x00426806, 0x0043EC01,
-    0x004D740C, 0x004E400A, 0x00500001, 0x0059B402, 0x005A0001,
-    0x005A6C02, 0x005BAC03, 0x005C4803, 0x005CC805, 0x005D4802,
-    0x005DC802, 0x005ED023, 0x005F6004, 0x005F7401, 0x0060000F,
-    0x0062A401, 0x0064800C, 0x0064C00C, 0x00650001, 0x00651002,
-    0x0066C011, 0x00672002, 0x00677822, 0x00685C05, 0x00687802,
-    0x0069540A, 0x0069801D, 0x0069FC01, 0x006A8007, 0x006AA006,
-    0x006C0005, 0x006CD011, 0x006D6823, 0x006E0003, 0x006E840D,
-    0x006F980E, 0x006FF004, 0x00709014, 0x0070EC05, 0x0071F802,
-    0x00730008, 0x00734019, 0x0073B401, 0x0073C803, 0x00770027,
-    0x0077F004, 0x007EF401, 0x007EFC03, 0x007F3403, 0x007F7403,
-    0x007FB403, 0x007FF402, 0x00800065, 0x0081A806, 0x0081E805,
-    0x00822805, 0x0082801A, 0x00834021, 0x00840002, 0x00840C04,
-    0x00842002, 0x00845001, 0x00845803, 0x00847806, 0x00849401,
-    0x00849C01, 0x0084A401, 0x0084B801, 0x0084E802, 0x00850005,
-    0x00852804, 0x00853C01, 0x00864264, 0x00900027, 0x0091000B,
-    0x0092704E, 0x00940200, 0x009C0475, 0x009E53B9, 0x00AD400A,
-    0x00B39406, 0x00B3BC03, 0x00B3E404, 0x00B3F802, 0x00B5C001,
-    0x00B5FC01, 0x00B7804F, 0x00B8C00C, 0x00BA001A, 0x00BA6C59,
-    0x00BC00D6, 0x00BFC00C, 0x00C00005, 0x00C02019, 0x00C0A807,
-    0x00C0D802, 0x00C0F403, 0x00C26404, 0x00C28001, 0x00C3EC01,
-    0x00C64002, 0x00C6580A, 0x00C70024, 0x00C8001F, 0x00C8A81E,
-    0x00C94001, 0x00C98020, 0x00CA2827, 0x00CB003F, 0x00CC0100,
-    0x01370040, 0x02924037, 0x0293F802, 0x02983403, 0x0299BC10,
-    0x029A7C01, 0x029BC008, 0x029C0017, 0x029C8002, 0x029E2402,
-    0x02A00801, 0x02A01801, 0x02A02C01, 0x02A08C09, 0x02A0D804,
-    0x02A1D004, 0x02A20002, 0x02A2D011, 0x02A33802, 0x02A38012,
-    0x02A3E003, 0x02A4980A, 0x02A51C0D, 0x02A57C01, 0x02A60004,
-    0x02A6CC1B, 0x02A77802, 0x02A8A40E, 0x02A90C01, 0x02A93002,
-    0x02A97004, 0x02A9DC03, 0x02A9EC01, 0x02AAC001, 0x02AAC803,
-    0x02AADC02, 0x02AAF802, 0x02AB0401, 0x02AB7802, 0x02ABAC07,
-    0x02ABD402, 0x02AF8C0B, 0x03600001, 0x036DFC02, 0x036FFC02,
-    0x037FFC01, 0x03EC7801, 0x03ECA401, 0x03EEC810, 0x03F4F802,
-    0x03F7F002, 0x03F8001A, 0x03F88007, 0x03F8C023, 0x03F95013,
-    0x03F9A004, 0x03FBFC01, 0x03FC040F, 0x03FC6807, 0x03FCEC06,
-    0x03FD6C0B, 0x03FF8007, 0x03FFA007, 0x03FFE405, 0x04040003,
-    0x0404DC09, 0x0405E411, 0x0406400C, 0x0407402E, 0x040E7C01,
-    0x040F4001, 0x04215C01, 0x04247C01, 0x0424FC01, 0x04280403,
-    0x04281402, 0x04283004, 0x0428E003, 0x0428FC01, 0x04294009,
-    0x0429FC01, 0x042CE407, 0x04400003, 0x0440E016, 0x04420003,
-    0x0442C012, 0x04440003, 0x04449C0E, 0x04450004, 0x04460003,
-    0x0446CC0E, 0x04471404, 0x045AAC0D, 0x0491C004, 0x05BD442E,
-    0x05BE3C04, 0x074000F6, 0x07440027, 0x0744A4B5, 0x07480046,
-    0x074C0057, 0x075B0401, 0x075B6C01, 0x075BEC01, 0x075C5401,
-    0x075CD401, 0x075D3C01, 0x075DBC01, 0x075E2401, 0x075EA401,
-    0x075F0C01, 0x07BBC002, 0x07C0002C, 0x07C0C064, 0x07C2800F,
-    0x07C2C40E, 0x07C3040F, 0x07C3440F, 0x07C4401F, 0x07C4C03C,
-    0x07C5C02B, 0x07C7981D, 0x07C8402B, 0x07C90009, 0x07C94002,
-    0x07CC0021, 0x07CCC006, 0x07CCDC46, 0x07CE0014, 0x07CE8025,
-    0x07CF1805, 0x07CF8011, 0x07D0003F, 0x07D10001, 0x07D108B6,
-    0x07D3E404, 0x07D4003E, 0x07D50004, 0x07D54018, 0x07D7EC46,
-    0x07D9140B, 0x07DA0046, 0x07DC0074, 0x38000401, 0x38008060,
-    0x380400F0,
-  };
-  static const unsigned int aAscii[4] = {
-    0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001,
-  };
-
-  if( (unsigned int)c<128 ){
-    return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 );
-  }else if( (unsigned int)c<(1<<22) ){
-    unsigned int key = (((unsigned int)c)<<10) | 0x000003FF;
-    int iRes = 0;
-    int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
-    int iLo = 0;
-    while( iHi>=iLo ){
-      int iTest = (iHi + iLo) / 2;
-      if( key >= aEntry[iTest] ){
-        iRes = iTest;
-        iLo = iTest+1;
-      }else{
-        iHi = iTest-1;
-      }
-    }
-    assert( aEntry[0]<key );
-    assert( key>=aEntry[iRes] );
-    return (((unsigned int)c) >= ((aEntry[iRes]>>10) + (aEntry[iRes]&0x3FF)));
-  }
-  return 1;
-}
 
 
 /*
@@ -358,3 +229,536 @@ int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic){
 
   return ret;
 }
+
+
+#if 0
+int sqlite3Fts5UnicodeNCat(void) { 
+  return 32;
+}
+#endif
+
+int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ 
+  aArray[0] = 1;
+  switch( zCat[0] ){
+    case 'C':
+          switch( zCat[1] ){
+            case 'c': aArray[1] = 1; break;
+            case 'f': aArray[2] = 1; break;
+            case 'n': aArray[3] = 1; break;
+            case 's': aArray[4] = 1; break;
+            case 'o': aArray[31] = 1; break;
+            case '*': 
+              aArray[1] = 1;
+              aArray[2] = 1;
+              aArray[3] = 1;
+              aArray[4] = 1;
+              aArray[31] = 1;
+              break;
+            default: return 1;          }
+          break;
+
+    case 'L':
+          switch( zCat[1] ){
+            case 'l': aArray[5] = 1; break;
+            case 'm': aArray[6] = 1; break;
+            case 'o': aArray[7] = 1; break;
+            case 't': aArray[8] = 1; break;
+            case 'u': aArray[9] = 1; break;
+            case 'C': aArray[30] = 1; break;
+            case '*': 
+              aArray[5] = 1;
+              aArray[6] = 1;
+              aArray[7] = 1;
+              aArray[8] = 1;
+              aArray[9] = 1;
+              aArray[30] = 1;
+              break;
+            default: return 1;          }
+          break;
+
+    case 'M':
+          switch( zCat[1] ){
+            case 'c': aArray[10] = 1; break;
+            case 'e': aArray[11] = 1; break;
+            case 'n': aArray[12] = 1; break;
+            case '*': 
+              aArray[10] = 1;
+              aArray[11] = 1;
+              aArray[12] = 1;
+              break;
+            default: return 1;          }
+          break;
+
+    case 'N':
+          switch( zCat[1] ){
+            case 'd': aArray[13] = 1; break;
+            case 'l': aArray[14] = 1; break;
+            case 'o': aArray[15] = 1; break;
+            case '*': 
+              aArray[13] = 1;
+              aArray[14] = 1;
+              aArray[15] = 1;
+              break;
+            default: return 1;          }
+          break;
+
+    case 'P':
+          switch( zCat[1] ){
+            case 'c': aArray[16] = 1; break;
+            case 'd': aArray[17] = 1; break;
+            case 'e': aArray[18] = 1; break;
+            case 'f': aArray[19] = 1; break;
+            case 'i': aArray[20] = 1; break;
+            case 'o': aArray[21] = 1; break;
+            case 's': aArray[22] = 1; break;
+            case '*': 
+              aArray[16] = 1;
+              aArray[17] = 1;
+              aArray[18] = 1;
+              aArray[19] = 1;
+              aArray[20] = 1;
+              aArray[21] = 1;
+              aArray[22] = 1;
+              break;
+            default: return 1;          }
+          break;
+
+    case 'S':
+          switch( zCat[1] ){
+            case 'c': aArray[23] = 1; break;
+            case 'k': aArray[24] = 1; break;
+            case 'm': aArray[25] = 1; break;
+            case 'o': aArray[26] = 1; break;
+            case '*': 
+              aArray[23] = 1;
+              aArray[24] = 1;
+              aArray[25] = 1;
+              aArray[26] = 1;
+              break;
+            default: return 1;          }
+          break;
+
+    case 'Z':
+          switch( zCat[1] ){
+            case 'l': aArray[27] = 1; break;
+            case 'p': aArray[28] = 1; break;
+            case 's': aArray[29] = 1; break;
+            case '*': 
+              aArray[27] = 1;
+              aArray[28] = 1;
+              aArray[29] = 1;
+              break;
+            default: return 1;          }
+          break;
+
+  }
+  return 0;
+}
+
+static u16 aFts5UnicodeBlock[] = {
+    0,     1471,  1753,  1760,  1760,  1760,  1760,  1760,  1760,  1760,  
+    1760,  1760,  1760,  1760,  1760,  1763,  1765,  
+  };
+static u16 aFts5UnicodeMap[] = {
+    0,     32,    33,    36,    37,    40,    41,    42,    43,    44,    
+    45,    46,    48,    58,    60,    63,    65,    91,    92,    93,    
+    94,    95,    96,    97,    123,   124,   125,   126,   127,   160,   
+    161,   162,   166,   167,   168,   169,   170,   171,   172,   173,   
+    174,   175,   176,   177,   178,   180,   181,   182,   184,   185,   
+    186,   187,   188,   191,   192,   215,   216,   223,   247,   248,   
+    256,   312,   313,   329,   330,   377,   383,   385,   387,   388,   
+    391,   394,   396,   398,   402,   403,   405,   406,   409,   412,   
+    414,   415,   417,   418,   423,   427,   428,   431,   434,   436,   
+    437,   440,   442,   443,   444,   446,   448,   452,   453,   454,   
+    455,   456,   457,   458,   459,   460,   461,   477,   478,   496,   
+    497,   498,   499,   500,   503,   505,   506,   564,   570,   572,   
+    573,   575,   577,   580,   583,   584,   592,   660,   661,   688,   
+    706,   710,   722,   736,   741,   748,   749,   750,   751,   768,   
+    880,   884,   885,   886,   890,   891,   894,   900,   902,   903,   
+    904,   908,   910,   912,   913,   931,   940,   975,   977,   978,   
+    981,   984,   1008,  1012,  1014,  1015,  1018,  1020,  1021,  1072,  
+    1120,  1154,  1155,  1160,  1162,  1217,  1231,  1232,  1329,  1369,  
+    1370,  1377,  1417,  1418,  1423,  1425,  1470,  1471,  1472,  1473,  
+    1475,  1476,  1478,  1479,  1488,  1520,  1523,  1536,  1542,  1545,  
+    1547,  1548,  1550,  1552,  1563,  1566,  1568,  1600,  1601,  1611,  
+    1632,  1642,  1646,  1648,  1649,  1748,  1749,  1750,  1757,  1758,  
+    1759,  1765,  1767,  1769,  1770,  1774,  1776,  1786,  1789,  1791,  
+    1792,  1807,  1808,  1809,  1810,  1840,  1869,  1958,  1969,  1984,  
+    1994,  2027,  2036,  2038,  2039,  2042,  2048,  2070,  2074,  2075,  
+    2084,  2085,  2088,  2089,  2096,  2112,  2137,  2142,  2208,  2210,  
+    2276,  2304,  2307,  2308,  2362,  2363,  2364,  2365,  2366,  2369,  
+    2377,  2381,  2382,  2384,  2385,  2392,  2402,  2404,  2406,  2416,  
+    2417,  2418,  2425,  2433,  2434,  2437,  2447,  2451,  2474,  2482,  
+    2486,  2492,  2493,  2494,  2497,  2503,  2507,  2509,  2510,  2519,  
+    2524,  2527,  2530,  2534,  2544,  2546,  2548,  2554,  2555,  2561,  
+    2563,  2565,  2575,  2579,  2602,  2610,  2613,  2616,  2620,  2622,  
+    2625,  2631,  2635,  2641,  2649,  2654,  2662,  2672,  2674,  2677,  
+    2689,  2691,  2693,  2703,  2707,  2730,  2738,  2741,  2748,  2749,  
+    2750,  2753,  2759,  2761,  2763,  2765,  2768,  2784,  2786,  2790,  
+    2800,  2801,  2817,  2818,  2821,  2831,  2835,  2858,  2866,  2869,  
+    2876,  2877,  2878,  2879,  2880,  2881,  2887,  2891,  2893,  2902,  
+    2903,  2908,  2911,  2914,  2918,  2928,  2929,  2930,  2946,  2947,  
+    2949,  2958,  2962,  2969,  2972,  2974,  2979,  2984,  2990,  3006,  
+    3008,  3009,  3014,  3018,  3021,  3024,  3031,  3046,  3056,  3059,  
+    3065,  3066,  3073,  3077,  3086,  3090,  3114,  3125,  3133,  3134,  
+    3137,  3142,  3146,  3157,  3160,  3168,  3170,  3174,  3192,  3199,  
+    3202,  3205,  3214,  3218,  3242,  3253,  3260,  3261,  3262,  3263,  
+    3264,  3270,  3271,  3274,  3276,  3285,  3294,  3296,  3298,  3302,  
+    3313,  3330,  3333,  3342,  3346,  3389,  3390,  3393,  3398,  3402,  
+    3405,  3406,  3415,  3424,  3426,  3430,  3440,  3449,  3450,  3458,  
+    3461,  3482,  3507,  3517,  3520,  3530,  3535,  3538,  3542,  3544,  
+    3570,  3572,  3585,  3633,  3634,  3636,  3647,  3648,  3654,  3655,  
+    3663,  3664,  3674,  3713,  3716,  3719,  3722,  3725,  3732,  3737,  
+    3745,  3749,  3751,  3754,  3757,  3761,  3762,  3764,  3771,  3773,  
+    3776,  3782,  3784,  3792,  3804,  3840,  3841,  3844,  3859,  3860,  
+    3861,  3864,  3866,  3872,  3882,  3892,  3893,  3894,  3895,  3896,  
+    3897,  3898,  3899,  3900,  3901,  3902,  3904,  3913,  3953,  3967,  
+    3968,  3973,  3974,  3976,  3981,  3993,  4030,  4038,  4039,  4046,  
+    4048,  4053,  4057,  4096,  4139,  4141,  4145,  4146,  4152,  4153,  
+    4155,  4157,  4159,  4160,  4170,  4176,  4182,  4184,  4186,  4190,  
+    4193,  4194,  4197,  4199,  4206,  4209,  4213,  4226,  4227,  4229,  
+    4231,  4237,  4238,  4239,  4240,  4250,  4253,  4254,  4256,  4295,  
+    4301,  4304,  4347,  4348,  4349,  4682,  4688,  4696,  4698,  4704,  
+    4746,  4752,  4786,  4792,  4800,  4802,  4808,  4824,  4882,  4888,  
+    4957,  4960,  4969,  4992,  5008,  5024,  5120,  5121,  5741,  5743,  
+    5760,  5761,  5787,  5788,  5792,  5867,  5870,  5888,  5902,  5906,  
+    5920,  5938,  5941,  5952,  5970,  5984,  5998,  6002,  6016,  6068,  
+    6070,  6071,  6078,  6086,  6087,  6089,  6100,  6103,  6104,  6107,  
+    6108,  6109,  6112,  6128,  6144,  6150,  6151,  6155,  6158,  6160,  
+    6176,  6211,  6212,  6272,  6313,  6314,  6320,  6400,  6432,  6435,  
+    6439,  6441,  6448,  6450,  6451,  6457,  6464,  6468,  6470,  6480,  
+    6512,  6528,  6576,  6593,  6600,  6608,  6618,  6622,  6656,  6679,  
+    6681,  6686,  6688,  6741,  6742,  6743,  6744,  6752,  6753,  6754,  
+    6755,  6757,  6765,  6771,  6783,  6784,  6800,  6816,  6823,  6824,  
+    6912,  6916,  6917,  6964,  6965,  6966,  6971,  6972,  6973,  6978,  
+    6979,  6981,  6992,  7002,  7009,  7019,  7028,  7040,  7042,  7043,  
+    7073,  7074,  7078,  7080,  7082,  7083,  7084,  7086,  7088,  7098,  
+    7142,  7143,  7144,  7146,  7149,  7150,  7151,  7154,  7164,  7168,  
+    7204,  7212,  7220,  7222,  7227,  7232,  7245,  7248,  7258,  7288,  
+    7294,  7360,  7376,  7379,  7380,  7393,  7394,  7401,  7405,  7406,  
+    7410,  7412,  7413,  7424,  7468,  7531,  7544,  7545,  7579,  7616,  
+    7676,  7680,  7830,  7838,  7936,  7944,  7952,  7960,  7968,  7976,  
+    7984,  7992,  8000,  8008,  8016,  8025,  8027,  8029,  8031,  8033,  
+    8040,  8048,  8064,  8072,  8080,  8088,  8096,  8104,  8112,  8118,  
+    8120,  8124,  8125,  8126,  8127,  8130,  8134,  8136,  8140,  8141,  
+    8144,  8150,  8152,  8157,  8160,  8168,  8173,  8178,  8182,  8184,  
+    8188,  8189,  8192,  8203,  8208,  8214,  8216,  8217,  8218,  8219,  
+    8221,  8222,  8223,  8224,  8232,  8233,  8234,  8239,  8240,  8249,  
+    8250,  8251,  8255,  8257,  8260,  8261,  8262,  8263,  8274,  8275,  
+    8276,  8277,  8287,  8288,  8298,  8304,  8305,  8308,  8314,  8317,  
+    8318,  8319,  8320,  8330,  8333,  8334,  8336,  8352,  8400,  8413,  
+    8417,  8418,  8421,  8448,  8450,  8451,  8455,  8456,  8458,  8459,  
+    8462,  8464,  8467,  8468,  8469,  8470,  8472,  8473,  8478,  8484,  
+    8485,  8486,  8487,  8488,  8489,  8490,  8494,  8495,  8496,  8500,  
+    8501,  8505,  8506,  8508,  8510,  8512,  8517,  8519,  8522,  8523,  
+    8524,  8526,  8527,  8528,  8544,  8579,  8581,  8585,  8592,  8597,  
+    8602,  8604,  8608,  8609,  8611,  8612,  8614,  8615,  8622,  8623,  
+    8654,  8656,  8658,  8659,  8660,  8661,  8692,  8960,  8968,  8972,  
+    8992,  8994,  9001,  9002,  9003,  9084,  9085,  9115,  9140,  9180,  
+    9186,  9216,  9280,  9312,  9372,  9450,  9472,  9655,  9656,  9665,  
+    9666,  9720,  9728,  9839,  9840,  9985,  10088, 10089, 10090, 10091, 
+    10092, 10093, 10094, 10095, 10096, 10097, 10098, 10099, 10100, 10101, 
+    10102, 10132, 10176, 10181, 10182, 10183, 10214, 10215, 10216, 10217, 
+    10218, 10219, 10220, 10221, 10222, 10223, 10224, 10240, 10496, 10627, 
+    10628, 10629, 10630, 10631, 10632, 10633, 10634, 10635, 10636, 10637, 
+    10638, 10639, 10640, 10641, 10642, 10643, 10644, 10645, 10646, 10647, 
+    10648, 10649, 10712, 10713, 10714, 10715, 10716, 10748, 10749, 10750, 
+    11008, 11056, 11077, 11079, 11088, 11264, 11312, 11360, 11363, 11365, 
+    11367, 11374, 11377, 11378, 11380, 11381, 11383, 11388, 11390, 11393, 
+    11394, 11492, 11493, 11499, 11503, 11506, 11513, 11517, 11518, 11520, 
+    11559, 11565, 11568, 11631, 11632, 11647, 11648, 11680, 11688, 11696, 
+    11704, 11712, 11720, 11728, 11736, 11744, 11776, 11778, 11779, 11780, 
+    11781, 11782, 11785, 11786, 11787, 11788, 11789, 11790, 11799, 11800, 
+    11802, 11803, 11804, 11805, 11806, 11808, 11809, 11810, 11811, 11812, 
+    11813, 11814, 11815, 11816, 11817, 11818, 11823, 11824, 11834, 11904, 
+    11931, 12032, 12272, 12288, 12289, 12292, 12293, 12294, 12295, 12296, 
+    12297, 12298, 12299, 12300, 12301, 12302, 12303, 12304, 12305, 12306, 
+    12308, 12309, 12310, 12311, 12312, 12313, 12314, 12315, 12316, 12317, 
+    12318, 12320, 12321, 12330, 12334, 12336, 12337, 12342, 12344, 12347, 
+    12348, 12349, 12350, 12353, 12441, 12443, 12445, 12447, 12448, 12449, 
+    12539, 12540, 12543, 12549, 12593, 12688, 12690, 12694, 12704, 12736, 
+    12784, 12800, 12832, 12842, 12872, 12880, 12881, 12896, 12928, 12938, 
+    12977, 12992, 13056, 13312, 19893, 19904, 19968, 40908, 40960, 40981, 
+    40982, 42128, 42192, 42232, 42238, 42240, 42508, 42509, 42512, 42528, 
+    42538, 42560, 42606, 42607, 42608, 42611, 42612, 42622, 42623, 42624, 
+    42655, 42656, 42726, 42736, 42738, 42752, 42775, 42784, 42786, 42800, 
+    42802, 42864, 42865, 42873, 42878, 42888, 42889, 42891, 42896, 42912, 
+    43000, 43002, 43003, 43010, 43011, 43014, 43015, 43019, 43020, 43043, 
+    43045, 43047, 43048, 43056, 43062, 43064, 43065, 43072, 43124, 43136, 
+    43138, 43188, 43204, 43214, 43216, 43232, 43250, 43256, 43259, 43264, 
+    43274, 43302, 43310, 43312, 43335, 43346, 43359, 43360, 43392, 43395, 
+    43396, 43443, 43444, 43446, 43450, 43452, 43453, 43457, 43471, 43472, 
+    43486, 43520, 43561, 43567, 43569, 43571, 43573, 43584, 43587, 43588, 
+    43596, 43597, 43600, 43612, 43616, 43632, 43633, 43639, 43642, 43643, 
+    43648, 43696, 43697, 43698, 43701, 43703, 43705, 43710, 43712, 43713, 
+    43714, 43739, 43741, 43742, 43744, 43755, 43756, 43758, 43760, 43762, 
+    43763, 43765, 43766, 43777, 43785, 43793, 43808, 43816, 43968, 44003, 
+    44005, 44006, 44008, 44009, 44011, 44012, 44013, 44016, 44032, 55203, 
+    55216, 55243, 55296, 56191, 56319, 57343, 57344, 63743, 63744, 64112, 
+    64256, 64275, 64285, 64286, 64287, 64297, 64298, 64312, 64318, 64320, 
+    64323, 64326, 64434, 64467, 64830, 64831, 64848, 64914, 65008, 65020, 
+    65021, 65024, 65040, 65047, 65048, 65049, 65056, 65072, 65073, 65075, 
+    65077, 65078, 65079, 65080, 65081, 65082, 65083, 65084, 65085, 65086, 
+    65087, 65088, 65089, 65090, 65091, 65092, 65093, 65095, 65096, 65097, 
+    65101, 65104, 65108, 65112, 65113, 65114, 65115, 65116, 65117, 65118, 
+    65119, 65122, 65123, 65124, 65128, 65129, 65130, 65136, 65142, 65279, 
+    65281, 65284, 65285, 65288, 65289, 65290, 65291, 65292, 65293, 65294, 
+    65296, 65306, 65308, 65311, 65313, 65339, 65340, 65341, 65342, 65343, 
+    65344, 65345, 65371, 65372, 65373, 65374, 65375, 65376, 65377, 65378, 
+    65379, 65380, 65382, 65392, 65393, 65438, 65440, 65474, 65482, 65490, 
+    65498, 65504, 65506, 65507, 65508, 65509, 65512, 65513, 65517, 65529, 
+    65532, 0,     13,    40,    60,    63,    80,    128,   256,   263,   
+    311,   320,   373,   377,   394,   400,   464,   509,   640,   672,   
+    768,   800,   816,   833,   834,   842,   896,   927,   928,   968,   
+    976,   977,   1024,  1064,  1104,  1184,  2048,  2056,  2058,  2103,  
+    2108,  2111,  2135,  2136,  2304,  2326,  2335,  2336,  2367,  2432,  
+    2494,  2560,  2561,  2565,  2572,  2576,  2581,  2585,  2616,  2623,  
+    2624,  2640,  2656,  2685,  2687,  2816,  2873,  2880,  2904,  2912,  
+    2936,  3072,  3680,  4096,  4097,  4098,  4099,  4152,  4167,  4178,  
+    4198,  4224,  4226,  4227,  4272,  4275,  4279,  4281,  4283,  4285,  
+    4286,  4304,  4336,  4352,  4355,  4391,  4396,  4397,  4406,  4416,  
+    4480,  4482,  4483,  4531,  4534,  4543,  4545,  4549,  4560,  5760,  
+    5803,  5804,  5805,  5806,  5808,  5814,  5815,  5824,  8192,  9216,  
+    9328,  12288, 26624, 28416, 28496, 28497, 28559, 28563, 45056, 53248, 
+    53504, 53545, 53605, 53607, 53610, 53613, 53619, 53627, 53635, 53637, 
+    53644, 53674, 53678, 53760, 53826, 53829, 54016, 54112, 54272, 54298, 
+    54324, 54350, 54358, 54376, 54402, 54428, 54430, 54434, 54437, 54441, 
+    54446, 54454, 54459, 54461, 54469, 54480, 54506, 54532, 54535, 54541, 
+    54550, 54558, 54584, 54587, 54592, 54598, 54602, 54610, 54636, 54662, 
+    54688, 54714, 54740, 54766, 54792, 54818, 54844, 54870, 54896, 54922, 
+    54952, 54977, 54978, 55003, 55004, 55010, 55035, 55036, 55061, 55062, 
+    55068, 55093, 55094, 55119, 55120, 55126, 55151, 55152, 55177, 55178, 
+    55184, 55209, 55210, 55235, 55236, 55242, 55246, 60928, 60933, 60961, 
+    60964, 60967, 60969, 60980, 60985, 60987, 60994, 60999, 61001, 61003, 
+    61005, 61009, 61012, 61015, 61017, 61019, 61021, 61023, 61025, 61028, 
+    61031, 61036, 61044, 61049, 61054, 61056, 61067, 61089, 61093, 61099, 
+    61168, 61440, 61488, 61600, 61617, 61633, 61649, 61696, 61712, 61744, 
+    61808, 61926, 61968, 62016, 62032, 62208, 62256, 62263, 62336, 62368, 
+    62406, 62432, 62464, 62528, 62530, 62713, 62720, 62784, 62800, 62971, 
+    63045, 63104, 63232, 0,     42710, 42752, 46900, 46912, 47133, 63488, 
+    1,     32,    256,   0,     65533, 
+  };
+static u16 aFts5UnicodeData[] = {
+    1025,  61,    117,   55,    117,   54,    50,    53,    57,    53,    
+    49,    85,    333,   85,    121,   85,    841,   54,    53,    50,    
+    56,    48,    56,    837,   54,    57,    50,    57,    1057,  61,    
+    53,    151,   58,    53,    56,    58,    39,    52,    57,    34,    
+    58,    56,    58,    57,    79,    56,    37,    85,    56,    47,    
+    39,    51,    111,   53,    745,   57,    233,   773,   57,    261,   
+    1822,  37,    542,   37,    1534,  222,   69,    73,    37,    126,   
+    126,   73,    69,    137,   37,    73,    37,    105,   101,   73,    
+    37,    73,    37,    190,   158,   37,    126,   126,   73,    37,    
+    126,   94,    37,    39,    94,    69,    135,   41,    40,    37,    
+    41,    40,    37,    41,    40,    37,    542,   37,    606,   37,    
+    41,    40,    37,    126,   73,    37,    1886,  197,   73,    37,    
+    73,    69,    126,   105,   37,    286,   2181,  39,    869,   582,   
+    152,   390,   472,   166,   248,   38,    56,    38,    568,   3596,  
+    158,   38,    56,    94,    38,    101,   53,    88,    41,    53,    
+    105,   41,    73,    37,    553,   297,   1125,  94,    37,    105,   
+    101,   798,   133,   94,    57,    126,   94,    37,    1641,  1541,  
+    1118,  58,    172,   75,    1790,  478,   37,    2846,  1225,  38,    
+    213,   1253,  53,    49,    55,    1452,  49,    44,    53,    76,    
+    53,    76,    53,    44,    871,   103,   85,    162,   121,   85,    
+    55,    85,    90,    364,   53,    85,    1031,  38,    327,   684,   
+    333,   149,   71,    44,    3175,  53,    39,    236,   34,    58,    
+    204,   70,    76,    58,    140,   71,    333,   103,   90,    39,    
+    469,   34,    39,    44,    967,   876,   2855,  364,   39,    333,   
+    1063,  300,   70,    58,    117,   38,    711,   140,   38,    300,   
+    38,    108,   38,    172,   501,   807,   108,   53,    39,    359,   
+    876,   108,   42,    1735,  44,    42,    44,    39,    106,   268,   
+    138,   44,    74,    39,    236,   327,   76,    85,    333,   53,    
+    38,    199,   231,   44,    74,    263,   71,    711,   231,   39,    
+    135,   44,    39,    106,   140,   74,    74,    44,    39,    42,    
+    71,    103,   76,    333,   71,    87,    207,   58,    55,    76,    
+    42,    199,   71,    711,   231,   71,    71,    71,    44,    106,   
+    76,    76,    108,   44,    135,   39,    333,   76,    103,   44,    
+    76,    42,    295,   103,   711,   231,   71,    167,   44,    39,    
+    106,   172,   76,    42,    74,    44,    39,    71,    76,    333,   
+    53,    55,    44,    74,    263,   71,    711,   231,   71,    167,   
+    44,    39,    42,    44,    42,    140,   74,    74,    44,    44,    
+    42,    71,    103,   76,    333,   58,    39,    207,   44,    39,    
+    199,   103,   135,   71,    39,    71,    71,    103,   391,   74,    
+    44,    74,    106,   106,   44,    39,    42,    333,   111,   218,   
+    55,    58,    106,   263,   103,   743,   327,   167,   39,    108,   
+    138,   108,   140,   76,    71,    71,    76,    333,   239,   58,    
+    74,    263,   103,   743,   327,   167,   44,    39,    42,    44,    
+    170,   44,    74,    74,    76,    74,    39,    71,    76,    333,   
+    71,    74,    263,   103,   1319,  39,    106,   140,   106,   106,   
+    44,    39,    42,    71,    76,    333,   207,   58,    199,   74,    
+    583,   775,   295,   39,    231,   44,    106,   108,   44,    266,   
+    74,    53,    1543,  44,    71,    236,   55,    199,   38,    268,   
+    53,    333,   85,    71,    39,    71,    39,    39,    135,   231,   
+    103,   39,    39,    71,    135,   44,    71,    204,   76,    39,    
+    167,   38,    204,   333,   135,   39,    122,   501,   58,    53,    
+    122,   76,    218,   333,   335,   58,    44,    58,    44,    58,    
+    44,    54,    50,    54,    50,    74,    263,   1159,  460,   42,    
+    172,   53,    76,    167,   364,   1164,  282,   44,    218,   90,    
+    181,   154,   85,    1383,  74,    140,   42,    204,   42,    76,    
+    74,    76,    39,    333,   213,   199,   74,    76,    135,   108,   
+    39,    106,   71,    234,   103,   140,   423,   44,    74,    76,    
+    202,   44,    39,    42,    333,   106,   44,    90,    1225,  41,    
+    41,    1383,  53,    38,    10631, 135,   231,   39,    135,   1319,  
+    135,   1063,  135,   231,   39,    135,   487,   1831,  135,   2151,  
+    108,   309,   655,   519,   346,   2727,  49,    19847, 85,    551,   
+    61,    839,   54,    50,    2407,  117,   110,   423,   135,   108,   
+    583,   108,   85,    583,   76,    423,   103,   76,    1671,  76,    
+    42,    236,   266,   44,    74,    364,   117,   38,    117,   55,    
+    39,    44,    333,   335,   213,   49,    149,   108,   61,    333,   
+    1127,  38,    1671,  1319,  44,    39,    2247,  935,   108,   138,   
+    76,    106,   74,    44,    202,   108,   58,    85,    333,   967,   
+    167,   1415,  554,   231,   74,    333,   47,    1114,  743,   76,    
+    106,   85,    1703,  42,    44,    42,    236,   44,    42,    44,    
+    74,    268,   202,   332,   44,    333,   333,   245,   38,    213,   
+    140,   42,    1511,  44,    42,    172,   42,    44,    170,   44,    
+    74,    231,   333,   245,   346,   300,   314,   76,    42,    967,   
+    42,    140,   74,    76,    42,    44,    74,    71,    333,   1415,  
+    44,    42,    76,    106,   44,    42,    108,   74,    149,   1159,  
+    266,   268,   74,    76,    181,   333,   103,   333,   967,   198,   
+    85,    277,   108,   53,    428,   42,    236,   135,   44,    135,   
+    74,    44,    71,    1413,  2022,  421,   38,    1093,  1190,  1260,  
+    140,   4830,  261,   3166,  261,   265,   197,   201,   261,   265,   
+    261,   265,   197,   201,   261,   41,    41,    41,    94,    229,   
+    265,   453,   261,   264,   261,   264,   261,   264,   165,   69,    
+    137,   40,    56,    37,    120,   101,   69,    137,   40,    120,   
+    133,   69,    137,   120,   261,   169,   120,   101,   69,    137,   
+    40,    88,    381,   162,   209,   85,    52,    51,    54,    84,    
+    51,    54,    52,    277,   59,    60,    162,   61,    309,   52,    
+    51,    149,   80,    117,   57,    54,    50,    373,   57,    53,    
+    48,    341,   61,    162,   194,   47,    38,    207,   121,   54,    
+    50,    38,    335,   121,   54,    50,    422,   855,   428,   139,   
+    44,    107,   396,   90,    41,    154,   41,    90,    37,    105,   
+    69,    105,   37,    58,    41,    90,    57,    169,   218,   41,    
+    58,    41,    58,    41,    58,    137,   58,    37,    137,   37,    
+    135,   37,    90,    69,    73,    185,   94,    101,   58,    57,    
+    90,    37,    58,    527,   1134,  94,    142,   47,    185,   186,   
+    89,    154,   57,    90,    57,    90,    57,    250,   57,    1018,  
+    89,    90,    57,    58,    57,    1018,  8601,  282,   153,   666,   
+    89,    250,   54,    50,    2618,  57,    986,   825,   1306,  217,   
+    602,   1274,  378,   1935,  2522,  719,   5882,  57,    314,   57,    
+    1754,  281,   3578,  57,    4634,  3322,  54,    50,    54,    50,    
+    54,    50,    54,    50,    54,    50,    54,    50,    54,    50,    
+    975,   1434,  185,   54,    50,    1017,  54,    50,    54,    50,    
+    54,    50,    54,    50,    54,    50,    537,   8218,  4217,  54,    
+    50,    54,    50,    54,    50,    54,    50,    54,    50,    54,    
+    50,    54,    50,    54,    50,    54,    50,    54,    50,    54,    
+    50,    2041,  54,    50,    54,    50,    1049,  54,    50,    8281,  
+    1562,  697,   90,    217,   346,   1513,  1509,  126,   73,    69,    
+    254,   105,   37,    94,    37,    94,    165,   70,    105,   37,    
+    3166,  37,    218,   158,   108,   94,    149,   47,    85,    1221,  
+    37,    37,    1799,  38,    53,    44,    743,   231,   231,   231,   
+    231,   231,   231,   231,   231,   1036,  85,    52,    51,    52,    
+    51,    117,   52,    51,    53,    52,    51,    309,   49,    85,    
+    49,    53,    52,    51,    85,    52,    51,    54,    50,    54,    
+    50,    54,    50,    54,    50,    181,   38,    341,   81,    858,   
+    2874,  6874,  410,   61,    117,   58,    38,    39,    46,    54,    
+    50,    54,    50,    54,    50,    54,    50,    54,    50,    90,    
+    54,    50,    54,    50,    54,    50,    54,    50,    49,    54,    
+    82,    58,    302,   140,   74,    49,    166,   90,    110,   38,    
+    39,    53,    90,    2759,  76,    88,    70,    39,    49,    2887,  
+    53,    102,   39,    1319,  3015,  90,    143,   346,   871,   1178,  
+    519,   1018,  335,   986,   271,   58,    495,   1050,  335,   1274,  
+    495,   2042,  8218,  39,    39,    2074,  39,    39,    679,   38,    
+    36583, 1786,  1287,  198,   85,    8583,  38,    117,   519,   333,   
+    71,    1502,  39,    44,    107,   53,    332,   53,    38,    798,   
+    44,    2247,  334,   76,    213,   760,   294,   88,    478,   69,    
+    2014,  38,    261,   190,   350,   38,    88,    158,   158,   382,   
+    70,    37,    231,   44,    103,   44,    135,   44,    743,   74,    
+    76,    42,    154,   207,   90,    55,    58,    1671,  149,   74,    
+    1607,  522,   44,    85,    333,   588,   199,   117,   39,    333,   
+    903,   268,   85,    743,   364,   74,    53,    935,   108,   42,    
+    1511,  44,    74,    140,   74,    44,    138,   437,   38,    333,   
+    85,    1319,  204,   74,    76,    74,    76,    103,   44,    263,   
+    44,    42,    333,   149,   519,   38,    199,   122,   39,    42,    
+    1543,  44,    39,    108,   71,    76,    167,   76,    39,    44,    
+    39,    71,    38,    85,    359,   42,    76,    74,    85,    39,    
+    70,    42,    44,    199,   199,   199,   231,   231,   1127,  74,    
+    44,    74,    44,    74,    53,    42,    44,    333,   39,    39,    
+    743,   1575,  36,    68,    68,    36,    63,    63,    11719, 3399,  
+    229,   165,   39,    44,    327,   57,    423,   167,   39,    71,    
+    71,    3463,  536,   11623, 54,    50,    2055,  1735,  391,   55,    
+    58,    524,   245,   54,    50,    53,    236,   53,    81,    80,    
+    54,    50,    54,    50,    54,    50,    54,    50,    54,    50,    
+    54,    50,    54,    50,    54,    50,    85,    54,    50,    149,   
+    112,   117,   149,   49,    54,    50,    54,    50,    54,    50,    
+    117,   57,    49,    121,   53,    55,    85,    167,   4327,  34,    
+    117,   55,    117,   54,    50,    53,    57,    53,    49,    85,    
+    333,   85,    121,   85,    841,   54,    53,    50,    56,    48,    
+    56,    837,   54,    57,    50,    57,    54,    50,    53,    54,    
+    50,    85,    327,   38,    1447,  70,    999,   199,   199,   199,   
+    103,   87,    57,    56,    58,    87,    58,    153,   90,    98,    
+    90,    391,   839,   615,   71,    487,   455,   3943,  117,   1455,  
+    314,   1710,  143,   570,   47,    410,   1466,  44,    935,   1575,  
+    999,   143,   551,   46,    263,   46,    967,   53,    1159,  263,   
+    53,    174,   1289,  1285,  2503,  333,   199,   39,    1415,  71,    
+    39,    743,   53,    271,   711,   207,   53,    839,   53,    1799,  
+    71,    39,    108,   76,    140,   135,   103,   871,   108,   44,    
+    271,   309,   935,   79,    53,    1735,  245,   711,   271,   615,   
+    271,   2343,  1007,  42,    44,    42,    1703,  492,   245,   655,   
+    333,   76,    42,    1447,  106,   140,   74,    76,    85,    34,    
+    149,   807,   333,   108,   1159,  172,   42,    268,   333,   149,   
+    76,    42,    1543,  106,   300,   74,    135,   149,   333,   1383,  
+    44,    42,    44,    74,    204,   42,    44,    333,   28135, 3182,  
+    149,   34279, 18215, 2215,  39,    1482,  140,   422,   71,    7898,  
+    1274,  1946,  74,    108,   122,   202,   258,   268,   90,    236,   
+    986,   140,   1562,  2138,  108,   58,    2810,  591,   841,   837,   
+    841,   229,   581,   841,   837,   41,    73,    41,    73,    137,   
+    265,   133,   37,    229,   357,   841,   837,   73,    137,   265,   
+    233,   837,   73,    137,   169,   41,    233,   837,   841,   837,   
+    841,   837,   841,   837,   841,   837,   841,   837,   841,   901,   
+    809,   57,    805,   57,    197,   809,   57,    805,   57,    197,   
+    809,   57,    805,   57,    197,   809,   57,    805,   57,    197,   
+    809,   57,    805,   57,    197,   94,    1613,  135,   871,   71,    
+    39,    39,    327,   135,   39,    39,    39,    39,    39,    39,    
+    103,   71,    39,    39,    39,    39,    39,    39,    71,    39,    
+    135,   231,   135,   135,   39,    327,   551,   103,   167,   551,   
+    89,    1434,  3226,  506,   474,   506,   506,   367,   1018,  1946,  
+    1402,  954,   1402,  314,   90,    1082,  218,   2266,  666,   1210,  
+    186,   570,   2042,  58,    5850,  154,   2010,  154,   794,   2266,  
+    378,   2266,  3738,  39,    39,    39,    39,    39,    39,    17351, 
+    34,    3074,  7692,  63,    63,    
+  };
+
+int sqlite3Fts5UnicodeCategory(int iCode) { 
+  int iRes = -1;
+  int iHi;
+  int iLo;
+  int ret;
+  u16 iKey;
+
+  if( iCode>=(1<<20) ){
+    return 0;
+  }
+  iLo = aFts5UnicodeBlock[(iCode>>16)];
+  iHi = aFts5UnicodeBlock[1+(iCode>>16)];
+  iKey = (iCode & 0xFFFF);
+  while( iHi>iLo ){
+    int iTest = (iHi + iLo) / 2;
+    assert( iTest>=iLo && iTest<iHi );
+    if( iKey>=aFts5UnicodeMap[iTest] ){
+      iRes = iTest;
+      iLo = iTest+1;
+    }else{
+      iHi = iTest;
+    }
+  }
+
+  if( iRes<0 ) return 0;
+  if( iKey>=(aFts5UnicodeMap[iRes]+(aFts5UnicodeData[iRes]>>5)) ) return 0;
+  ret = aFts5UnicodeData[iRes] & 0x1F;
+  if( ret!=30 ) return ret;
+  return ((iKey - aFts5UnicodeMap[iRes]) & 0x01) ? 5 : 9;
+}
+
+void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){
+  int i = 0;
+  int iTbl = 0;
+  while( i<128 ){
+    int bToken = aArray[ aFts5UnicodeData[iTbl] & 0x1F ];
+    int n = (aFts5UnicodeData[iTbl] >> 5) + i;
+    for(; i<128 && i<n; i++){
+      aAscii[i] = (u8)bToken;
+    }
+    iTbl++;
+  }
+}
+

ext/fts5/fts5_vocab.c

@@ -29,6 +29,11 @@
 **   the number of fts5 rows that contain at least one instance of term
 **   $term. Field $cnt is set to the total number of instances of term 
 **   $term in the database.
+**
+** instance:
+**     CREATE TABLE vocab(term, doc, col, offset, PRIMARY KEY(<all-fields>));
+**
+**   One row for each term instance in the database. 
 */
 
 
@@ -44,7 +49,7 @@ struct Fts5VocabTable {
   char *zFts5Db;                  /* Db containing fts5 table */
   sqlite3 *db;                    /* Database handle */
   Fts5Global *pGlobal;            /* FTS5 global object for this database */
-  int eType;                      /* FTS5_VOCAB_COL or ROW */
+  int eType;                      /* FTS5_VOCAB_COL, ROW or INSTANCE */
 };
 
 struct Fts5VocabCursor {
@@ -64,16 +69,22 @@ struct Fts5VocabCursor {
   i64 *aCnt;
   i64 *aDoc;
 
-  /* Output values used by 'row' and 'col' tables */
+  /* Output values used by all tables. */
   i64 rowid;                      /* This table's current rowid value */
   Fts5Buffer term;                /* Current value of 'term' column */
+
+  /* Output values Used by 'instance' tables only */
+  i64 iInstPos;
+  int iInstOff;
 };
 
-#define FTS5_VOCAB_COL    0
+#define FTS5_VOCAB_COL      0
-#define FTS5_VOCAB_ROW    1
+#define FTS5_VOCAB_ROW      1
+#define FTS5_VOCAB_INSTANCE 2
 
 #define FTS5_VOCAB_COL_SCHEMA  "term, col, doc, cnt"
 #define FTS5_VOCAB_ROW_SCHEMA  "term, doc, cnt"
+#define FTS5_VOCAB_INST_SCHEMA "term, doc, col, offset"
 
 /*
 ** Bits for the mask used as the idxNum value by xBestIndex/xFilter.
@@ -101,6 +112,9 @@ static int fts5VocabTableType(const char *zType, char **pzErr, int *peType){
     if( sqlite3_stricmp(zCopy, "row")==0 ){
       *peType = FTS5_VOCAB_ROW;
     }else
+    if( sqlite3_stricmp(zCopy, "instance")==0 ){
+      *peType = FTS5_VOCAB_INSTANCE;
+    }else
     {
       *pzErr = sqlite3_mprintf("fts5vocab: unknown table type: %Q", zCopy);
       rc = SQLITE_ERROR;
@@ -161,7 +175,8 @@ static int fts5VocabInitVtab(
 ){
   const char *azSchema[] = { 
     "CREATE TABlE vocab(" FTS5_VOCAB_COL_SCHEMA  ")", 
-    "CREATE TABlE vocab(" FTS5_VOCAB_ROW_SCHEMA  ")"
+    "CREATE TABlE vocab(" FTS5_VOCAB_ROW_SCHEMA  ")",
+    "CREATE TABlE vocab(" FTS5_VOCAB_INST_SCHEMA ")"
   };
 
   Fts5VocabTable *pRet = 0;
@@ -235,6 +250,15 @@ static int fts5VocabCreateMethod(
 
 /* 
 ** Implementation of the xBestIndex method.
+**
+** Only constraints of the form:
+**
+**     term <= ?
+**     term == ?
+**     term >= ?
+**
+** are interpreted. Less-than and less-than-or-equal are treated 
+** identically, as are greater-than and greater-than-or-equal.
 */
 static int fts5VocabBestIndexMethod(
   sqlite3_vtab *pUnused,
@@ -378,6 +402,54 @@ static int fts5VocabCloseMethod(sqlite3_vtab_cursor *pCursor){
   return SQLITE_OK;
 }
 
+static int fts5VocabInstanceNewTerm(Fts5VocabCursor *pCsr){
+  int rc = SQLITE_OK;
+  
+  if( sqlite3Fts5IterEof(pCsr->pIter) ){
+    pCsr->bEof = 1;
+  }else{
+    const char *zTerm;
+    int nTerm;
+    zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
+    if( pCsr->nLeTerm>=0 ){
+      int nCmp = MIN(nTerm, pCsr->nLeTerm);
+      int bCmp = memcmp(pCsr->zLeTerm, zTerm, nCmp);
+      if( bCmp<0 || (bCmp==0 && pCsr->nLeTerm<nTerm) ){
+        pCsr->bEof = 1;
+      }
+    }
+
+    sqlite3Fts5BufferSet(&rc, &pCsr->term, nTerm, (const u8*)zTerm);
+  }
+  return rc;
+}
+
+static int fts5VocabInstanceNext(Fts5VocabCursor *pCsr){
+  int eDetail = pCsr->pConfig->eDetail;
+  int rc = SQLITE_OK;
+  Fts5IndexIter *pIter = pCsr->pIter;
+  i64 *pp = &pCsr->iInstPos;
+  int *po = &pCsr->iInstOff;
+  
+  while( eDetail==FTS5_DETAIL_NONE
+      || sqlite3Fts5PoslistNext64(pIter->pData, pIter->nData, po, pp) 
+  ){
+    pCsr->iInstPos = 0;
+    pCsr->iInstOff = 0;
+
+    rc = sqlite3Fts5IterNextScan(pCsr->pIter);
+    if( rc==SQLITE_OK ){
+      rc = fts5VocabInstanceNewTerm(pCsr);
+      if( eDetail==FTS5_DETAIL_NONE ) break;
+    }
+    if( rc ){
+      pCsr->bEof = 1;
+      break;
+    }
+  }
+
+  return rc;
+}
 
 /*
 ** Advance the cursor to the next row in the table.
@@ -390,13 +462,17 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
 
   pCsr->rowid++;
 
+  if( pTab->eType==FTS5_VOCAB_INSTANCE ){
+    return fts5VocabInstanceNext(pCsr);
+  }
+
   if( pTab->eType==FTS5_VOCAB_COL ){
     for(pCsr->iCol++; pCsr->iCol<nCol; pCsr->iCol++){
       if( pCsr->aDoc[pCsr->iCol] ) break;
     }
   }
 
-  if( pTab->eType==FTS5_VOCAB_ROW || pCsr->iCol>=nCol ){
+  if( pTab->eType!=FTS5_VOCAB_COL || pCsr->iCol>=nCol ){
     if( sqlite3Fts5IterEof(pCsr->pIter) ){
       pCsr->bEof = 1;
     }else{
@@ -420,22 +496,26 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
 
       assert( pTab->eType==FTS5_VOCAB_COL || pTab->eType==FTS5_VOCAB_ROW );
       while( rc==SQLITE_OK ){
+        int eDetail = pCsr->pConfig->eDetail;
         const u8 *pPos; int nPos;   /* Position list */
         i64 iPos = 0;               /* 64-bit position read from poslist */
         int iOff = 0;               /* Current offset within position list */
 
         pPos = pCsr->pIter->pData;
         nPos = pCsr->pIter->nData;
-        switch( pCsr->pConfig->eDetail ){
+
-          case FTS5_DETAIL_FULL:
+        switch( pTab->eType ){
-            pPos = pCsr->pIter->pData;
+          case FTS5_VOCAB_ROW:
-            nPos = pCsr->pIter->nData;
+            if( eDetail==FTS5_DETAIL_FULL ){
-            if( pTab->eType==FTS5_VOCAB_ROW ){
               while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
                 pCsr->aCnt[0]++;
               }
-              pCsr->aDoc[0]++;
+            }
-            }else{
+            pCsr->aDoc[0]++;
+            break;
+
+          case FTS5_VOCAB_COL:
+            if( eDetail==FTS5_DETAIL_FULL ){
               int iCol = -1;
               while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
                 int ii = FTS5_POS2COLUMN(iPos);
@@ -449,13 +529,7 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
                   iCol = ii;
                 }
               }
-            }
+            }else if( eDetail==FTS5_DETAIL_COLUMNS ){
-            break;
-
-          case FTS5_DETAIL_COLUMNS:
-            if( pTab->eType==FTS5_VOCAB_ROW ){
-              pCsr->aDoc[0]++;
-            }else{
               while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff,&iPos) ){
                 assert_nc( iPos>=0 && iPos<nCol );
                 if( iPos>=nCol ){
@@ -464,18 +538,21 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
                 }
                 pCsr->aDoc[iPos]++;
               }
+            }else{
+              assert( eDetail==FTS5_DETAIL_NONE );
+              pCsr->aDoc[0]++;
             }
             break;
 
-          default: 
+          default:
-            assert( pCsr->pConfig->eDetail==FTS5_DETAIL_NONE );
+            assert( pTab->eType==FTS5_VOCAB_INSTANCE );
-            pCsr->aDoc[0]++;
             break;
         }
 
         if( rc==SQLITE_OK ){
           rc = sqlite3Fts5IterNextScan(pCsr->pIter);
         }
+        if( pTab->eType==FTS5_VOCAB_INSTANCE ) break;
 
         if( rc==SQLITE_OK ){
           zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
@@ -505,7 +582,9 @@ static int fts5VocabFilterMethod(
   int nUnused,                    /* Number of elements in apVal */
   sqlite3_value **apVal           /* Arguments for the indexing scheme */
 ){
+  Fts5VocabTable *pTab = (Fts5VocabTable*)pCursor->pVtab;
   Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
+  int eType = pTab->eType;
   int rc = SQLITE_OK;
 
   int iVal = 0;
@@ -545,11 +624,16 @@ static int fts5VocabFilterMethod(
     }
   }
 
-
   if( rc==SQLITE_OK ){
     rc = sqlite3Fts5IndexQuery(pCsr->pIndex, zTerm, nTerm, f, 0, &pCsr->pIter);
   }
-  if( rc==SQLITE_OK ){
+  if( rc==SQLITE_OK && eType==FTS5_VOCAB_INSTANCE ){
+    rc = fts5VocabInstanceNewTerm(pCsr);
+  }
+  if( rc==SQLITE_OK 
+   && !pCsr->bEof 
+   && (eType!=FTS5_VOCAB_INSTANCE || pCsr->pConfig->eDetail!=FTS5_DETAIL_NONE)
+  ){
     rc = fts5VocabNextMethod(pCursor);
   }
 
@@ -591,13 +675,41 @@ static int fts5VocabColumnMethod(
     }else{
       iVal = pCsr->aCnt[pCsr->iCol];
     }
-  }else{
+  }else if( eType==FTS5_VOCAB_ROW ){
     assert( iCol==1 || iCol==2 );
     if( iCol==1 ){
       iVal = pCsr->aDoc[0];
     }else{
       iVal = pCsr->aCnt[0];
     }
+  }else{
+    assert( eType==FTS5_VOCAB_INSTANCE );
+    switch( iCol ){
+      case 1:
+        sqlite3_result_int64(pCtx, pCsr->pIter->iRowid);
+        break;
+      case 2: {
+        int ii = -1;
+        if( eDetail==FTS5_DETAIL_FULL ){
+          ii = FTS5_POS2COLUMN(pCsr->iInstPos);
+        }else if( eDetail==FTS5_DETAIL_COLUMNS ){
+          ii = (int)pCsr->iInstPos;
+        }
+        if( ii>=0 && ii<pCsr->pConfig->nCol ){
+          const char *z = pCsr->pConfig->azCol[ii];
+          sqlite3_result_text(pCtx, z, -1, SQLITE_STATIC);
+        }
+        break;
+      }
+      default: {
+        assert( iCol==3 );
+        if( eDetail==FTS5_DETAIL_FULL ){
+          int ii = FTS5_POS2OFFSET(pCsr->iInstPos);
+          sqlite3_result_int(pCtx, ii);
+        }
+        break;
+      }
+    }
   }
 
   if( iVal>0 ) sqlite3_result_int64(pCtx, iVal);

ext/fts5/fts5parse.y

@@ -148,7 +148,11 @@ cnearset(A) ::= colset(X) COLON nearset(Y). {
 %destructor nearset { sqlite3Fts5ParseNearsetFree($$); }
 %destructor nearphrases { sqlite3Fts5ParseNearsetFree($$); }
 
-nearset(A) ::= phrase(X). { A = sqlite3Fts5ParseNearset(pParse, 0, X); }
+nearset(A) ::= phrase(Y). { A = sqlite3Fts5ParseNearset(pParse, 0, Y); }
+nearset(A) ::= CARET phrase(Y). { 
+  sqlite3Fts5ParseSetCaret(Y);
+  A = sqlite3Fts5ParseNearset(pParse, 0, Y); 
+}
 nearset(A) ::= STRING(X) LP nearphrases(Y) neardist_opt(Z) RP. {
   sqlite3Fts5ParseNear(pParse, &X);
   sqlite3Fts5ParseSetDistance(pParse, Y, &Z);
@@ -189,6 +193,5 @@ phrase(A) ::= STRING(Y) star_opt(Z). {
 ** Optional "*" character.
 */
 %type star_opt {int}
-
 star_opt(A) ::= STAR. { A = 1; }
 star_opt(A) ::= . { A = 0; }

ext/fts5/test/fts5aa.test

@@ -591,7 +591,19 @@ do_execsql_test 22.1 {
   SELECT rowid FROM t9('a*')
 } {1}
 
+#-------------------------------------------------------------------------
+do_execsql_test 23.0 {
+  CREATE VIRTUAL TABLE t10 USING fts5(x, detail=%DETAIL%);
+  CREATE TABLE t11(x);
+}
+do_execsql_test 23.1 {
+  SELECT * FROM t11, t10 WHERE t11.x = t10.x AND t10.rowid IS NULL;
+}
+do_execsql_test 23.2 {
+  SELECT * FROM t11, t10 WHERE t10.rowid IS NULL;
 }
 
+}
 
+expand_all_sql db
 finish_test

ext/fts5/test/fts5af.test

@@ -175,6 +175,16 @@ do_execsql_test 5.1 {
   SELECT snippet(p1, 0, '[', ']', '...', 6) FROM p1('x');
 } {{[x] a a a a a...}}
 
+do_execsql_test 5.2 {
+  SELECT snippet(p1, 0, '[', ']', NULL, 6) FROM p1('x');
+} {{[x] a a a a a}}
+do_execsql_test 5.3 {
+  SELECT snippet(p1, 0, NULL, ']', '...', 6) FROM p1('x');
+} {{x] a a a a a...}}
+do_execsql_test 5.4 {
+  SELECT snippet(p1, 0, '[', NULL, '...', 6) FROM p1('x');
+} {{[x a a a a a...}}
+
 } ;# foreach_detail_mode 
 
 finish_test

ext/fts5/test/fts5cat.test

@@ -0,0 +1,59 @@
+# 2016 Jan 15
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+ifcapable !fts5 { finish_test ; return }
+set ::testprefix fts5cat
+
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize="unicode61 categories 'L*'");
+  INSERT INTO t1 VALUES ('Unlike1option2values3and4column5names');
+}
+
+do_execsql_test 1.1 {
+  SELECT rowid FROM t1('option');
+} {1}
+
+do_execsql_test 1.2 {
+  CREATE VIRTUAL TABLE t2 USING fts5(x);
+  CREATE VIRTUAL TABLE t2t USING fts5vocab(t2, row);
+
+  CREATE VIRTUAL TABLE t3 USING fts5(
+    x, tokenize="unicode61 categories 'L* N* Co Mn'"
+  );
+  CREATE VIRTUAL TABLE t3t USING fts5vocab(t3, row);
+
+  CREATE VIRTUAL TABLE t4 USING fts5(
+    x, tokenize="unicode61 categories 'L* N* Co M*'"
+  );
+  CREATE VIRTUAL TABLE t4t USING fts5vocab(t4, row);
+
+  INSERT INTO t2 VALUES ('สนามกีฬา');
+  INSERT INTO t3 VALUES ('สนามกีฬา');
+  INSERT INTO t4 VALUES ('สนามกีฬา');
+}
+
+do_execsql_test 1.3 {
+  SELECT * FROM t2t
+} {สนามก 1 1 ฬา 1 1}
+
+do_execsql_test 1.4 {
+  SELECT * FROM t3t
+} {สนามกีฬา 1 1}
+
+do_execsql_test 1.5 {
+  SELECT * FROM t4t
+} {สนามกีฬา 1 1}
+
+
+finish_test

ext/fts5/test/fts5connect.test

@@ -0,0 +1,247 @@
+# 2017 August 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+
+
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5connect
+
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+#-------------------------------------------------------------------------
+# The tests in this file test the outcome of a schema-reset happening 
+# within the xConnect() method of an FTS5 table. At one point this
+# was causing a problem in SQLite. Each test proceeds as follows:
+#
+#   1. Connection [db] opens the db and reads from some unrelated, non-FTS5
+#      table causing SQLite to load the db schema into memory.
+#
+#   2. Connection [db2] opens the db and modifies the db schema.
+#
+#   3. Connection [db] reads or writes an existing fts5 table. That the
+#      schema has been modified is detected inside the fts5 xConnect() 
+#      callback that is invoked by sqlite3_prepare(). 
+#
+#   4. Verify that the statement in 3 has worked. SQLite should detect
+#      that the schema has changed and successfully prepare the 
+#      statement against the new schema.
+#
+# Test plan:
+#
+#   1.*: Trigger the xConnect()/schema-reset using statements executed
+#        directly against an FTS5 table.
+#
+#   2.*: Using various statements executed by various BEFORE triggers.
+#
+#   3.*: Using various statements executed by various AFTER triggers.
+#
+#   4.*: Using various statements executed by various INSTEAD OF triggers.
+#
+
+
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE ft1 USING fts5(a, b);
+  CREATE TABLE abc(x INTEGER PRIMARY KEY);
+  CREATE TABLE t1(i INTEGER PRIMARY KEY, a, b);
+
+  INSERT INTO ft1 VALUES('one', 'two');
+  INSERT INTO ft1 VALUES('three', 'four');
+}
+
+foreach {tn sql res} {
+  1 "SELECT * FROM ft1" {one two three four}
+  2 "REPLACE INTO ft1(rowid, a, b) VALUES(1, 'five', 'six')" {}
+  3 "SELECT * FROM ft1" {five six three four}
+  4 "INSERT INTO ft1 VALUES('seven', 'eight')" {}
+  5 "SELECT * FROM ft1" {five six three four seven eight}
+  6 "DELETE FROM ft1 WHERE rowid=2" {}
+  7 "UPDATE ft1 SET b='nine' WHERE rowid=1" {}
+  8 "SELECT * FROM ft1" {five nine seven eight}
+} {
+
+  catch { db close }
+  catch { db2 close }
+  sqlite3 db  test.db
+  sqlite3 db2 test.db
+
+  do_test 1.$tn.1 {
+    db eval { INSERT INTO abc DEFAULT VALUES }
+    db2 eval { CREATE TABLE newtable(x,y); DROP TABLE newtable }
+  } {}
+
+  do_execsql_test 1.$tn.2 $sql $res
+
+  do_execsql_test 1.$tn.3 {
+    INSERT INTO ft1(ft1) VALUES('integrity-check');
+  }
+}
+
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE ft2 USING fts5(a, b);
+  CREATE TABLE t2(a, b);
+  CREATE TABLE log(txt);
+
+  CREATE TRIGGER t2_ai AFTER INSERT ON t2 BEGIN
+    INSERT INTO ft2(rowid, a, b) VALUES(new.rowid, new.a, new.b);
+    INSERT INTO log VALUES('insert');
+  END;
+
+  CREATE TRIGGER t2_ad AFTER DELETE ON t2 BEGIN
+    DELETE FROM ft2 WHERE rowid = old.rowid;
+    INSERT INTO log VALUES('delete');
+  END;
+
+  CREATE TRIGGER t2_au AFTER UPDATE ON t2 BEGIN
+    UPDATE ft2 SET a=new.a, b=new.b WHERE rowid=new.rowid;
+    INSERT INTO log VALUES('update');
+  END;
+
+  INSERT INTO t2 VALUES('one', 'two');
+  INSERT INTO t2 VALUES('three', 'four');
+}
+
+foreach {tn sql res} {
+  1 "SELECT * FROM t2" {one two three four}
+  2 "REPLACE INTO t2(rowid, a, b) VALUES(1, 'five', 'six')" {}
+  3 "SELECT * FROM ft2" {five six three four}
+  4 "INSERT INTO t2 VALUES('seven', 'eight')" {}
+  5 "SELECT * FROM ft2" {five six three four seven eight}
+  6 "DELETE FROM t2 WHERE rowid=2" {}
+  7 "UPDATE t2 SET b='nine' WHERE rowid=1" {}
+  8 "SELECT * FROM ft2" {five nine seven eight}
+} {
+
+  catch { db close }
+  catch { db2 close }
+  sqlite3 db  test.db
+  sqlite3 db2 test.db
+
+  do_test 2.$tn.1 {
+    db eval { INSERT INTO abc DEFAULT VALUES }
+    db2 eval { CREATE TABLE newtable(x,y); DROP TABLE newtable }
+  } {}
+
+  do_execsql_test 2.$tn.2 $sql $res
+
+  do_execsql_test 2.$tn.3 {
+    INSERT INTO ft2(ft2) VALUES('integrity-check');
+  }
+}
+
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE ft3 USING fts5(a, b);
+  CREATE TABLE t3(a, b);
+
+  CREATE TRIGGER t3_ai BEFORE INSERT ON t3 BEGIN
+    INSERT INTO ft3(rowid, a, b) VALUES(new.rowid, new.a, new.b);
+    INSERT INTO log VALUES('insert');
+  END;
+
+  CREATE TRIGGER t3_ad BEFORE DELETE ON t3 BEGIN
+    DELETE FROM ft3 WHERE rowid = old.rowid;
+    INSERT INTO log VALUES('delete');
+  END;
+
+  CREATE TRIGGER t3_au BEFORE UPDATE ON t3 BEGIN
+    UPDATE ft3 SET a=new.a, b=new.b WHERE rowid=new.rowid;
+    INSERT INTO log VALUES('update');
+  END;
+
+  INSERT INTO t3(rowid, a, b) VALUES(1, 'one', 'two');
+  INSERT INTO t3(rowid, a, b) VALUES(2, 'three', 'four');
+}
+
+foreach {tn sql res} {
+  1 "SELECT * FROM t3" {one two three four}
+  2 "REPLACE INTO t3(rowid, a, b) VALUES(1, 'five', 'six')" {}
+  3 "SELECT * FROM ft3" {five six three four}
+  4 "INSERT INTO t3(rowid, a, b) VALUES(3, 'seven', 'eight')" {}
+  5 "SELECT * FROM ft3" {five six three four seven eight}
+  6 "DELETE FROM t3 WHERE rowid=2" {}
+  7 "UPDATE t3 SET b='nine' WHERE rowid=1" {}
+  8 "SELECT * FROM ft3" {five nine seven eight}
+} {
+
+  catch { db close }
+  catch { db2 close }
+  sqlite3 db  test.db
+  sqlite3 db2 test.db
+
+  do_test 3.$tn.1 {
+    db eval { INSERT INTO abc DEFAULT VALUES }
+    db2 eval { CREATE TABLE newtable(x,y); DROP TABLE newtable }
+  } {}
+
+  do_execsql_test 3.$tn.2 $sql $res
+
+  do_execsql_test 3.$tn.3 {
+    INSERT INTO ft3(ft3) VALUES('integrity-check');
+  }
+}
+
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE ft4 USING fts5(a, b);
+  CREATE VIEW v4 AS SELECT rowid, * FROM ft4;
+
+  CREATE TRIGGER t4_ai INSTEAD OF INSERT ON v4 BEGIN
+    INSERT INTO ft4(rowid, a, b) VALUES(new.rowid, new.a, new.b);
+    INSERT INTO log VALUES('insert');
+  END;
+
+  CREATE TRIGGER t4_ad INSTEAD OF DELETE ON v4 BEGIN
+    DELETE FROM ft4 WHERE rowid = old.rowid;
+    INSERT INTO log VALUES('delete');
+  END;
+
+  CREATE TRIGGER t4_au INSTEAD OF UPDATE ON v4 BEGIN
+    UPDATE ft4 SET a=new.a, b=new.b WHERE rowid=new.rowid;
+    INSERT INTO log VALUES('update');
+  END;
+
+  INSERT INTO ft4(rowid, a, b) VALUES(1, 'one', 'two');
+  INSERT INTO ft4(rowid, a, b) VALUES(2, 'three', 'four');
+}
+
+foreach {tn sql res} {
+  1 "SELECT * FROM ft4" {one two three four}
+  2 "REPLACE INTO v4(rowid, a, b) VALUES(1, 'five', 'six')" {}
+  3 "SELECT * FROM ft4" {five six three four}
+  4 "INSERT INTO v4(rowid, a, b) VALUES(3, 'seven', 'eight')" {}
+  5 "SELECT * FROM ft4" {five six three four seven eight}
+  6 "DELETE FROM v4 WHERE rowid=2" {}
+  7 "UPDATE v4 SET b='nine' WHERE rowid=1" {}
+  8 "SELECT * FROM ft4" {five nine seven eight}
+} {
+
+  catch { db close }
+  catch { db2 close }
+  sqlite3 db  test.db
+  sqlite3 db2 test.db
+
+  do_test 4.$tn.1 {
+    db eval { INSERT INTO abc DEFAULT VALUES }
+    db2 eval { CREATE TABLE newtable(x,y); DROP TABLE newtable }
+  } {}
+
+  do_execsql_test 4.$tn.2 $sql $res
+
+  do_execsql_test 4.$tn.3 {
+    INSERT INTO ft3(ft3) VALUES('integrity-check');
+  }
+}
+
+finish_test
+

ext/fts5/test/fts5fault6.test

@@ -253,7 +253,7 @@ do_faultsim_test 5.2 -faults oom* -prep {
     SELECT rowid, mit(matchinfo(t1, 'x')) FROM t1 WHERE t1 MATCH 'a AND c'
   }
 } -test {
-  faultsim_test_result [list 0 $::res]
+  faultsim_test_result [list 0 $::res] {1 {SQL logic error}}
 }
 
 do_faultsim_test 5.3 -faults oom* -prep {
@@ -264,7 +264,7 @@ do_faultsim_test 5.3 -faults oom* -prep {
     SELECT count(*) FROM t1 WHERE t1 MATCH 'd AND e AND f'
   }
 } -test {
-  faultsim_test_result {0 29}
+  faultsim_test_result {0 29} {1 {SQL logic error}}
 }
 
 do_faultsim_test 5.4 -faults oom* -prep {
@@ -275,7 +275,7 @@ do_faultsim_test 5.4 -faults oom* -prep {
     SELECT count(*) FROM t1 WHERE t1 MATCH 'x + e'
   }
 } -test {
-  faultsim_test_result {0 1}
+  faultsim_test_result {0 1} {1 {SQL logic error}}
 }
 
 #-------------------------------------------------------------------------

ext/fts5/test/fts5fault9.test

@@ -24,6 +24,8 @@ ifcapable !fts5 {
 
 foreach_detail_mode $testprefix {
 
+if {"%DETAIL%" != "none"} continue
+
 fts5_aux_test_functions db
 
 do_execsql_test 1.0 {
@@ -98,14 +100,16 @@ do_faultsim_test 4.1 -faults oom-t* -body {
   execsql { SELECT rowid, fts5_test_collist(t4) FROM t4('2') }
 } -test {
   faultsim_test_result \
-      {0 {1 {0.0 0.1 0.2} 2 {0.0 0.1 0.2} 3 {0.0 0.1 0.2}}} {1 SQLITE_NOMEM}
+      {0 {1 {0.0 0.1 0.2} 2 {0.0 0.1 0.2} 3 {0.0 0.1 0.2}}} \
+      {1 SQLITE_NOMEM} {1 SQLITE_ERROR} {1 {SQL logic error}}
 }
 
 do_faultsim_test 4.2 -faults oom-t* -body {
   execsql { SELECT rowid, fts5_test_collist(t4) FROM t4('a5 OR b5 OR c5') }
 } -test {
   faultsim_test_result \
-      {0 {4 {0.0 0.1 0.2} 5 {1.0 1.1 1.2} 6 {2.0 2.1 2.2}}} {1 SQLITE_NOMEM}
+      {0 {4 {0.0 0.1 0.2} 5 {1.0 1.1 1.2} 6 {2.0 2.1 2.2}}} \
+      {1 SQLITE_NOMEM} {1 SQLITE_ERROR} {1 {SQL logic error}}
 }
 
 

ext/fts5/test/fts5faultB.test

@@ -130,5 +130,22 @@ do_faultsim_test 4.2 -faults oom* -body {
   faultsim_test_result {0 {2 3}}
 }
 
+#-------------------------------------------------------------------------
+# Test OOM injection while parsing a CARET expression
+#
+reset_db
+do_execsql_test 5.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a);
+  INSERT INTO t1 VALUES('a b c d');  -- 1
+  INSERT INTO t1 VALUES('d a b c');  -- 2
+  INSERT INTO t1 VALUES('c d a b');  -- 3
+  INSERT INTO t1 VALUES('b c d a');  -- 4
+}
+do_faultsim_test 5.1 -faults oom* -body {
+  execsql { SELECT rowid FROM t1('^a OR ^b') }
+} -test {
+  faultsim_test_result {0 {1 4}}
+}
+
 
 finish_test

ext/fts5/test/fts5first.test

@@ -0,0 +1,96 @@
+# 2017 November 25
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5first
+
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(a, b);
+}
+
+foreach {tn expr ok} {
+  1 {^abc}           1
+  2 {^abc + def}     1
+  3 {^ "abc def"}    1
+  4 {^"abc def"}     1
+  5 {abc ^def}       1
+  6 {abc + ^def}     0
+  7 {abc ^+ def}     0
+  8 {"^abc"}         1
+  9 {NEAR(^abc def)} 0
+} {
+  set res(0) {/1 {fts5: syntax error near .*}/}
+  set res(1) {0 {}}
+
+  do_catchsql_test 1.$tn { SELECT * FROM x1($expr) } $res($ok)
+}
+
+#-------------------------------------------------------------------------
+# 
+do_execsql_test 2.0 {
+  INSERT INTO x1 VALUES('a b c', 'b c a');
+}
+
+foreach {tn expr match} {
+  1 {^a} 1
+  2 {^b} 1
+  3 {^c} 0
+  4 {^a + b} 1
+  5 {^b + c} 1
+  6 {^c + a} 0
+  7 {^"c a"} 0
+  8 {a:^a} 1
+  9 {a:^b} 0
+  10 {a:^"a b"} 1
+} {
+  do_execsql_test 2.$tn { SELECT EXISTS (SELECT rowid FROM x1($expr)) } $match
+}
+
+#-------------------------------------------------------------------------
+# 
+do_execsql_test 3.0 {
+  DELETE FROM x1;
+  INSERT INTO x1 VALUES('b a', 'c a');
+  INSERT INTO x1 VALUES('a a', 'c c');
+  INSERT INTO x1 VALUES('a b', 'a a');
+}
+fts5_aux_test_functions db
+
+foreach {tn expr expect} {
+  1 {^a} {{2 1}}
+  2 {^c AND ^b} {{0 2} {1 0}}
+} {
+  do_execsql_test 3.$tn {
+    SELECT fts5_test_queryphrase(x1) FROM x1($expr) LIMIT 1
+  } [list $expect]
+}
+
+#-------------------------------------------------------------------------
+# 
+do_execsql_test 3.1 {
+  CREATE VIRTUAL TABLE x2 USING fts5(a, b, c, detail=column);
+}
+
+do_catchsql_test 3.2 {
+  SELECT * FROM x2('a + b');
+} {1 {fts5: phrase queries are not supported (detail!=full)}}
+
+do_catchsql_test 3.3 {
+  SELECT * FROM x2('^a');
+} {1 {fts5: phrase queries are not supported (detail!=full)}}
+finish_test
+

ext/fts5/test/fts5plan.test

@@ -29,38 +29,37 @@ do_execsql_test 1.0 {
 do_eqp_test 1.1 {
   SELECT * FROM t1, f1 WHERE f1 MATCH t1.x
 } {
-  0 0 0 {SCAN TABLE t1} 
+  QUERY PLAN
-  0 1 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 65537:}
+  |--SCAN TABLE t1
+  `--SCAN TABLE f1 VIRTUAL TABLE INDEX 65537:
 }
 
 do_eqp_test 1.2 {
   SELECT * FROM t1, f1 WHERE f1 > t1.x
 } {
-  0 0 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 0:}
+  QUERY PLAN
-  0 1 0 {SCAN TABLE t1} 
+  |--SCAN TABLE f1 VIRTUAL TABLE INDEX 0:
+  `--SCAN TABLE t1
 }
 
 do_eqp_test 1.3 {
   SELECT * FROM f1 WHERE f1 MATCH ? ORDER BY ff
 } {
-  0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 65537:}
+  QUERY PLAN
-  0 0 0 {USE TEMP B-TREE FOR ORDER BY}
+  |--SCAN TABLE f1 VIRTUAL TABLE INDEX 65537:
+  `--USE TEMP B-TREE FOR ORDER BY
 }
 
 do_eqp_test 1.4 {
   SELECT * FROM f1 ORDER BY rank
 } {
-  0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 0:}
+  QUERY PLAN
-  0 0 0 {USE TEMP B-TREE FOR ORDER BY}
+  |--SCAN TABLE f1 VIRTUAL TABLE INDEX 0:
+  `--USE TEMP B-TREE FOR ORDER BY
 }
 
 do_eqp_test 1.5 {
   SELECT * FROM f1 WHERE rank MATCH ?
-} {
+} {SCAN TABLE f1 VIRTUAL TABLE INDEX 2:}
-  0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 2:}
-}
-
-
-
 
 finish_test

ext/fts5/test/fts5query.test

@@ -64,7 +64,7 @@ for {set tn 1 ; set pgsz 64} {$tn<32} {incr tn; incr pgsz 16} {
     execsql COMMIT
   } {}
 
-  do_execsql_test 1.$tn.2 {
+  do_execsql_test 2.$tn.2 {
     INSERT INTO t1(t1) VALUES('integrity-check');
   }
 
@@ -77,5 +77,15 @@ for {set tn 1 ; set pgsz 64} {$tn<32} {incr tn; incr pgsz 16} {
   }
 }
 
+reset_db
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(a);
+  INSERT INTO x1(rowid, a) VALUES(-1000000000000, 'toyota');
+  INSERT INTO x1(rowid, a) VALUES(1, 'tarago');
+}
+do_execsql_test 3.1 {
+  SELECT rowid FROM x1('t*');
+} {-1000000000000 1}
+
 
 finish_test

ext/fts5/test/fts5rank.test

@@ -148,7 +148,19 @@ do_execsql_test 4.1 {
   VTest MATCH 'wrinkle in time OR a wrinkle in time' ORDER BY rank;
 } {{wrinkle in time} {Bill Smith}}
 
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 5.0 {
+  CREATE VIRTUAL TABLE ttt USING fts5(a);
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100
+  )
+  INSERT INTO ttt SELECT 'word ' || i FROM s;
+}
 
-
+do_execsql_test 5.1 {
+  SELECT rowid FROM ttt('word') WHERE rowid BETWEEN 30 AND 40 ORDER BY rank;
+} {30 31 32 33 34 35 36 37 38 39 40}
 
 finish_test
+

ext/fts5/test/fts5unicode.test

@@ -41,7 +41,6 @@ foreach {tn t} {1 ascii 2 unicode61} {
 #-------------------------------------------------------------------------
 # Check that "unicode61" really is the default tokenizer.
 #
-
 do_execsql_test 2.0 "
   CREATE VIRTUAL TABLE t1 USING fts5(x);
   CREATE VIRTUAL TABLE t2 USING fts5(x, tokenize = unicode61);
@@ -56,5 +55,31 @@ do_execsql_test 2.1 "
   SELECT 't3' FROM t3 WHERE t3 MATCH '\xE0\xE8\xEC';
 " {t1 t2}
 
+#-------------------------------------------------------------------------
+# Check that codepoints that require 4 bytes to store in utf-8 (those that
+# require 17 or more bits to store).
+#
+
+set A [db one {SELECT char(0x1F75E)}]    ;# Type So
+set B [db one {SELECT char(0x1F5FD)}]    ;# Type So
+set C [db one {SELECT char(0x2F802)}]    ;# Type Lo
+set D [db one {SELECT char(0x2F808)}]    ;# Type Lo
+
+do_execsql_test 3.0 "
+  CREATE VIRTUAL TABLE xyz USING fts5(x,
+    tokenize = \"unicode61 separators '$C' tokenchars '$A'\"
+  );
+  CREATE VIRTUAL TABLE xyz_v USING fts5vocab(xyz, row);
+
+  INSERT INTO xyz VALUES('$A$B$C$D');
+"
+
+do_execsql_test 3.1 {
+  SELECT * FROM xyz_v;
+} [list $A 1 1 $D 1 1]
+  
+
+
+
 
 finish_test

ext/fts5/test/fts5unicode4.test

@@ -0,0 +1,31 @@
+# 2018 July 25
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5unicode4
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE sss USING fts5(a, prefix=3); 
+}
+
+do_execsql_test 1.1 {
+  INSERT INTO sss VALUES('まりや');
+}
+
+finish_test

ext/fts5/test/fts5vocab2.test

@@ -0,0 +1,209 @@
+# 2017 August 10
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# The tests in this file focus on testing the fts5vocab module.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5vocab
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b);
+  CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, instance);
+
+  INSERT INTO t1 VALUES('one two', 'two three');
+  INSERT INTO t1 VALUES('three four', 'four five five five');
+}
+
+do_execsql_test 1.1 {
+  SELECT * FROM v1;
+} {
+  five  2 b 1
+  five  2 b 2
+  five  2 b 3
+  four  2 a 1
+  four  2 b 0
+  one   1 a 0
+  three 1 b 1
+  three 2 a 0
+  two   1 a 1
+  two   1 b 0
+}
+
+do_execsql_test 1.2 {
+  SELECT * FROM v1 WHERE term='three';
+} {
+  three 1 b 1
+  three 2 a 0
+}
+
+do_execsql_test 1.3 {
+  BEGIN;
+    DELETE FROM t1 WHERE rowid=2;
+    SELECT * FROM v1;
+  ROLLBACK;
+} {
+  one   1 a 0
+  three 1 b 1
+  two   1 a 1
+  two   1 b 0
+}
+
+do_execsql_test 1.4 {
+  BEGIN;
+    DELETE FROM t1 WHERE rowid=1;
+    SELECT * FROM v1;
+  ROLLBACK;
+} {
+  five  2 b 1
+  five  2 b 2
+  five  2 b 3
+  four  2 a 1
+  four  2 b 0
+  three 2 a 0
+}
+
+do_execsql_test 1.5 {
+  DELETE FROM t1;
+  SELECT * FROM v1;
+} {
+}
+
+#-------------------------------------------------------------------------
+#
+do_execsql_test 2.0 {
+  DROP TABLE IF EXISTS t1;
+  DROP TABLE IF EXISTS v1;
+
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=column);
+  CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, instance);
+
+  INSERT INTO t1 VALUES('one two', 'two three');
+  INSERT INTO t1 VALUES('three four', 'four five five five');
+}
+
+do_execsql_test 2.1 {
+  SELECT * FROM v1;
+} {
+  five  2 b {}
+  four  2 a {}
+  four  2 b {}
+  one   1 a {}
+  three 1 b {}
+  three 2 a {}
+  two   1 a {}
+  two   1 b {}
+}
+
+do_execsql_test 2.2 {
+  SELECT * FROM v1 WHERE term='three';
+} {
+  three 1 b {}
+  three 2 a {}
+}
+
+do_execsql_test 2.3 {
+  BEGIN;
+    DELETE FROM t1 WHERE rowid=2;
+    SELECT * FROM v1;
+  ROLLBACK;
+} {
+  one   1 a {}
+  three 1 b {}
+  two   1 a {}
+  two   1 b {}
+}
+
+do_execsql_test 2.4 {
+  BEGIN;
+    DELETE FROM t1 WHERE rowid=1;
+    SELECT * FROM v1;
+  ROLLBACK;
+} {
+  five  2 b {}
+  four  2 a {}
+  four  2 b {}
+  three 2 a {}
+}
+
+do_execsql_test 2.5 {
+  DELETE FROM t1;
+  SELECT * FROM v1;
+} {
+}
+
+#-------------------------------------------------------------------------
+#
+do_execsql_test 3.0 {
+  DROP TABLE IF EXISTS t1;
+  DROP TABLE IF EXISTS v1;
+
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=none);
+  CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, instance);
+
+  INSERT INTO t1 VALUES('one two', 'two three');
+  INSERT INTO t1 VALUES('three four', 'four five five five');
+}
+
+do_execsql_test 3.1 {
+  SELECT * FROM v1;
+} {
+  five  2 {} {}
+  four  2 {} {}
+  one   1 {} {}
+  three 1 {} {}
+  three 2 {} {}
+  two   1 {} {}
+}
+
+do_execsql_test 3.2 {
+  SELECT * FROM v1 WHERE term='three';
+} {
+  three 1 {} {}
+  three 2 {} {}
+}
+
+do_execsql_test 3.3 {
+  BEGIN;
+    DELETE FROM t1 WHERE rowid=2;
+    SELECT * FROM v1;
+  ROLLBACK;
+} {
+  one   1 {} {}
+  three 1 {} {}
+  two   1 {} {}
+}
+
+do_execsql_test 3.4 {
+  BEGIN;
+    DELETE FROM t1 WHERE rowid=1;
+    SELECT * FROM v1;
+  ROLLBACK;
+} {
+  five  2 {} {}
+  four  2 {} {}
+  three 2 {} {}
+}
+
+do_execsql_test 3.5 {
+  DELETE FROM t1;
+  SELECT * FROM v1;
+} {
+}
+
+finish_test
+

ext/icu/README.txt

@@ -39,8 +39,8 @@ SQLite. Documentation follows.
     To utilise "general" case mapping, the upper() or lower() scalar 
     functions are invoked with one argument:
 
-        upper('ABC') -> 'abc'
+        upper('abc') -> 'ABC'
-        lower('abc') -> 'ABC'
+        lower('ABC') -> 'abc'
 
     To access ICU "language specific" case mapping, upper() or lower()
     should be invoked with two arguments. The second argument is the name

ext/icu/icu.c

@@ -28,7 +28,9 @@
 **     provide case-independent matching.
 */
 
-#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU)
+#if !defined(SQLITE_CORE)                  \
+ || defined(SQLITE_ENABLE_ICU)             \
+ || defined(SQLITE_ENABLE_ICU_COLLATIONS)
 
 /* Include ICU headers */
 #include <unicode/utypes.h>
@@ -45,6 +47,26 @@
   #include "sqlite3.h"
 #endif
 
+/*
+** This function is called when an ICU function called from within
+** the implementation of an SQL scalar function returns an error.
+**
+** The scalar function context passed as the first argument is 
+** loaded with an error message based on the following two args.
+*/
+static void icuFunctionError(
+  sqlite3_context *pCtx,       /* SQLite scalar function context */
+  const char *zName,           /* Name of ICU function that failed */
+  UErrorCode e                 /* Error code returned by ICU function */
+){
+  char zBuf[128];
+  sqlite3_snprintf(128, zBuf, "ICU error: %s(): %s", zName, u_errorName(e));
+  zBuf[127] = '\0';
+  sqlite3_result_error(pCtx, zBuf, -1);
+}
+
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU)
+
 /*
 ** Maximum length (in bytes) of the pattern in a LIKE or GLOB
 ** operator.
@@ -102,15 +124,15 @@ static int icuLikeCompare(
   const uint8_t *zString,    /* The UTF-8 string to compare against */
   const UChar32 uEsc         /* The escape character */
 ){
-  static const int MATCH_ONE = (UChar32)'_';
+  static const uint32_t MATCH_ONE = (uint32_t)'_';
-  static const int MATCH_ALL = (UChar32)'%';
+  static const uint32_t MATCH_ALL = (uint32_t)'%';
 
   int prevEscape = 0;     /* True if the previous character was uEsc */
 
   while( 1 ){
 
     /* Read (and consume) the next character from the input pattern. */
-    UChar32 uPattern;
+    uint32_t uPattern;
     SQLITE_ICU_READ_UTF8(zPattern, uPattern);
     if( uPattern==0 ) break;
 
@@ -152,16 +174,16 @@ static int icuLikeCompare(
       if( *zString==0 ) return 0;
       SQLITE_ICU_SKIP_UTF8(zString);
 
-    }else if( !prevEscape && uPattern==uEsc){
+    }else if( !prevEscape && uPattern==(uint32_t)uEsc){
       /* Case 3. */
       prevEscape = 1;
 
     }else{
       /* Case 4. */
-      UChar32 uString;
+      uint32_t uString;
       SQLITE_ICU_READ_UTF8(zString, uString);
-      uString = u_foldCase(uString, U_FOLD_CASE_DEFAULT);
+      uString = (uint32_t)u_foldCase((UChar32)uString, U_FOLD_CASE_DEFAULT);
-      uPattern = u_foldCase(uPattern, U_FOLD_CASE_DEFAULT);
+      uPattern = (uint32_t)u_foldCase((UChar32)uPattern, U_FOLD_CASE_DEFAULT);
       if( uString!=uPattern ){
         return 0;
       }
@@ -224,24 +246,6 @@ static void icuLikeFunc(
   }
 }
 
-/*
-** This function is called when an ICU function called from within
-** the implementation of an SQL scalar function returns an error.
-**
-** The scalar function context passed as the first argument is 
-** loaded with an error message based on the following two args.
-*/
-static void icuFunctionError(
-  sqlite3_context *pCtx,       /* SQLite scalar function context */
-  const char *zName,           /* Name of ICU function that failed */
-  UErrorCode e                 /* Error code returned by ICU function */
-){
-  char zBuf[128];
-  sqlite3_snprintf(128, zBuf, "ICU error: %s(): %s", zName, u_errorName(e));
-  zBuf[127] = '\0';
-  sqlite3_result_error(pCtx, zBuf, -1);
-}
-
 /*
 ** Function to delete compiled regexp objects. Registered as
 ** a destructor function with sqlite3_set_auxdata().
@@ -407,6 +411,8 @@ static void icuCaseFunc16(sqlite3_context *p, int nArg, sqlite3_value **apArg){
   assert( 0 );     /* Unreachable */
 }
 
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) */
+
 /*
 ** Collation sequence destructor function. The pCtx argument points to
 ** a UCollator structure previously allocated using ucol_open().
@@ -501,6 +507,7 @@ int sqlite3IcuInit(sqlite3 *db){
     void (*xFunc)(sqlite3_context*,int,sqlite3_value**);
   } scalars[] = {
     {"icu_load_collation",  2, SQLITE_UTF8,                1, icuLoadCollation},
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU)
     {"regexp", 2, SQLITE_ANY|SQLITE_DETERMINISTIC,         0, icuRegexpFunc},
     {"lower",  1, SQLITE_UTF16|SQLITE_DETERMINISTIC,       0, icuCaseFunc16},
     {"lower",  2, SQLITE_UTF16|SQLITE_DETERMINISTIC,       0, icuCaseFunc16},
@@ -512,10 +519,10 @@ int sqlite3IcuInit(sqlite3 *db){
     {"upper",  2, SQLITE_UTF8|SQLITE_DETERMINISTIC,        1, icuCaseFunc16},
     {"like",   2, SQLITE_UTF8|SQLITE_DETERMINISTIC,        0, icuLikeFunc},
     {"like",   3, SQLITE_UTF8|SQLITE_DETERMINISTIC,        0, icuLikeFunc},
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) */
   };
   int rc = SQLITE_OK;
   int i;
-
   
   for(i=0; rc==SQLITE_OK && i<(int)(sizeof(scalars)/sizeof(scalars[0])); i++){
     const struct IcuScalar *p = &scalars[i];

ext/lsm1/Makefile

@@ -43,7 +43,7 @@ LSMTESTSRC = $(LSMDIR)/lsm-test/lsmtest1.c $(LSMDIR)/lsm-test/lsmtest2.c     \
 
 # all: lsm.so
 
-LSMOPTS += -DLSM_MUTEX_PTHREADS=1 -I$(LSMDIR)
+LSMOPTS += -DLSM_MUTEX_PTHREADS=1 -I$(LSMDIR) -DHAVE_ZLIB
 
 lsm.so:	$(LSMOBJ)
 	$(TCCX) -shared -o lsm.so $(LSMOBJ)
@@ -53,4 +53,4 @@ lsm.so:	$(LSMOBJ)
 	
 lsmtest$(EXE): $(LSMOBJ) $(LSMTESTSRC) $(LSMTESTHDR) sqlite3.o
 	# $(TCPPX) -c $(TOP)/lsm-test/lsmtest_tdb2.cc
-	$(TCCX) $(LSMOPTS) $(LSMTESTSRC) $(LSMOBJ) sqlite3.o -o lsmtest$(EXE) $(THREADLIB)
+	$(TCCX) $(LSMOPTS) $(LSMTESTSRC) $(LSMOBJ) sqlite3.o -o lsmtest$(EXE) $(THREADLIB) -lz

ext/lsm1/lsm-test/lsmtest.h

@@ -121,6 +121,7 @@ int test_mdb_scan(TestDb *, void *, int, void *, int, void *, int,
 */
 int test_lsm_open(const char*, const char *zFile, int bClear, TestDb **ppDb);
 int test_lsm_lomem_open(const char*, const char*, int bClear, TestDb **ppDb);
+int test_lsm_lomem2_open(const char*, const char*, int bClear, TestDb **ppDb);
 int test_lsm_zip_open(const char*, const char*, int bClear, TestDb **ppDb);
 int test_lsm_small_open(const char*, const char*, int bClear, TestDb **ppDb);
 int test_lsm_mt2(const char*, const char *zFile, int bClear, TestDb **ppDb);

ext/lsm1/lsm-test/lsmtest1.c

@@ -274,6 +274,7 @@ static void doDataTest1(
   int rc = LSM_OK;
   Datasource *pData;
   TestDb *pDb;
+  int iToggle = 0;
 
   /* Start the test case, open a database and allocate the datasource. */
   pDb = testOpen(zSystem, 1, &rc);
@@ -287,8 +288,11 @@ static void doDataTest1(
     testWriteDatasourceRange(pDb, pData, i, p->nVerify, &rc);
     i += p->nVerify;
 
+    if( iToggle ) testBegin(pDb, 1, &rc);
     /* Check that the db content is correct. */
     testDbContents(pDb, pData, p->nRow, 0, i-1, p->nTest, p->bTestScan, &rc);
+    if( iToggle ) testCommit(pDb, 0, &rc);
+    iToggle = (iToggle+1)%2;
 
     if( bRecover ){
       testReopenRecover(&pDb, &rc);

ext/lsm1/lsm-test/lsmtest_tdb.c

@@ -721,6 +721,7 @@ static struct Lib {
   { "sqlite3",      "testdb.sqlite",    sql_open },
   { "lsm_small",    "testdb.lsm_small", test_lsm_small_open },
   { "lsm_lomem",    "testdb.lsm_lomem", test_lsm_lomem_open },
+  { "lsm_lomem2",   "testdb.lsm_lomem2", test_lsm_lomem2_open },
 #ifdef HAVE_ZLIB
   { "lsm_zip",      "testdb.lsm_zip",   test_lsm_zip_open },
 #endif

ext/lsm1/lsm-test/lsmtest_tdb3.c

@@ -617,8 +617,12 @@ static int test_lsm_fetch(
 
   if( pKey==0 ) return LSM_OK;
 
-  rc = lsm_csr_open(pDb->db, &csr);
+  if( pDb->pCsr==0 ){
-  if( rc!=LSM_OK ) return rc;
+    rc = lsm_csr_open(pDb->db, &csr);
+    if( rc!=LSM_OK ) return rc;
+  }else{
+    csr = pDb->pCsr;
+  }
 
   rc = lsm_csr_seek(csr, pKey, nKey, LSM_SEEK_EQ);
   if( rc==LSM_OK ){
@@ -638,7 +642,9 @@ static int test_lsm_fetch(
       *pnVal = -1;
     }
   }
-  lsm_csr_close(csr);
+  if( pDb->pCsr==0 ){
+    lsm_csr_close(csr);
+  }
   return rc;
 }
 
@@ -652,10 +658,28 @@ static int test_lsm_scan(
 ){
   LsmDb *pDb = (LsmDb *)pTestDb;
   lsm_cursor *csr;
+  lsm_cursor *csr2 = 0;
   int rc;
 
-  rc = lsm_csr_open(pDb->db, &csr);
+  if( pDb->pCsr==0 ){
-  if( rc!=LSM_OK ) return rc;
+    rc = lsm_csr_open(pDb->db, &csr);
+    if( rc!=LSM_OK ) return rc;
+  }else{
+    rc = LSM_OK;
+    csr = pDb->pCsr;
+  }
+
+  /* To enhance testing, if both pLast and pFirst are defined, seek the
+  ** cursor to the "end" boundary here. Then the next block seeks it to
+  ** the "start" ready for the scan. The point is to test that cursors
+  ** can be reused.  */
+  if( pLast && pFirst ){
+    if( bReverse ){
+      rc = lsm_csr_seek(csr, pFirst, nFirst, LSM_SEEK_LE);
+    }else{
+      rc = lsm_csr_seek(csr, pLast, nLast, LSM_SEEK_GE);
+    }
+  }
 
   if( bReverse ){
     if( pLast ){
@@ -696,7 +720,9 @@ static int test_lsm_scan(
     }
   }
 
-  lsm_csr_close(csr);
+  if( pDb->pCsr==0 ){
+    lsm_csr_close(csr);
+  }
   return rc;
 }
 
@@ -762,6 +788,7 @@ static void xWorkHook(lsm_db *db, void *pArg){
 #define TEST_MT_MIN_CKPT -4
 #define TEST_MT_MAX_CKPT -5
 
+
 int test_lsm_config_str(
   LsmDb *pLsm,
   lsm_db *db, 
@@ -1033,6 +1060,19 @@ int test_lsm_lomem_open(
   return testLsmOpen(zCfg, zFilename, bClear, ppDb);
 }
 
+int test_lsm_lomem2_open(
+  const char *zSpec, 
+  const char *zFilename, 
+  int bClear, 
+  TestDb **ppDb
+){
+    /* "max_freelist=4 autocheckpoint=32" */
+  const char *zCfg = 
+    "page_size=512 block_size=64 autoflush=0 mmap=0 "
+  ;
+  return testLsmOpen(zCfg, zFilename, bClear, ppDb);
+}
+
 int test_lsm_zip_open(
   const char *zSpec, 
   const char *zFilename, 

ext/lsm1/lsmInt.h

@@ -110,7 +110,7 @@ typedef unsigned long long int u64;
 #endif
 
 /* A page number is a 64-bit integer. */
-typedef i64 Pgno;
+typedef i64 LsmPgno;
 
 #ifdef LSM_DEBUG
 int lsmErrorBkpt(int);
@@ -402,9 +402,9 @@ struct lsm_db {
 };
 
 struct Segment {
-  Pgno iFirst;                     /* First page of this run */
+  LsmPgno iFirst;                  /* First page of this run */
-  Pgno iLastPg;                    /* Last page of this run */
+  LsmPgno iLastPg;                 /* Last page of this run */
-  Pgno iRoot;                      /* Root page number (if any) */
+  LsmPgno iRoot;                   /* Root page number (if any) */
   int nSize;                       /* Size of this run in pages */
 
   Redirect *pRedirect;             /* Block redirects (or NULL) */
@@ -456,7 +456,7 @@ struct Level {
 **   output segment.
 */
 struct MergeInput {
-  Pgno iPg;                       /* Page on which next input is stored */
+  LsmPgno iPg;                    /* Page on which next input is stored */
   int iCell;                      /* Cell containing next input to merge */
 };
 struct Merge {
@@ -465,7 +465,7 @@ struct Merge {
   MergeInput splitkey;            /* Location in file of current splitkey */
   int nSkip;                      /* Number of separators entries to skip */
   int iOutputOff;                 /* Write offset on output page */
-  Pgno iCurrentPtr;               /* Current pointer value */
+  LsmPgno iCurrentPtr;            /* Current pointer value */
 };
 
 /* 
@@ -579,10 +579,10 @@ struct Snapshot {
   Redirect redirect;              /* Block redirection array */
 
   /* Used by worker snapshots only */
-  int nBlock;                     /* Number of blocks in database file */
+  int nBlock;                        /* Number of blocks in database file */
-  Pgno aiAppend[LSM_APPLIST_SZ];  /* Append point list */
+  LsmPgno aiAppend[LSM_APPLIST_SZ];  /* Append point list */
-  Freelist freelist;              /* Free block list */
+  Freelist freelist;                 /* Free block list */
-  u32 nWrite;                     /* Total number of pages written to disk */
+  u32 nWrite;                        /* Total number of pages written to disk */
 };
 #define LSM_INITIAL_SNAPSHOT_ID 11
 
@@ -710,7 +710,7 @@ void lsmFsSetPageSize(FileSystem *, int);
 int lsmFsFileid(lsm_db *pDb, void **ppId, int *pnId);
 
 /* Creating, populating, gobbling and deleting sorted runs. */
-void lsmFsGobble(lsm_db *, Segment *, Pgno *, int);
+void lsmFsGobble(lsm_db *, Segment *, LsmPgno *, int);
 int lsmFsSortedDelete(FileSystem *, Snapshot *, int, Segment *);
 int lsmFsSortedFinish(FileSystem *, Segment *);
 int lsmFsSortedAppend(FileSystem *, Snapshot *, Level *, int, Page **);
@@ -727,14 +727,14 @@ void lsmSortedSplitkey(lsm_db *, Level *, int *);
 
 /* Reading sorted run content. */
 int lsmFsDbPageLast(FileSystem *pFS, Segment *pSeg, Page **ppPg);
-int lsmFsDbPageGet(FileSystem *, Segment *, Pgno, Page **);
+int lsmFsDbPageGet(FileSystem *, Segment *, LsmPgno, Page **);
 int lsmFsDbPageNext(Segment *, Page *, int eDir, Page **);
 
 u8 *lsmFsPageData(Page *, int *);
 int lsmFsPageRelease(Page *);
 int lsmFsPagePersist(Page *);
 void lsmFsPageRef(Page *);
-Pgno lsmFsPageNumber(Page *);
+LsmPgno lsmFsPageNumber(Page *);
 
 int lsmFsNRead(FileSystem *);
 int lsmFsNWrite(FileSystem *);
@@ -748,7 +748,7 @@ int lsmFsDbPageIsLast(Segment *pSeg, Page *pPg);
 int lsmFsIntegrityCheck(lsm_db *);
 #endif
 
-Pgno lsmFsRedirectPage(FileSystem *, Redirect *, Pgno);
+LsmPgno lsmFsRedirectPage(FileSystem *, Redirect *, LsmPgno);
 
 int lsmFsPageWritable(Page *);
 
@@ -768,8 +768,8 @@ int lsmFsSyncDb(FileSystem *, int);
 void lsmFsFlushWaiting(FileSystem *, int *);
 
 /* Used by lsm_info(ARRAY_STRUCTURE) and lsm_config(MMAP) */
-int lsmInfoArrayStructure(lsm_db *pDb, int bBlock, Pgno iFirst, char **pzOut);
+int lsmInfoArrayStructure(lsm_db *pDb, int bBlock, LsmPgno iFirst, char **pz);
-int lsmInfoArrayPages(lsm_db *pDb, Pgno iFirst, char **pzOut);
+int lsmInfoArrayPages(lsm_db *pDb, LsmPgno iFirst, char **pzOut);
 int lsmConfigMmap(lsm_db *pDb, int *piParam);
 
 int lsmEnvOpen(lsm_env *, const char *, int, lsm_file **);
@@ -785,7 +785,7 @@ void lsmEnvSleep(lsm_env *, int);
 
 int lsmFsReadSyncedId(lsm_db *db, int, i64 *piVal);
 
-int lsmFsSegmentContainsPg(FileSystem *pFS, Segment *, Pgno, int *);
+int lsmFsSegmentContainsPg(FileSystem *pFS, Segment *, LsmPgno, int *);
 
 void lsmFsPurgeCache(FileSystem *);
 
@@ -796,7 +796,7 @@ void lsmFsPurgeCache(FileSystem *);
 /* 
 ** Functions from file "lsm_sorted.c".
 */
-int lsmInfoPageDump(lsm_db *, Pgno, int, char **);
+int lsmInfoPageDump(lsm_db *, LsmPgno, int, char **);
 void lsmSortedCleanup(lsm_db *);
 int lsmSortedAutoWork(lsm_db *, int nUnit);
 

ext/lsm1/lsm_ckpt.c

@@ -389,7 +389,7 @@ static void ckptExportAppendlist(
   int *pRc                        /* IN/OUT: Error code */
 ){
   int i;
-  Pgno *aiAppend = db->pWorker->aiAppend;
+  LsmPgno *aiAppend = db->pWorker->aiAppend;
 
   for(i=0; i<LSM_APPLIST_SZ; i++){
     ckptAppend64(p, piOut, aiAppend[i], pRc);

ext/lsm1/lsm_file.c

@@ -269,7 +269,7 @@ struct FileSystem {
 struct Page {
   u8 *aData;                      /* Buffer containing page data */
   int nData;                      /* Bytes of usable data at aData[] */
-  Pgno iPg;                       /* Page number */
+  LsmPgno iPg;                    /* Page number */
   int nRef;                       /* Number of outstanding references */
   int flags;                      /* Combination of PAGE_XXX flags */
   Page *pHashNext;                /* Next page in hash table slot */
@@ -332,7 +332,7 @@ static int IOERR_WRAPPER(int rc){
 #ifdef NDEBUG
 # define assert_lists_are_ok(x)
 #else
-static Page *fsPageFindInHash(FileSystem *pFS, Pgno iPg, int *piHash);
+static Page *fsPageFindInHash(FileSystem *pFS, LsmPgno iPg, int *piHash);
 
 static void assert_lists_are_ok(FileSystem *pFS){
 #if 0
@@ -532,7 +532,7 @@ int lsmFsCloseAndDeleteLog(FileSystem *pFS){
 ** Return true if page iReal of the database should be accessed using mmap.
 ** False otherwise.
 */
-static int fsMmapPage(FileSystem *pFS, Pgno iReal){
+static int fsMmapPage(FileSystem *pFS, LsmPgno iReal){
   return ((i64)iReal*pFS->nPagesize <= pFS->nMapLimit);
 }
 
@@ -540,7 +540,7 @@ static int fsMmapPage(FileSystem *pFS, Pgno iReal){
 ** Given that there are currently nHash slots in the hash table, return 
 ** the hash key for file iFile, page iPg.
 */
-static int fsHashKey(int nHash, Pgno iPg){
+static int fsHashKey(int nHash, LsmPgno iPg){
   return (iPg % nHash);
 }
 
@@ -880,13 +880,13 @@ void lsmFsSetBlockSize(FileSystem *pFS, int nBlocksize){
 ** page on each block is the byte offset immediately following the 4-byte
 ** "previous block" pointer at the start of each block.
 */
-static Pgno fsFirstPageOnBlock(FileSystem *pFS, int iBlock){
+static LsmPgno fsFirstPageOnBlock(FileSystem *pFS, int iBlock){
-  Pgno iPg;
+  LsmPgno iPg;
   if( pFS->pCompress ){
     if( iBlock==1 ){
       iPg = pFS->nMetasize * 2 + 4;
     }else{
-      iPg = pFS->nBlocksize * (Pgno)(iBlock-1) + 4;
+      iPg = pFS->nBlocksize * (LsmPgno)(iBlock-1) + 4;
     }
   }else{
     const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize);
@@ -907,9 +907,9 @@ static Pgno fsFirstPageOnBlock(FileSystem *pFS, int iBlock){
 ** page on each block is the byte offset of the byte immediately before 
 ** the 4-byte "next block" pointer at the end of each block.
 */
-static Pgno fsLastPageOnBlock(FileSystem *pFS, int iBlock){
+static LsmPgno fsLastPageOnBlock(FileSystem *pFS, int iBlock){
   if( pFS->pCompress ){
-    return pFS->nBlocksize * (Pgno)iBlock - 1 - 4;
+    return pFS->nBlocksize * (LsmPgno)iBlock - 1 - 4;
   }else{
     const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize);
     return iBlock * nPagePerBlock;
@@ -920,7 +920,7 @@ static Pgno fsLastPageOnBlock(FileSystem *pFS, int iBlock){
 ** Return the block number of the block that page iPg is located on. 
 ** Blocks are numbered starting from 1.
 */
-static int fsPageToBlock(FileSystem *pFS, Pgno iPg){
+static int fsPageToBlock(FileSystem *pFS, LsmPgno iPg){
   if( pFS->pCompress ){
     return (int)((iPg / pFS->nBlocksize) + 1);
   }else{
@@ -933,7 +933,7 @@ static int fsPageToBlock(FileSystem *pFS, Pgno iPg){
 **
 ** This function is only called in non-compressed database mode.
 */
-static int fsIsLast(FileSystem *pFS, Pgno iPg){
+static int fsIsLast(FileSystem *pFS, LsmPgno iPg){
   const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize);
   assert( !pFS->pCompress );
   return ( iPg && (iPg % nPagePerBlock)==0 );
@@ -944,7 +944,7 @@ static int fsIsLast(FileSystem *pFS, Pgno iPg){
 **
 ** This function is only called in non-compressed database mode.
 */
-static int fsIsFirst(FileSystem *pFS, Pgno iPg){
+static int fsIsFirst(FileSystem *pFS, LsmPgno iPg){
   const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize);
   assert( !pFS->pCompress );
   return ( (iPg % nPagePerBlock)==1
@@ -967,7 +967,7 @@ u8 *lsmFsPageData(Page *pPage, int *pnData){
 /*
 ** Return the page number of a page.
 */
-Pgno lsmFsPageNumber(Page *pPage){
+LsmPgno lsmFsPageNumber(Page *pPage){
   /* assert( (pPage->flags & PAGE_DIRTY)==0 ); */
   return pPage ? pPage->iPg : 0;
 }
@@ -1058,7 +1058,7 @@ void lsmFsPurgeCache(FileSystem *pFS){
 ** Either way, if argument piHash is not NULL set *piHash to the hash slot
 ** number that page iPg would be stored in before returning.
 */
-static Page *fsPageFindInHash(FileSystem *pFS, Pgno iPg, int *piHash){
+static Page *fsPageFindInHash(FileSystem *pFS, LsmPgno iPg, int *piHash){
   Page *p;                        /* Return value */
   int iHash = fsHashKey(pFS->nHash, iPg);
 
@@ -1189,8 +1189,8 @@ static int fsRedirectBlock(Redirect *p, int iBlk){
 ** object passed as the second argument, return the destination page to
 ** which it is redirected. Otherwise, return a copy of iPg.
 */
-Pgno lsmFsRedirectPage(FileSystem *pFS, Redirect *pRedir, Pgno iPg){
+LsmPgno lsmFsRedirectPage(FileSystem *pFS, Redirect *pRedir, LsmPgno iPg){
-  Pgno iReal = iPg;
+  LsmPgno iReal = iPg;
 
   if( pRedir ){
     const int nPagePerBlock = (
@@ -1203,7 +1203,7 @@ Pgno lsmFsRedirectPage(FileSystem *pFS, Redirect *pRedir, Pgno iPg){
       if( iFrom>iBlk ) break;
       if( iFrom==iBlk ){
         int iTo = pRedir->a[i].iTo;
-        iReal = iPg - (Pgno)(iFrom - iTo) * nPagePerBlock;
+        iReal = iPg - (LsmPgno)(iFrom - iTo) * nPagePerBlock;
         if( iTo==1 ){
           iReal += (fsFirstPageOnBlock(pFS, 1)-1);
         }
@@ -1217,7 +1217,7 @@ Pgno lsmFsRedirectPage(FileSystem *pFS, Redirect *pRedir, Pgno iPg){
 }
 
 /* Required by the circular fsBlockNext<->fsPageGet dependency. */
-static int fsPageGet(FileSystem *, Segment *, Pgno, int, Page **, int *);
+static int fsPageGet(FileSystem *, Segment *, LsmPgno, int, Page **, int *);
 
 /*
 ** Parameter iBlock is a database file block. This function reads the value 
@@ -1269,7 +1269,7 @@ static int fsBlockNext(
 /*
 ** Return the page number of the last page on the same block as page iPg.
 */
-Pgno fsLastPageOnPagesBlock(FileSystem *pFS, Pgno iPg){
+LsmPgno fsLastPageOnPagesBlock(FileSystem *pFS, LsmPgno iPg){
   return fsLastPageOnBlock(pFS, fsPageToBlock(pFS, iPg));
 }
 
@@ -1537,7 +1537,7 @@ static int fsReadPagedata(
 static int fsPageGet(
   FileSystem *pFS,                /* File-system handle */
   Segment *pSeg,                  /* Block redirection to use (or NULL) */
-  Pgno iPg,                       /* Page id */
+  LsmPgno iPg,                    /* Page id */
   int noContent,                  /* True to not load content from disk */
   Page **ppPg,                    /* OUT: New page handle */
   int *pnSpace                    /* OUT: Bytes of free space */
@@ -1549,7 +1549,7 @@ static int fsPageGet(
   /* In most cases iReal is the same as iPg. Except, if pSeg->pRedirect is 
   ** not NULL, and the block containing iPg has been redirected, then iReal
   ** is the page number after redirection.  */
-  Pgno iReal = lsmFsRedirectPage(pFS, (pSeg ? pSeg->pRedirect : 0), iPg);
+  LsmPgno iReal = lsmFsRedirectPage(pFS, (pSeg ? pSeg->pRedirect : 0), iPg);
 
   assert_lists_are_ok(pFS);
   assert( iPg>=fsFirstPageOnBlock(pFS, 1) );
@@ -1689,8 +1689,8 @@ int lsmFsReadSyncedId(lsm_db *db, int iMeta, i64 *piVal){
 static int fsRunEndsBetween(
   Segment *pRun, 
   Segment *pIgnore, 
-  Pgno iFirst, 
+  LsmPgno iFirst, 
-  Pgno iLast
+  LsmPgno iLast
 ){
   return (pRun!=pIgnore && (
         (pRun->iFirst>=iFirst && pRun->iFirst<=iLast)
@@ -1705,8 +1705,8 @@ static int fsRunEndsBetween(
 static int fsLevelEndsBetween(
   Level *pLevel, 
   Segment *pIgnore, 
-  Pgno iFirst, 
+  LsmPgno iFirst, 
-  Pgno iLast
+  LsmPgno iLast
 ){
   int i;
 
@@ -1733,13 +1733,13 @@ static int fsFreeBlock(
   int iBlk                        /* Block number of block to free */
 ){
   int rc = LSM_OK;                /* Return code */
-  Pgno iFirst;                    /* First page on block iBlk */
+  LsmPgno iFirst;                 /* First page on block iBlk */
-  Pgno iLast;                     /* Last page on block iBlk */
+  LsmPgno iLast;                  /* Last page on block iBlk */
   Level *pLevel;                  /* Used to iterate through levels */
 
   int iIn;                        /* Used to iterate through append points */
   int iOut = 0;                   /* Used to output append points */
-  Pgno *aApp = pSnapshot->aiAppend;
+  LsmPgno *aApp = pSnapshot->aiAppend;
 
   iFirst = fsFirstPageOnBlock(pFS, iBlk);
   iLast = fsLastPageOnBlock(pFS, iBlk);
@@ -1811,11 +1811,16 @@ int lsmFsSortedDelete(
 ** number from the array that falls on block iBlk. Or, if none of the pages
 ** in aPgno[] fall on block iBlk, return 0.
 */
-static Pgno firstOnBlock(FileSystem *pFS, int iBlk, Pgno *aPgno, int nPgno){
+static LsmPgno firstOnBlock(
-  Pgno iRet = 0;
+  FileSystem *pFS, 
+  int iBlk, 
+  LsmPgno *aPgno, 
+  int nPgno
+){
+  LsmPgno iRet = 0;
   int i;
   for(i=0; i<nPgno; i++){
-    Pgno iPg = aPgno[i];
+    LsmPgno iPg = aPgno[i];
     if( fsPageToBlock(pFS, iPg)==iBlk && (iRet==0 || iPg<iRet) ){
       iRet = iPg;
     }
@@ -1828,7 +1833,7 @@ static Pgno firstOnBlock(FileSystem *pFS, int iBlk, Pgno *aPgno, int nPgno){
 ** Return true if page iPg, which is a part of segment p, lies on
 ** a redirected block. 
 */
-static int fsPageRedirects(FileSystem *pFS, Segment *p, Pgno iPg){
+static int fsPageRedirects(FileSystem *pFS, Segment *p, LsmPgno iPg){
   return (iPg!=0 && iPg!=lsmFsRedirectPage(pFS, p->pRedirect, iPg));
 }
 
@@ -1854,7 +1859,7 @@ static int fsSegmentRedirects(FileSystem *pFS, Segment *p){
 void lsmFsGobble(
   lsm_db *pDb,
   Segment *pRun, 
-  Pgno *aPgno,
+  LsmPgno *aPgno,
   int nPgno
 ){
   int rc = LSM_OK;
@@ -1871,7 +1876,7 @@ void lsmFsGobble(
 
   while( rc==LSM_OK ){
     int iNext = 0;
-    Pgno iFirst = firstOnBlock(pFS, iBlk, aPgno, nPgno);
+    LsmPgno iFirst = firstOnBlock(pFS, iBlk, aPgno, nPgno);
     if( iFirst ){
       pRun->iFirst = iFirst;
       break;
@@ -1905,11 +1910,11 @@ void lsmFsGobble(
 static int fsNextPageOffset(
   FileSystem *pFS,                /* File system object */
   Segment *pSeg,                  /* Segment to move within */
-  Pgno iPg,                       /* Offset of current page */
+  LsmPgno iPg,                    /* Offset of current page */
   int nByte,                      /* Size of current page including headers */
-  Pgno *piNext                    /* OUT: Offset of next page. Or zero (EOF) */
+  LsmPgno *piNext                 /* OUT: Offset of next page. Or zero (EOF) */
 ){
-  Pgno iNext;
+  LsmPgno iNext;
   int rc;
 
   assert( pFS->pCompress );
@@ -1939,8 +1944,8 @@ static int fsNextPageOffset(
 static int fsGetPageBefore(
   FileSystem *pFS, 
   Segment *pSeg, 
-  Pgno iPg, 
+  LsmPgno iPg, 
-  Pgno *piPrev
+  LsmPgno *piPrev
 ){
   u8 aSz[3];
   int rc;
@@ -1990,7 +1995,7 @@ static int fsGetPageBefore(
 int lsmFsDbPageNext(Segment *pRun, Page *pPg, int eDir, Page **ppNext){
   int rc = LSM_OK;
   FileSystem *pFS = pPg->pFS;
-  Pgno iPg = pPg->iPg;
+  LsmPgno iPg = pPg->iPg;
 
   assert( 0==fsSegmentRedirects(pFS, pRun) );
   if( pFS->pCompress ){
@@ -2062,10 +2067,10 @@ int lsmFsDbPageNext(Segment *pRun, Page *pPg, int eDir, Page **ppNext){
 ** start the new segment immediately following any segment that is part
 ** of the right-hand-side of pLvl.
 */
-static Pgno findAppendPoint(FileSystem *pFS, Level *pLvl){
+static LsmPgno findAppendPoint(FileSystem *pFS, Level *pLvl){
   int i;
-  Pgno *aiAppend = pFS->pDb->pWorker->aiAppend;
+  LsmPgno *aiAppend = pFS->pDb->pWorker->aiAppend;
-  Pgno iRet = 0;
+  LsmPgno iRet = 0;
 
   for(i=LSM_APPLIST_SZ-1; iRet==0 && i>=0; i--){
     if( (iRet = aiAppend[i]) ){
@@ -2098,10 +2103,10 @@ int lsmFsSortedAppend(
 ){
   int rc = LSM_OK;
   Page *pPg = 0;
-  Pgno iApp = 0;
+  LsmPgno iApp = 0;
-  Pgno iNext = 0;
+  LsmPgno iNext = 0;
   Segment *p = &pLvl->lhs;
-  Pgno iPrev = p->iLastPg;
+  LsmPgno iPrev = p->iLastPg;
 
   *ppOut = 0;
   assert( p->pRedirect==0 );
@@ -2195,7 +2200,7 @@ int lsmFsSortedFinish(FileSystem *pFS, Segment *p){
     */
     if( fsLastPageOnPagesBlock(pFS, p->iLastPg)!=p->iLastPg ){
       int i;
-      Pgno *aiAppend = pFS->pDb->pWorker->aiAppend;
+      LsmPgno *aiAppend = pFS->pDb->pWorker->aiAppend;
       for(i=0; i<LSM_APPLIST_SZ; i++){
         if( aiAppend[i]==0 ){
           aiAppend[i] = p->iLastPg+1;
@@ -2226,7 +2231,7 @@ int lsmFsSortedFinish(FileSystem *pFS, Segment *p){
 **
 ** Return LSM_OK if successful, or an lsm error code if an error occurs.
 */
-int lsmFsDbPageGet(FileSystem *pFS, Segment *pSeg, Pgno iPg, Page **ppPg){
+int lsmFsDbPageGet(FileSystem *pFS, Segment *pSeg, LsmPgno iPg, Page **ppPg){
   return fsPageGet(pFS, pSeg, iPg, 0, ppPg, 0);
 }
 
@@ -2238,7 +2243,7 @@ int lsmFsDbPageGet(FileSystem *pFS, Segment *pSeg, Pgno iPg, Page **ppPg){
 */
 int lsmFsDbPageLast(FileSystem *pFS, Segment *pSeg, Page **ppPg){
   int rc;
-  Pgno iPg = pSeg->iLastPg;
+  LsmPgno iPg = pSeg->iLastPg;
   if( pFS->pCompress ){
     int nSpace;
     iPg++;
@@ -2366,14 +2371,14 @@ static void fsMovePage(
   FileSystem *pFS,                /* File system object */
   int iTo,                        /* Destination block */
   int iFrom,                      /* Source block */
-  Pgno *piPg                      /* IN/OUT: Page number */
+  LsmPgno *piPg                   /* IN/OUT: Page number */
 ){
-  Pgno iPg = *piPg;
+  LsmPgno iPg = *piPg;
   if( iFrom==fsPageToBlock(pFS, iPg) ){
     const int nPagePerBlock = (
         pFS->pCompress ? pFS ->nBlocksize : (pFS->nBlocksize / pFS->nPagesize)
     );
-    *piPg = iPg - (Pgno)(iFrom - iTo) * nPagePerBlock;
+    *piPg = iPg - (LsmPgno)(iFrom - iTo) * nPagePerBlock;
   }
 }
 
@@ -2457,21 +2462,21 @@ int lsmFsMoveBlock(FileSystem *pFS, Segment *pSeg, int iTo, int iFrom){
 **
 ** This function is only used in compressed database mode.
 */
-static Pgno fsAppendData(
+static LsmPgno fsAppendData(
   FileSystem *pFS,                /* File-system handle */
   Segment *pSeg,                  /* Segment to append to */
   const u8 *aData,                /* Buffer containing data to write */
   int nData,                      /* Size of buffer aData[] in bytes */
   int *pRc                        /* IN/OUT: Error code */
 ){
-  Pgno iRet = 0;
+  LsmPgno iRet = 0;
   int rc = *pRc;
   assert( pFS->pCompress );
   if( rc==LSM_OK ){
     int nRem = 0;
     int nWrite = 0;
-    Pgno iLastOnBlock;
+    LsmPgno iLastOnBlock;
-    Pgno iApp = pSeg->iLastPg+1;
+    LsmPgno iApp = pSeg->iLastPg+1;
 
     /* If this is the first data written into the segment, find an append-point
     ** or allocate a new block.  */
@@ -2519,7 +2524,7 @@ static Pgno fsAppendData(
 
         /* Set the "prev" pointer on the new block */
         if( rc==LSM_OK ){
-          Pgno iWrite;
+          LsmPgno iWrite;
           lsmPutU32(aPtr, fsPageToBlock(pFS, iApp));
           iWrite = fsFirstPageOnBlock(pFS, iBlk);
           rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iWrite-4, aPtr, sizeof(aPtr));
@@ -2588,11 +2593,11 @@ static int fsCompressIntoBuffer(FileSystem *pFS, Page *pPg){
 static int fsAppendPage(
   FileSystem *pFS, 
   Segment *pSeg,
-  Pgno *piNew,
+  LsmPgno *piNew,
   int *piPrev,
   int *piNext
 ){
-  Pgno iPrev = pSeg->iLastPg;
+  LsmPgno iPrev = pSeg->iLastPg;
   int rc;
   assert( iPrev!=0 );
 
@@ -2650,7 +2655,7 @@ void lsmFsFlushWaiting(FileSystem *pFS, int *pRc){
 /*
 ** If there exists a hash-table entry associated with page iPg, remove it.
 */
-static void fsRemoveHashEntry(FileSystem *pFS, Pgno iPg){
+static void fsRemoveHashEntry(FileSystem *pFS, LsmPgno iPg){
   Page *p;
   int iHash = fsHashKey(pFS->nHash, iPg);
 
@@ -2803,9 +2808,9 @@ int lsmFsSortedPadding(
   Segment *pSeg
 ){
   int rc = LSM_OK;
-  if( pFS->pCompress ){
+  if( pFS->pCompress && pSeg->iFirst ){
-    Pgno iLast2;
+    LsmPgno iLast2;
-    Pgno iLast = pSeg->iLastPg;     /* Current last page of segment */
+    LsmPgno iLast = pSeg->iLastPg;  /* Current last page of segment */
     int nPad;                       /* Bytes of padding required */
     u8 aSz[3];
 
@@ -2935,7 +2940,7 @@ int lsmFsSectorSize(FileSystem *pFS){
 /*
 ** Helper function for lsmInfoArrayStructure().
 */
-static Segment *startsWith(Segment *pRun, Pgno iFirst){
+static Segment *startsWith(Segment *pRun, LsmPgno iFirst){
   return (iFirst==pRun->iFirst) ? pRun : 0;
 }
 
@@ -2943,7 +2948,7 @@ static Segment *startsWith(Segment *pRun, Pgno iFirst){
 ** Return the segment that starts with page iFirst, if any. If no such segment
 ** can be found, return NULL.
 */
-static Segment *findSegment(Snapshot *pWorker, Pgno iFirst){
+static Segment *findSegment(Snapshot *pWorker, LsmPgno iFirst){
   Level *pLvl;                    /* Used to iterate through db levels */
   Segment *pSeg = 0;              /* Pointer to segment to return */
 
@@ -2970,7 +2975,7 @@ static Segment *findSegment(Snapshot *pWorker, Pgno iFirst){
 int lsmInfoArrayStructure(
   lsm_db *pDb, 
   int bBlock,                     /* True for block numbers only */
-  Pgno iFirst,
+  LsmPgno iFirst,
   char **pzOut
 ){
   int rc = LSM_OK;
@@ -3035,7 +3040,7 @@ int lsmInfoArrayStructure(
 int lsmFsSegmentContainsPg(
   FileSystem *pFS, 
   Segment *pSeg, 
-  Pgno iPg, 
+  LsmPgno iPg, 
   int *pbRes
 ){
   Redirect *pRedir = pSeg->pRedirect;
@@ -3064,7 +3069,7 @@ int lsmFsSegmentContainsPg(
 **
 ** If an error occurs, *pzOut is set to NULL and an LSM error code returned.
 */
-int lsmInfoArrayPages(lsm_db *pDb, Pgno iFirst, char **pzOut){
+int lsmInfoArrayPages(lsm_db *pDb, LsmPgno iFirst, char **pzOut){
   int rc = LSM_OK;
   Snapshot *pWorker;              /* Worker snapshot */
   Segment *pSeg = 0;              /* Array to report on */
@@ -3297,7 +3302,7 @@ int lsmFsIntegrityCheck(lsm_db *pDb){
 */
 int lsmFsDbPageIsLast(Segment *pSeg, Page *pPg){
   if( pPg->pFS->pCompress ){
-    Pgno iNext = 0;
+    LsmPgno iNext = 0;
     int rc;
     rc = fsNextPageOffset(pPg->pFS, pSeg, pPg->iPg, pPg->nCompress+6, &iNext);
     return (rc!=LSM_OK || iNext==0);

ext/lsm1/lsm_main.c

@@ -583,14 +583,14 @@ int lsm_info(lsm_db *pDb, int eParam, ...){
     }
 
     case LSM_INFO_ARRAY_STRUCTURE: {
-      Pgno pgno = va_arg(ap, Pgno);
+      LsmPgno pgno = va_arg(ap, LsmPgno);
       char **pzVal = va_arg(ap, char **);
       rc = lsmInfoArrayStructure(pDb, 0, pgno, pzVal);
       break;
     }
 
     case LSM_INFO_ARRAY_PAGES: {
-      Pgno pgno = va_arg(ap, Pgno);
+      LsmPgno pgno = va_arg(ap, LsmPgno);
       char **pzVal = va_arg(ap, char **);
       rc = lsmInfoArrayPages(pDb, pgno, pzVal);
       break;
@@ -598,7 +598,7 @@ int lsm_info(lsm_db *pDb, int eParam, ...){
 
     case LSM_INFO_PAGE_HEX_DUMP:
     case LSM_INFO_PAGE_ASCII_DUMP: {
-      Pgno pgno = va_arg(ap, Pgno);
+      LsmPgno pgno = va_arg(ap, LsmPgno);
       char **pzVal = va_arg(ap, char **);
       int bUnlock = 0;
       rc = infoGetWorker(pDb, 0, &bUnlock);
@@ -683,7 +683,7 @@ static int doWriteOp(
     int nDiff;
 
     if( nQuant>pDb->nTreeLimit ){
-      nQuant = pDb->nTreeLimit;
+      nQuant = LSM_MAX(pDb->nTreeLimit, pgsz);
     }
 
     nBefore = lsmTreeSize(pDb);

ext/lsm1/lsm_shared.c

@@ -340,9 +340,6 @@ static int doDbConnect(lsm_db *pDb){
   /* Obtain a pointer to the shared-memory header */
   assert( pDb->pShmhdr==0 );
   assert( pDb->bReadonly==0 );
-  rc = lsmShmCacheChunks(pDb, 1);
-  if( rc!=LSM_OK ) return rc;
-  pDb->pShmhdr = (ShmHeader *)pDb->apShm[0];
 
   /* Block for an exclusive lock on DMS1. This lock serializes all calls
   ** to doDbConnect() and doDbDisconnect() across all processes.  */
@@ -353,10 +350,11 @@ static int doDbConnect(lsm_db *pDb){
     nUs = nUs * 2;
     if( nUs>nUsMax ) nUs = nUsMax;
   }
-  if( rc!=LSM_OK ){
+  if( rc==LSM_OK ){
-    pDb->pShmhdr = 0;
+    rc = lsmShmCacheChunks(pDb, 1);
-    return rc;
   }
+  if( rc!=LSM_OK ) return rc;
+  pDb->pShmhdr = (ShmHeader *)pDb->apShm[0];
 
   /* Try an exclusive lock on DMS2/DMS3. If successful, this is the first 
   ** and only connection to the database. In this case initialize the 
@@ -522,13 +520,11 @@ int lsmDbDatabaseConnect(
   ** recovery as necessary. Or, if this is a read-only database handle,
   ** defer attempting to connect to the system until a read-transaction
   ** is opened.  */
-  if( pDb->bReadonly==0 ){
+  if( rc==LSM_OK ){
-    if( rc==LSM_OK ){
+    rc = lsmFsConfigure(pDb);
-      rc = lsmFsConfigure(pDb);
+  }
-    }
+  if( rc==LSM_OK && pDb->bReadonly==0 ){
-    if( rc==LSM_OK ){
+    rc = doDbConnect(pDb);
-      rc = doDbConnect(pDb);
-    }
   }
 
   return rc;

ext/lsm1/lsm_sorted.c

@@ -92,7 +92,7 @@
 #define SEGMENT_POINTER_OFFSET(pgsz)        ((pgsz) - 2 - 2 - 8)
 #define SEGMENT_CELLPTR_OFFSET(pgsz, iCell) ((pgsz) - 2 - 2 - 8 - 2 - (iCell)*2)
 
-#define SEGMENT_EOF(pgsz, nEntry) SEGMENT_CELLPTR_OFFSET(pgsz, nEntry)
+#define SEGMENT_EOF(pgsz, nEntry) SEGMENT_CELLPTR_OFFSET(pgsz, nEntry-1)
 
 #define SEGMENT_BTREE_FLAG     0x0001
 #define PGFTR_SKIP_NEXT_FLAG   0x0002
@@ -104,9 +104,9 @@
 #endif
 
 typedef struct SegmentPtr SegmentPtr;
-typedef struct Blob Blob;
+typedef struct LsmBlob LsmBlob;
 
-struct Blob {
+struct LsmBlob {
   lsm_env *pEnv;
   void *pData;
   int nData;
@@ -129,18 +129,18 @@ struct SegmentPtr {
   Page *pPg;                    /* Current page */
   u16 flags;                    /* Copy of page flags field */
   int nCell;                    /* Number of cells on pPg */
-  Pgno iPtr;                    /* Base cascade pointer */
+  LsmPgno iPtr;                 /* Base cascade pointer */
 
   /* Current cell. See segmentPtrLoadCell() */
   int iCell;                    /* Current record within page pPg */
   int eType;                    /* Type of current record */
-  Pgno iPgPtr;                  /* Cascade pointer offset */
+  LsmPgno iPgPtr;               /* Cascade pointer offset */
   void *pKey; int nKey;         /* Key associated with current record */
   void *pVal; int nVal;         /* Current record value (eType==WRITE only) */
 
   /* Blobs used to allocate buffers for pKey and pVal as required */
-  Blob blob1;
+  LsmBlob blob1;
-  Blob blob2;
+  LsmBlob blob2;
 };
 
 /*
@@ -171,10 +171,10 @@ struct BtreeCursor {
   void *pKey;
   int nKey;
   int eType;
-  Pgno iPtr;
+  LsmPgno iPtr;
 
   /* Storage for key, if not local */
-  Blob blob;
+  LsmBlob blob;
 };
 
 
@@ -203,8 +203,8 @@ struct MultiCursor {
   int flags;                      /* Mask of CURSOR_XXX flags */
 
   int eType;                      /* Cache of current key type */
-  Blob key;                       /* Cache of current key (or NULL) */
+  LsmBlob key;                    /* Cache of current key (or NULL) */
-  Blob val;                       /* Cache of current value */
+  LsmBlob val;                    /* Cache of current value */
 
   /* All the component cursors: */
   TreeCursor *apTreeCsr[2];       /* Up to two tree cursors */
@@ -221,7 +221,7 @@ struct MultiCursor {
   void *pSystemVal;               /* Pointer to buffer to free */
 
   /* Used by worker cursors only */
-  Pgno *pPrevMergePtr;
+  LsmPgno *pPrevMergePtr;
 };
 
 /*
@@ -295,11 +295,11 @@ struct MergeWorker {
   Hierarchy hier;                 /* B-tree hierarchy under construction */
   Page *pPage;                    /* Current output page */
   int nWork;                      /* Number of calls to mergeWorkerNextPage() */
-  Pgno *aGobble;                  /* Gobble point for each input segment */
+  LsmPgno *aGobble;               /* Gobble point for each input segment */
 
-  Pgno iIndirect;
+  LsmPgno iIndirect;
   struct SavedPgno {
-    Pgno iPgno;
+    LsmPgno iPgno;
     int bStore;
   } aSave[2];
 };
@@ -371,7 +371,7 @@ void lsmPutU64(u8 *aOut, u64 nVal){
   aOut[7] = (u8)((nVal    ) & 0xFF);
 }
 
-static int sortedBlobGrow(lsm_env *pEnv, Blob *pBlob, int nData){
+static int sortedBlobGrow(lsm_env *pEnv, LsmBlob *pBlob, int nData){
   assert( pBlob->pEnv==pEnv || (pBlob->pEnv==0 && pBlob->pData==0) );
   if( pBlob->nAlloc<nData ){
     pBlob->pData = lsmReallocOrFree(pEnv, pBlob->pData, nData);
@@ -382,7 +382,7 @@ static int sortedBlobGrow(lsm_env *pEnv, Blob *pBlob, int nData){
   return LSM_OK;
 }
 
-static int sortedBlobSet(lsm_env *pEnv, Blob *pBlob, void *pData, int nData){
+static int sortedBlobSet(lsm_env *pEnv, LsmBlob *pBlob, void *pData, int nData){
   if( sortedBlobGrow(pEnv, pBlob, nData) ) return LSM_NOMEM;
   memcpy(pBlob->pData, pData, nData);
   pBlob->nData = nData;
@@ -390,15 +390,15 @@ static int sortedBlobSet(lsm_env *pEnv, Blob *pBlob, void *pData, int nData){
 }
 
 #if 0
-static int sortedBlobCopy(Blob *pDest, Blob *pSrc){
+static int sortedBlobCopy(LsmBlob *pDest, LsmBlob *pSrc){
   return sortedBlobSet(pDest, pSrc->pData, pSrc->nData);
 }
 #endif
 
-static void sortedBlobFree(Blob *pBlob){
+static void sortedBlobFree(LsmBlob *pBlob){
   assert( pBlob->pEnv || pBlob->pData==0 );
   if( pBlob->pData ) lsmFree(pBlob->pEnv, pBlob->pData);
-  memset(pBlob, 0, sizeof(Blob));
+  memset(pBlob, 0, sizeof(LsmBlob));
 }
 
 static int sortedReadData(
@@ -407,7 +407,7 @@ static int sortedReadData(
   int iOff,
   int nByte,
   void **ppData,
-  Blob *pBlob
+  LsmBlob *pBlob
 ){
   int rc = LSM_OK;
   int iEnd;
@@ -481,8 +481,8 @@ static int pageGetNRec(u8 *aData, int nData){
   return (int)lsmGetU16(&aData[SEGMENT_NRECORD_OFFSET(nData)]);
 }
 
-static Pgno pageGetPtr(u8 *aData, int nData){
+static LsmPgno pageGetPtr(u8 *aData, int nData){
-  return (Pgno)lsmGetU64(&aData[SEGMENT_POINTER_OFFSET(nData)]);
+  return (LsmPgno)lsmGetU64(&aData[SEGMENT_POINTER_OFFSET(nData)]);
 }
 
 static int pageGetFlags(u8 *aData, int nData){
@@ -506,8 +506,8 @@ static int pageObjGetNRec(Page *pPg){
 ** Return the decoded (possibly relative) pointer value stored in cell 
 ** iCell from page aData/nData.
 */
-static Pgno pageGetRecordPtr(u8 *aData, int nData, int iCell){
+static LsmPgno pageGetRecordPtr(u8 *aData, int nData, int iCell){
-  Pgno iRet;                      /* Return value */
+  LsmPgno iRet;                   /* Return value */
   u8 *aCell;                      /* Pointer to cell iCell */
 
   assert( iCell<pageGetNRec(aData, nData) && iCell>=0 );
@@ -522,7 +522,7 @@ static u8 *pageGetKey(
   int iCell,                      /* Index of cell on page to read */
   int *piTopic,                   /* OUT: Topic associated with this key */
   int *pnKey,                     /* OUT: Size of key in bytes */
-  Blob *pBlob                     /* If required, use this for dynamic memory */
+  LsmBlob *pBlob                  /* If required, use this for dynamic memory */
 ){
   u8 *pKey;
   int nDummy;
@@ -554,7 +554,7 @@ static int pageGetKeyCopy(
   Page *pPg,                      /* Page to read from */
   int iCell,                      /* Index of cell on page to read */
   int *piTopic,                   /* OUT: Topic associated with this key */
-  Blob *pBlob                     /* If required, use this for dynamic memory */
+  LsmBlob *pBlob                  /* If required, use this for dynamic memory */
 ){
   int rc = LSM_OK;
   int nKey;
@@ -569,8 +569,8 @@ static int pageGetKeyCopy(
   return rc;
 }
 
-static Pgno pageGetBtreeRef(Page *pPg, int iKey){
+static LsmPgno pageGetBtreeRef(Page *pPg, int iKey){
-  Pgno iRef;
+  LsmPgno iRef;
   u8 *aData;
   int nData;
   u8 *aCell;
@@ -592,11 +592,11 @@ static int pageGetBtreeKey(
   Segment *pSeg,                  /* Segment page pPg belongs to */
   Page *pPg,
   int iKey, 
-  Pgno *piPtr, 
+  LsmPgno *piPtr, 
   int *piTopic, 
   void **ppKey,
   int *pnKey,
-  Blob *pBlob
+  LsmBlob *pBlob
 ){
   u8 *aData;
   int nData;
@@ -613,7 +613,7 @@ static int pageGetBtreeKey(
 
   if( eType==0 ){
     int rc;
-    Pgno iRef;                  /* Page number of referenced page */
+    LsmPgno iRef;               /* Page number of referenced page */
     Page *pRef;
     aCell += GETVARINT64(aCell, iRef);
     rc = lsmFsDbPageGet(lsmPageFS(pPg), pSeg, iRef, &pRef);
@@ -638,7 +638,7 @@ static int btreeCursorLoadKey(BtreeCursor *pCsr){
     pCsr->nKey = 0;
     pCsr->eType = 0;
   }else{
-    Pgno dummy;
+    LsmPgno dummy;
     int iPg = pCsr->iPg;
     int iCell = pCsr->aPg[iPg].iCell;
     while( iCell<0 && (--iPg)>=0 ){
@@ -683,7 +683,7 @@ static int btreeCursorNext(BtreeCursor *pCsr){
   assert( pPg->iCell<=nCell );
   pPg->iCell++;
   if( pPg->iCell==nCell ){
-    Pgno iLoad;
+    LsmPgno iLoad;
 
     /* Up to parent. */
     lsmFsPageRelease(pPg->pPage);
@@ -842,7 +842,7 @@ static int btreeCursorRestore(
   if( p->iPg ){
     lsm_env *pEnv = lsmFsEnv(pCsr->pFS);
     int iCell;                    /* Current cell number on leaf page */
-    Pgno iLeaf;                   /* Page number of current leaf page */
+    LsmPgno iLeaf;                /* Page number of current leaf page */
     int nDepth;                   /* Depth of b-tree structure */
     Segment *pSeg = pCsr->pSeg;
 
@@ -866,7 +866,7 @@ static int btreeCursorRestore(
 
     /* Populate any other aPg[] array entries */
     if( rc==LSM_OK && nDepth>1 ){
-      Blob blob = {0,0,0};
+      LsmBlob blob = {0,0,0};
       void *pSeek;
       int nSeek;
       int iTopicSeek;
@@ -883,7 +883,7 @@ static int btreeCursorRestore(
         pSeek = 0;
         nSeek = 0;
       }else{
-        Pgno dummy;
+        LsmPgno dummy;
         rc = pageGetBtreeKey(pSeg, pPg,
             0, &dummy, &iTopicSeek, &pSeek, &nSeek, &pCsr->blob
         );
@@ -912,7 +912,7 @@ static int btreeCursorRestore(
             int iTry = (iMin+iMax)/2;
             void *pKey; int nKey;         /* Key for cell iTry */
             int iTopic;                   /* Topic for key pKeyT/nKeyT */
-            Pgno iPtr;                    /* Pointer for cell iTry */
+            LsmPgno iPtr;                 /* Pointer for cell iTry */
             int res;                      /* (pSeek - pKeyT) */
 
             rc = pageGetBtreeKey(
@@ -955,7 +955,7 @@ static int btreeCursorRestore(
       aData = fsPageData(pBtreePg->pPage, &nData);
       pCsr->iPtr = btreeCursorPtr(aData, nData, pBtreePg->iCell+1);
       if( pBtreePg->iCell<0 ){
-        Pgno dummy;
+        LsmPgno dummy;
         int i;
         for(i=pCsr->iPg-1; i>=0; i--){
           if( pCsr->aPg[i].iCell>0 ) break;
@@ -1030,7 +1030,7 @@ static int segmentPtrReadData(
   int iOff,
   int nByte,
   void **ppData,
-  Blob *pBlob
+  LsmBlob *pBlob
 ){
   return sortedReadData(pPtr->pSeg, pPtr->pPg, iOff, nByte, ppData, pBlob);
 }
@@ -1123,7 +1123,7 @@ static void sortedSplitkey(lsm_db *pDb, Level *pLevel, int *pRc){
   }
   if( rc==LSM_OK ){
     int iTopic;
-    Blob blob = {0, 0, 0, 0};
+    LsmBlob blob = {0, 0, 0, 0};
     u8 *aData;
     int nData;
   
@@ -1131,7 +1131,7 @@ static void sortedSplitkey(lsm_db *pDb, Level *pLevel, int *pRc){
     if( pageGetFlags(aData, nData) & SEGMENT_BTREE_FLAG ){
       void *pKey;
       int nKey;
-      Pgno dummy;
+      LsmPgno dummy;
       rc = pageGetBtreeKey(pSeg,
           pPg, pMerge->splitkey.iCell, &dummy, &iTopic, &pKey, &nKey, &blob
       );
@@ -1342,7 +1342,7 @@ static int assertKeyLocation(
   void *pKey, int nKey
 ){
   lsm_env *pEnv = lsmFsEnv(pCsr->pDb->pFS);
-  Blob blob = {0, 0, 0};
+  LsmBlob blob = {0, 0, 0};
   int eDir;
   int iTopic = 0;                 /* TODO: Fix me */
 
@@ -1488,7 +1488,7 @@ static int ptrFwdPointer(
   Page *pPage,
   int iCell,
   Segment *pSeg,
-  Pgno *piPtr,
+  LsmPgno *piPtr,
   int *pbFound
 ){
   Page *pPg = pPage;
@@ -1573,14 +1573,14 @@ static int sortedRhsFirst(MultiCursor *pCsr, Level *pLvl, SegmentPtr *pPtr){
 static int segmentPtrFwdPointer(
   MultiCursor *pCsr,              /* Multi-cursor pPtr belongs to */
   SegmentPtr *pPtr,               /* Segment-pointer to extract FC ptr from */
-  Pgno *piPtr                     /* OUT: FC pointer value */
+  LsmPgno *piPtr                  /* OUT: FC pointer value */
 ){
   Level *pLvl = pPtr->pLevel;
   Level *pNext = pLvl->pNext;
   Page *pPg = pPtr->pPg;
   int rc;
   int bFound;
-  Pgno iOut = 0;
+  LsmPgno iOut = 0;
 
   if( pPtr->pSeg==&pLvl->lhs || pPtr->pSeg==&pLvl->aRhs[pLvl->nRight-1] ){
     if( pNext==0 
@@ -1641,7 +1641,7 @@ static int segmentPtrSeek(
   int rc = LSM_OK;
   int iMin;
   int iMax;
-  Pgno iPtrOut = 0;
+  LsmPgno iPtrOut = 0;
 
   /* If the current page contains an oversized entry, then there are no
   ** pointers to one or more of the subsequent pages in the sorted run.
@@ -1768,18 +1768,18 @@ static int seekInBtree(
   Segment *pSeg,                  /* Seek within this segment */
   int iTopic,
   void *pKey, int nKey,           /* Key to seek to */
-  Pgno *aPg,                      /* OUT: Page numbers */
+  LsmPgno *aPg,                   /* OUT: Page numbers */
   Page **ppPg                     /* OUT: Leaf (sorted-run) page reference */
 ){
   int i = 0;
   int rc;
   int iPg;
   Page *pPg = 0;
-  Blob blob = {0, 0, 0};
+  LsmBlob blob = {0, 0, 0};
 
   iPg = (int)pSeg->iRoot;
   do {
-    Pgno *piFirst = 0;
+    LsmPgno *piFirst = 0;
     if( aPg ){
       aPg[i++] = iPg;
       piFirst = &aPg[i];
@@ -1808,7 +1808,7 @@ static int seekInBtree(
         int iTry = (iMin+iMax)/2;
         void *pKeyT; int nKeyT;       /* Key for cell iTry */
         int iTopicT;                  /* Topic for key pKeyT/nKeyT */
-        Pgno iPtr;                    /* Pointer associated with cell iTry */
+        LsmPgno iPtr;                 /* Pointer associated with cell iTry */
         int res;                      /* (pKey - pKeyT) */
 
         rc = pageGetBtreeKey(
@@ -1899,7 +1899,7 @@ static int seekInLevel(
   int eSeek,                      /* Search bias - see above */
   int iTopic,                     /* Key topic to search for */
   void *pKey, int nKey,           /* Key to search for */
-  Pgno *piPgno,                   /* IN/OUT: fraction cascade pointer (or 0) */
+  LsmPgno *piPgno,                /* IN/OUT: fraction cascade pointer (or 0) */
   int *pbStop                     /* OUT: See above */
 ){
   Level *pLvl = aPtr[0].pLevel;   /* Level to seek within */
@@ -1922,6 +1922,7 @@ static int seekInLevel(
   ** is not a composite level and there is no split-key). Search the 
   ** left-hand-side of the level in this case.  */
   if( res<0 ){
+    int i;
     int iPtr = 0;
     if( nRhs==0 ) iPtr = (int)*piPgno;
 
@@ -1931,12 +1932,16 @@ static int seekInLevel(
     if( rc==LSM_OK && nRhs>0 && eSeek==LSM_SEEK_GE && aPtr[0].pPg==0 ){
       res = 0;
     }
+    for(i=1; i<=nRhs; i++){
+      segmentPtrReset(&aPtr[i], LSM_SEGMENTPTR_FREE_THRESHOLD);
+    }
   }
   
   if( res>=0 ){
     int bHit = 0;                 /* True if at least one rhs is not EOF */
     int iPtr = (int)*piPgno;
     int i;
+    segmentPtrReset(&aPtr[0], LSM_SEGMENTPTR_FREE_THRESHOLD);
     for(i=1; rc==LSM_OK && i<=nRhs && bStop==0; i++){
       SegmentPtr *pPtr = &aPtr[i];
       iOut = 0;
@@ -2868,7 +2873,7 @@ static int multiCursorEnd(MultiCursor *pCsr, int bLast){
   int rc = LSM_OK;
   int i;
 
-  pCsr->flags &= ~(CURSOR_NEXT_OK | CURSOR_PREV_OK);
+  pCsr->flags &= ~(CURSOR_NEXT_OK | CURSOR_PREV_OK | CURSOR_SEEK_EQ);
   pCsr->flags |= (bLast ? CURSOR_PREV_OK : CURSOR_NEXT_OK);
   pCsr->iFree = 0;
 
@@ -3055,7 +3060,7 @@ int lsmMCursorSeek(
   int bStop = 0;                  /* Set to true to halt search operation */
   int rc = LSM_OK;                /* Return code */
   int iPtr = 0;                   /* Used to iterate through pCsr->aPtr[] */
-  Pgno iPgno = 0;                 /* FC pointer value */
+  LsmPgno iPgno = 0;              /* FC pointer value */
 
   assert( pCsr->apTreeCsr[0]==0 || iTopic==0 );
   assert( pCsr->apTreeCsr[1]==0 || iTopic==0 );
@@ -3537,7 +3542,7 @@ static int mergeWorkerLoadHierarchy(MergeWorker *pMW){
 **         + Type byte (always SORTED_SEPARATOR or SORTED_SYSTEM_SEPARATOR),
 **         + Absolute pointer value (varint),
 **         + Number of bytes in key (varint),
-**         + Blob containing key data.
+**         + LsmBlob containing key data.
 **
 **   2. All pointer values are stored as absolute values (not offsets 
 **      relative to the footer pointer value).
@@ -3571,8 +3576,8 @@ static int mergeWorkerLoadHierarchy(MergeWorker *pMW){
 static int mergeWorkerBtreeWrite(
   MergeWorker *pMW,
   u8 eType,
-  Pgno iPtr,
+  LsmPgno iPtr,
-  Pgno iKeyPg,
+  LsmPgno iKeyPg,
   void *pKey,
   int nKey
 ){
@@ -3682,7 +3687,7 @@ static int mergeWorkerBtreeWrite(
 static int mergeWorkerBtreeIndirect(MergeWorker *pMW){
   int rc = LSM_OK;
   if( pMW->iIndirect ){
-    Pgno iKeyPg = pMW->aSave[1].iPgno;
+    LsmPgno iKeyPg = pMW->aSave[1].iPgno;
     rc = mergeWorkerBtreeWrite(pMW, 0, pMW->iIndirect, iKeyPg, 0, 0);
     pMW->iIndirect = 0;
   }
@@ -3703,7 +3708,7 @@ static int mergeWorkerPushHierarchy(
   int nKey                        /* Size of pKey buffer in bytes */
 ){
   int rc = LSM_OK;                /* Return Code */
-  Pgno iPtr;                      /* Pointer value to accompany pKey/nKey */
+  LsmPgno iPtr;                   /* Pointer value to accompany pKey/nKey */
 
   assert( pMW->aSave[0].bStore==0 );
   assert( pMW->aSave[1].bStore==0 );
@@ -3734,7 +3739,7 @@ static int mergeWorkerFinishHierarchy(
 ){
   int i;                          /* Used to loop through apHier[] */
   int rc = LSM_OK;                /* Return code */
-  Pgno iPtr;                      /* New right-hand-child pointer value */
+  LsmPgno iPtr;                   /* New right-hand-child pointer value */
 
   iPtr = pMW->aSave[0].iPgno;
   for(i=0; i<pMW->hier.nHier && rc==LSM_OK; i++){
@@ -3830,7 +3835,7 @@ static int mergeWorkerPersistAndRelease(MergeWorker *pMW){
 */
 static int mergeWorkerNextPage(
   MergeWorker *pMW,               /* Merge worker object to append page to */
-  Pgno iFPtr                      /* Pointer value for footer of new page */
+  LsmPgno iFPtr                   /* Pointer value for footer of new page */
 ){
   int rc = LSM_OK;                /* Return code */
   Page *pNext = 0;                /* New page appended to run */
@@ -3999,6 +4004,11 @@ static int mergeWorkerWrite(
     ** marked read-only, advance to the next page of the output run. */
     iOff = pMerge->iOutputOff;
     if( iOff<0 || pPg==0 || iOff+nHdr > SEGMENT_EOF(nData, nRec+1) ){
+      if( iOff>=0 && pPg ){
+        /* Zero any free space on the page */
+        assert( aData );
+        memset(&aData[iOff], 0, SEGMENT_EOF(nData, nRec)-iOff);
+      }
       iFPtr = (int)*pMW->pCsr->pPrevMergePtr;
       iRPtr = iPtr - iFPtr;
       iOff = 0;
@@ -4069,36 +4079,49 @@ static void mergeWorkerShutdown(MergeWorker *pMW, int *pRc){
   /* Unless the merge has finished, save the cursor position in the
   ** Merge.aInput[] array. See function mergeWorkerInit() for the 
   ** code to restore a cursor position based on aInput[].  */
-  if( rc==LSM_OK && pCsr && lsmMCursorValid(pCsr) ){
+  if( rc==LSM_OK && pCsr ){
     Merge *pMerge = pMW->pLevel->pMerge;
-    int bBtree = (pCsr->pBtCsr!=0);
+    if( lsmMCursorValid(pCsr) ){
-    int iPtr;
+      int bBtree = (pCsr->pBtCsr!=0);
+      int iPtr;
 
-    /* pMerge->nInput==0 indicates that this is a FlushTree() operation. */
+      /* pMerge->nInput==0 indicates that this is a FlushTree() operation. */
-    assert( pMerge->nInput==0 || pMW->pLevel->nRight>0 );
+      assert( pMerge->nInput==0 || pMW->pLevel->nRight>0 );
-    assert( pMerge->nInput==0 || pMerge->nInput==(pCsr->nPtr+bBtree) );
+      assert( pMerge->nInput==0 || pMerge->nInput==(pCsr->nPtr+bBtree) );
 
-    for(i=0; i<(pMerge->nInput-bBtree); i++){
+      for(i=0; i<(pMerge->nInput-bBtree); i++){
-      SegmentPtr *pPtr = &pCsr->aPtr[i];
+        SegmentPtr *pPtr = &pCsr->aPtr[i];
-      if( pPtr->pPg ){
+        if( pPtr->pPg ){
-        pMerge->aInput[i].iPg = lsmFsPageNumber(pPtr->pPg);
+          pMerge->aInput[i].iPg = lsmFsPageNumber(pPtr->pPg);
-        pMerge->aInput[i].iCell = pPtr->iCell;
+          pMerge->aInput[i].iCell = pPtr->iCell;
+        }else{
+          pMerge->aInput[i].iPg = 0;
+          pMerge->aInput[i].iCell = 0;
+        }
+      }
+      if( bBtree && pMerge->nInput ){
+        assert( i==pCsr->nPtr );
+        btreeCursorPosition(pCsr->pBtCsr, &pMerge->aInput[i]);
+      }
+
+      /* Store the location of the split-key */
+      iPtr = pCsr->aTree[1] - CURSOR_DATA_SEGMENT;
+      if( iPtr<pCsr->nPtr ){
+        pMerge->splitkey = pMerge->aInput[iPtr];
       }else{
-        pMerge->aInput[i].iPg = 0;
+        btreeCursorSplitkey(pCsr->pBtCsr, &pMerge->splitkey);
-        pMerge->aInput[i].iCell = 0;
       }
     }
-    if( bBtree && pMerge->nInput ){
-      assert( i==pCsr->nPtr );
-      btreeCursorPosition(pCsr->pBtCsr, &pMerge->aInput[i]);
-    }
 
-    /* Store the location of the split-key */
+    /* Zero any free space left on the final page. This helps with
-    iPtr = pCsr->aTree[1] - CURSOR_DATA_SEGMENT;
+    ** compression if using a compression hook. And prevents valgrind
-    if( iPtr<pCsr->nPtr ){
+    ** from complaining about uninitialized byte passed to write(). */
-      pMerge->splitkey = pMerge->aInput[iPtr];
+    if( pMW->pPage ){
-    }else{
+      int nData;
-      btreeCursorSplitkey(pCsr->pBtCsr, &pMerge->splitkey);
+      u8 *aData = fsPageData(pMW->pPage, &nData);
+      int iOff = pMerge->iOutputOff;
+      int iEof = SEGMENT_EOF(nData, pageGetNRec(aData, nData));
+      memset(&aData[iOff], 0, iEof - iOff);
     }
     
     pMerge->iOutputOff = -1;
@@ -4200,7 +4223,7 @@ static int mergeWorkerStep(MergeWorker *pMW){
   int rc = LSM_OK;              /* Return code */
   int eType;                    /* SORTED_SEPARATOR, WRITE or DELETE */
   void *pKey; int nKey;         /* Key */
-  Pgno iPtr;
+  LsmPgno iPtr;
   int iVal;
 
   pCsr = pMW->pCsr;
@@ -4353,7 +4376,7 @@ static int sortedNewToplevel(
   if( rc!=LSM_OK ){
     lsmMCursorClose(pCsr, 0);
   }else{
-    Pgno iLeftPtr = 0;
+    LsmPgno iLeftPtr = 0;
     Merge merge;                  /* Merge object used to create new level */
     MergeWorker mergeworker;      /* MergeWorker object for the same purpose */
 
@@ -4530,7 +4553,7 @@ static int mergeWorkerInit(
   memset(pMW, 0, sizeof(MergeWorker));
   pMW->pDb = pDb;
   pMW->pLevel = pLevel;
-  pMW->aGobble = lsmMallocZeroRc(pDb->pEnv, sizeof(Pgno) * pLevel->nRight, &rc);
+  pMW->aGobble = lsmMallocZeroRc(pDb->pEnv, sizeof(LsmPgno)*pLevel->nRight,&rc);
 
   /* Create a multi-cursor to read the data to write to the new
   ** segment. The new segment contains:
@@ -4612,7 +4635,7 @@ static int sortedBtreeGobble(
   int rc = LSM_OK;
   if( rtTopic(pCsr->eType)==0 ){
     Segment *pSeg = pCsr->aPtr[iGobble].pSeg;
-    Pgno *aPg;
+    LsmPgno *aPg;
     int nPg;
 
     /* Seek from the root of the b-tree to the segment leaf that may contain
@@ -4621,7 +4644,7 @@ static int sortedBtreeGobble(
     ** gobbled up to (but not including) the first of these page numbers.
     */
     assert( pSeg->iRoot>0 );
-    aPg = lsmMallocZeroRc(pDb->pEnv, sizeof(Pgno)*32, &rc);
+    aPg = lsmMallocZeroRc(pDb->pEnv, sizeof(LsmPgno)*32, &rc);
     if( rc==LSM_OK ){
       rc = seekInBtree(pCsr, pSeg, 
           rtTopic(pCsr->eType), pCsr->key.pData, pCsr->key.nData, aPg, 0
@@ -5234,16 +5257,15 @@ static int doLsmSingleWork(
   /* If the in-memory part of the free-list is too large, write a new 
   ** top-level containing just the in-memory free-list entries to disk. */
   if( rc==LSM_OK && pDb->pWorker->freelist.nEntry > pDb->nMaxFreelist ){
-    int nPg = 0;
     while( rc==LSM_OK && lsmDatabaseFull(pDb) ){
+      int nPg = 0;
       rc = sortedWork(pDb, 16, nMerge, 1, &nPg);
       nRem -= nPg;
     }
     if( rc==LSM_OK ){
       rc = sortedNewFreelistOnly(pDb);
     }
-    nRem -= nPg;
+    bDirty = 1;
-    if( nPg ) bDirty = 1;
   }
 
   if( rc==LSM_OK ){
@@ -5448,9 +5470,9 @@ int lsmFlushTreeToDisk(lsm_db *pDb){
 */
 static char *segToString(lsm_env *pEnv, Segment *pSeg, int nMin){
   int nSize = pSeg->nSize;
-  Pgno iRoot = pSeg->iRoot;
+  LsmPgno iRoot = pSeg->iRoot;
-  Pgno iFirst = pSeg->iFirst;
+  LsmPgno iFirst = pSeg->iFirst;
-  Pgno iLast = pSeg->iLastPg;
+  LsmPgno iLast = pSeg->iLastPg;
   char *z;
 
   char *z1;
@@ -5509,7 +5531,7 @@ static int fileToString(
 }
 
 void sortedDumpPage(lsm_db *pDb, Segment *pRun, Page *pPg, int bVals){
-  Blob blob = {0, 0, 0};         /* Blob used for keys */
+  LsmBlob blob = {0, 0, 0};       /* LsmBlob used for keys */
   LsmString s;
   int i;
 
@@ -5545,7 +5567,7 @@ void sortedDumpPage(lsm_db *pDb, Segment *pRun, Page *pPg, int bVals){
     aCell += lsmVarintGet32(aCell, &iPgPtr);
 
     if( eType==0 ){
-      Pgno iRef;                  /* Page number of referenced page */
+      LsmPgno iRef;               /* Page number of referenced page */
       aCell += lsmVarintGet64(aCell, &iRef);
       lsmFsDbPageGet(pDb->pFS, pRun, iRef, &pRef);
       aKey = pageGetKey(pRun, pRef, 0, &iTopic, &nKey, &blob);
@@ -5589,7 +5611,7 @@ static void infoCellDump(
   int *piPgPtr,
   u8 **paKey, int *pnKey,
   u8 **paVal, int *pnVal,
-  Blob *pBlob
+  LsmBlob *pBlob
 ){
   u8 *aData; int nData;           /* Page data */
   u8 *aKey; int nKey = 0;         /* Key */
@@ -5607,7 +5629,7 @@ static void infoCellDump(
 
   if( eType==0 ){
     int dummy;
-    Pgno iRef;                  /* Page number of referenced page */
+    LsmPgno iRef;                 /* Page number of referenced page */
     aCell += lsmVarintGet64(aCell, &iRef);
     if( bIndirect ){
       lsmFsDbPageGet(pDb->pFS, pSeg, iRef, &pRef);
@@ -5653,7 +5675,7 @@ static int infoAppendBlob(LsmString *pStr, int bHex, u8 *z, int n){
 
 static int infoPageDump(
   lsm_db *pDb,                    /* Database handle */
-  Pgno iPg,                       /* Page number of page to dump */
+  LsmPgno iPg,                    /* Page number of page to dump */
   int flags,
   char **pzOut                    /* OUT: lsmMalloc'd string */
 ){
@@ -5694,7 +5716,7 @@ static int infoPageDump(
   }
 
   if( rc==LSM_OK ){
-    Blob blob = {0, 0, 0, 0};
+    LsmBlob blob = {0, 0, 0, 0};
     int nKeyWidth = 0;
     LsmString str;
     int nRec;
@@ -5729,7 +5751,7 @@ static int infoPageDump(
       u8 *aVal; int nVal = 0;       /* Value */
       int iPgPtr;
       int eType;
-      Pgno iAbsPtr;
+      LsmPgno iAbsPtr;
       char zFlags[8];
 
       infoCellDump(pDb, pSeg, bIndirect, pPg, iCell, &eType, &iPgPtr,
@@ -5795,7 +5817,7 @@ static int infoPageDump(
 
 int lsmInfoPageDump(
   lsm_db *pDb,                    /* Database handle */
-  Pgno iPg,                       /* Page number of page to dump */
+  LsmPgno iPg,                    /* Page number of page to dump */
   int bHex,                       /* True to output key/value in hex form */
   char **pzOut                    /* OUT: lsmMalloc'd string */
 ){
@@ -5971,8 +5993,8 @@ void lsmSortedExpandBtreePage(Page *pPg, int nOrig){
 #ifdef LSM_DEBUG_EXPENSIVE
 static void assertRunInOrder(lsm_db *pDb, Segment *pSeg){
   Page *pPg = 0;
-  Blob blob1 = {0, 0, 0, 0};
+  LsmBlob blob1 = {0, 0, 0, 0};
-  Blob blob2 = {0, 0, 0, 0};
+  LsmBlob blob2 = {0, 0, 0, 0};
 
   lsmFsDbPageGet(pDb->pFS, pSeg, pSeg->iFirst, &pPg);
   while( pPg ){
@@ -6034,7 +6056,7 @@ static int assertPointersOk(
   int rc = LSM_OK;                /* Error code */
   SegmentPtr ptr1;                /* Iterates through pOne */
   SegmentPtr ptr2;                /* Iterates through pTwo */
-  Pgno iPrev;
+  LsmPgno iPrev;
 
   assert( pOne && pTwo );
 
@@ -6057,7 +6079,7 @@ static int assertPointersOk(
   }
       
   while( rc==LSM_OK && ptr2.pPg ){
-    Pgno iThis;
+    LsmPgno iThis;
 
     /* Advance to the next page of segment pTwo that contains at least
     ** one cell. Break out of the loop if the iterator reaches EOF.  */
@@ -6119,7 +6141,7 @@ static int assertBtreeOk(
 ){
   int rc = LSM_OK;                /* Return code */
   if( pSeg->iRoot ){
-    Blob blob = {0, 0, 0};        /* Buffer used to cache overflow keys */
+    LsmBlob blob = {0, 0, 0};     /* Buffer used to cache overflow keys */
     FileSystem *pFS = pDb->pFS;   /* File system to read from */
     Page *pPg = 0;                /* Main run page */
     BtreeCursor *pCsr = 0;        /* Btree cursor */

ext/lsm1/lsm_vtab.c

@@ -10,16 +10,54 @@
 **
 *************************************************************************
 **
-** This file implements a simple virtual table wrapper around the LSM
+** This file implements a virtual table for SQLite3 around the LSM
 ** storage engine from SQLite4.
 **
 ** USAGE
 **
 **   CREATE VIRTUAL TABLE demo USING lsm1(filename,key,keytype,value1,...);
 **
+** The filename parameter is the name of the LSM database file, which is
+** separate and distinct from the SQLite3 database file.
+**
 ** The keytype must be one of: UINT, TEXT, BLOB.  All keys must be of that
-** one type.  "UINT" means unsigned integer.  The values may be any
+** one type.  "UINT" means unsigned integer.  The values may be of any
-** SQLite datatype.
+** SQLite datatype: BLOB, TEXT, INTEGER, FLOAT, or NULL.
+**
+** The virtual table contains read-only hidden columns:
+**
+**     lsm1_key	      A BLOB which is the raw LSM key.  If the "keytype"
+**                    is BLOB or TEXT then this column is exactly the
+**                    same as the key.  For the UINT keytype, this column
+**                    will be a variable-length integer encoding of the key.
+**
+**     lsm1_value     A BLOB which is the raw LSM value.  All of the value
+**                    columns are packed into this BLOB using the encoding
+**                    described below.
+**
+** Attempts to write values into the lsm1_key and lsm1_value columns are
+** silently ignored.
+**
+** EXAMPLE
+**
+** The virtual table declared this way:
+**
+**    CREATE VIRTUAL TABLE demo2 USING lsm1('x.lsm',id,UINT,a,b,c,d);
+**
+** Results in a new virtual table named "demo2" that acts as if it has
+** the following schema:
+**
+**    CREATE TABLE demo2(
+**      id UINT PRIMARY KEY ON CONFLICT REPLACE,
+**      a ANY,
+**      b ANY,
+**      c ANY,
+**      d ANY,
+**      lsm1_key BLOB HIDDEN,
+**      lsm1_value BLOB HIDDEN
+**    ) WITHOUT ROWID;
+**
+** 
 **
 ** INTERNALS
 **
@@ -243,13 +281,16 @@ static int lsm1Connect(
   lsm1VblobAppendText(&sql, argv[4]);
   lsm1VblobAppendText(&sql, " ");
   lsm1VblobAppendText(&sql, argv[5]);
+  lsm1VblobAppendText(&sql, " PRIMARY KEY");
   for(i=6; i<argc; i++){
     lsm1VblobAppendText(&sql, ", ");
     lsm1VblobAppendText(&sql, argv[i]);
     pNew->nVal++;
   }
   lsm1VblobAppendText(&sql, 
-      ", lsm1_command HIDDEN, lsm1_key HIDDEN, lsm1_value HIDDEN)");
+      ", lsm1_command HIDDEN"
+      ", lsm1_key HIDDEN"
+      ", lsm1_value HIDDEN) WITHOUT ROWID");
   lsm1VblobAppend(&sql, (u8*)"", 1);
   if( sql.errNoMem ){
     rc = SQLITE_NOMEM;
@@ -669,6 +710,31 @@ static int lsm1Column(
   return SQLITE_OK;
 }
 
+/* Parameter "pValue" contains an SQL value that is to be used as
+** a key in an LSM table.  The type of the key is determined by
+** "keyType".  Extract the raw bytes used for the key in LSM1.
+*/
+static void lsm1KeyFromValue(
+  int keyType,                 /* The key type */
+  sqlite3_value *pValue,       /* The key value */
+  u8 *pBuf,                    /* Storage space for a generated key */
+  const u8 **ppKey,            /* OUT: the bytes of the key */
+  int *pnKey                   /* OUT: size of the key */
+){
+  if( keyType==SQLITE_BLOB ){
+    *ppKey = (const u8*)sqlite3_value_blob(pValue);
+    *pnKey = sqlite3_value_bytes(pValue);
+  }else if( keyType==SQLITE_TEXT ){
+    *ppKey = (const u8*)sqlite3_value_text(pValue);
+    *pnKey = sqlite3_value_bytes(pValue);
+  }else{
+    sqlite3_int64 v = sqlite3_value_int64(pValue);
+    if( v<0 ) v = 0;
+    *pnKey = lsm1PutVarint64(pBuf, v);
+    *ppKey = pBuf;
+  }
+}
+
 /* Move to the first row to return.
 */
 static int lsm1Filter(
@@ -680,7 +746,7 @@ static int lsm1Filter(
   lsm1_vtab *pTab = (lsm1_vtab*)(pCur->base.pVtab);
   int rc = LSM_OK;
   int seekType = -1;
-  const void *pVal = 0;
+  const u8 *pVal = 0;
   int nVal;
   u8 keyType = pTab->keyType;
   u8 aKey1[16];
@@ -689,18 +755,7 @@ static int lsm1Filter(
   sqlite3_free(pCur->pKey2);
   pCur->pKey2 = 0;
   if( idxNum<99 ){
-    if( keyType==SQLITE_BLOB ){
+    lsm1KeyFromValue(keyType, argv[0], aKey1, &pVal, &nVal);
-      pVal = sqlite3_value_blob(argv[0]);
-      nVal = sqlite3_value_bytes(argv[0]);
-    }else if( keyType==SQLITE_TEXT ){
-      pVal = sqlite3_value_text(argv[0]);
-      nVal = sqlite3_value_bytes(argv[0]);
-    }else{
-      sqlite3_int64 v = sqlite3_value_int64(argv[0]);
-      if( v<0 ) v = 0;
-      nVal = lsm1PutVarint64(aKey1, v);
-      pVal = aKey1;
-    }
   }
   switch( idxNum ){
     case 0: {   /* key==argv[0] */
@@ -870,21 +925,30 @@ int lsm1Update(
   sqlite_int64 *pRowid
 ){
   lsm1_vtab *p = (lsm1_vtab*)pVTab;
-  int nKey;
+  int nKey, nKey2;
   int i;
   int rc = LSM_OK;
-  unsigned char *pKey;
+  const u8 *pKey, *pKey2;
   unsigned char aKey[16];
   unsigned char pSpace[16];
   lsm1_vblob val;
 
   if( argc==1 ){
-    pVTab->zErrMsg = sqlite3_mprintf("cannot DELETE");
+    /* DELETE the record whose key is argv[0] */
-    return SQLITE_ERROR;
+    lsm1KeyFromValue(p->keyType, argv[0], aKey, &pKey, &nKey);
+    lsm_delete(p->pDb, pKey, nKey);
+    return SQLITE_OK;
   }
+
   if( sqlite3_value_type(argv[0])!=SQLITE_NULL ){
-    pVTab->zErrMsg = sqlite3_mprintf("cannot UPDATE");
+    /* An UPDATE */
-    return SQLITE_ERROR;
+    lsm1KeyFromValue(p->keyType, argv[0], aKey, &pKey, &nKey);
+    lsm1KeyFromValue(p->keyType, argv[1], pSpace, &pKey2, &nKey2);
+    if( nKey!=nKey2 || memcmp(pKey, pKey2, nKey)!=0 ){
+      /* The UPDATE changes the PRIMARY KEY value.  DELETE the old key */
+      lsm_delete(p->pDb, pKey, nKey);
+    }
+    /* Fall through into the INSERT case to complete the UPDATE */
   }
 
   /* "INSERT INTO tab(lsm1_command) VALUES('....')" is used to implement
@@ -893,22 +957,7 @@ int lsm1Update(
   if( sqlite3_value_type(argv[3+p->nVal])!=SQLITE_NULL ){
     return SQLITE_OK;
   }
-  if( p->keyType==SQLITE_BLOB ){
+  lsm1KeyFromValue(p->keyType, argv[2], aKey, &pKey, &nKey);
-    pKey = (u8*)sqlite3_value_blob(argv[2]);
-    nKey = sqlite3_value_bytes(argv[2]);
-  }else if( p->keyType==SQLITE_TEXT ){
-    pKey = (u8*)sqlite3_value_text(argv[2]);
-    nKey = sqlite3_value_bytes(argv[2]);
-  }else{
-    sqlite3_int64 v = sqlite3_value_int64(argv[2]);
-    if( v>=0 ){
-      nKey = lsm1PutVarint64(aKey, (sqlite3_uint64)v);
-      pKey = aKey;
-    }else{
-      pVTab->zErrMsg = sqlite3_mprintf("key must be non-negative");
-      return SQLITE_ERROR;
-    }
-  }
   memset(&val, 0, sizeof(val));
   for(i=0; i<p->nVal; i++){
     sqlite3_value *pArg = argv[3+i];

ext/lsm1/test/lsm1_simple.test

@@ -19,37 +19,75 @@ load_lsm1_vtab db
 
 forcedelete testlsm.db
 
-do_execsql_test 1.0 {
+do_execsql_test 100 {
   CREATE VIRTUAL TABLE x1 USING lsm1(testlsm.db,a,UINT,b,c,d);
   PRAGMA table_info(x1);
 } {
-  0 a UINT 0 {} 0 
+  0 a UINT 1 {} 1 
   1 b {} 0 {} 0 
   2 c {} 0 {} 0 
   3 d {} 0 {} 0
 }
 
-do_execsql_test 1.1 {
+do_execsql_test 110 {
   INSERT INTO x1(a,b,c,d) VALUES(15, 11, 22, 33),(8,'banjo',x'333231',NULL),
       (12,NULL,3.25,-559281390);
   SELECT a, quote(b), quote(c), quote(d) FROM x1;
 } {8 'banjo' X'333231' NULL 12 NULL 3.25 -559281390 15 11 22 33}
+do_execsql_test 111 {
+  SELECT a, quote(lsm1_key), quote(lsm1_value) FROM x1;
+} {8 X'08' X'2162616E6A6F1633323105' 12 X'0C' X'05320000000000000A401FFB42ABE9DB' 15 X'0F' X'4284C6'}
 
-do_catchsql_test 1.2 {
+do_execsql_test 120 {
   UPDATE x1 SET d = d+1.0 WHERE a=15;
-} {1 {cannot UPDATE}}
+  SELECT a, quote(b), quote(c), quote(d) FROM x1;
+} {8 'banjo' X'333231' NULL 12 NULL 3.25 -559281390 15 11 22 34.0}
 
-do_catchsql_test 1.3 {
+do_execsql_test 130 {
+  UPDATE x1 SET a=123456789 WHERE a=12;
+  SELECT a, quote(b), quote(c), quote(d) FROM x1;
+} {8 'banjo' X'333231' NULL 15 11 22 34.0 123456789 NULL 3.25 -559281390}
+do_execsql_test 131 {
+  SELECT quote(lsm1_key), printf('0x%x',a) FROM x1 WHERE a > 100000000;
+} {X'FB075BCD15' 0x75bcd15}
+
+do_execsql_test 140 {
   DELETE FROM x1 WHERE a=15;
-} {1 {cannot DELETE}}
+  SELECT a, quote(b), quote(c), quote(d) FROM x1;
+} {8 'banjo' X'333231' NULL 123456789 NULL 3.25 -559281390}
 
-do_test 1.4 {
+do_test 150 {
   lsort [glob testlsm.db*]
 } {testlsm.db testlsm.db-log testlsm.db-shm}
 
 db close
-do_test 1.5 {
+do_test 160 {
   lsort [glob testlsm.db*]
 } {testlsm.db}
 
+forcedelete testlsm.db
+forcedelete test.db
+sqlite3 db test.db
+load_lsm1_vtab db
+
+
+do_execsql_test 200 {
+  CREATE VIRTUAL TABLE x1 USING lsm1(testlsm.db,a,TEXT,b,c,d);
+  PRAGMA table_info(x1);
+} {
+  0 a TEXT 1 {} 1 
+  1 b {} 0 {} 0 
+  2 c {} 0 {} 0 
+  3 d {} 0 {} 0
+}
+do_execsql_test 210 {
+  INSERT INTO x1(a,b,c,d) VALUES(15, 11, 22, 33),(8,'banjo',x'333231',NULL),
+      (12,NULL,3.25,-559281390);
+  SELECT quote(a), quote(b), quote(c), quote(d), '|' FROM x1;
+} {'12' NULL 3.25 -559281390 | '15' 11 22 33 | '8' 'banjo' X'333231' NULL |}
+do_execsql_test 211 {
+  SELECT quote(a), quote(lsm1_key), quote(lsm1_value), '|' FROM x1;
+} {'12' X'3132' X'05320000000000000A401FFB42ABE9DB' | '15' X'3135' X'4284C6' | '8' X'38' X'2162616E6A6F1633323105' |}
+
+
 finish_test

ext/lsm1/tool/mklsm1c.tcl

@@ -0,0 +1,88 @@
+#!/bin/sh
+# restart with tclsh \
+exec tclsh "$0" "$@"
+
+set srcdir [file dirname [file dirname [info script]]]
+set G(src) [string map [list %dir% $srcdir] {
+  %dir%/lsm.h
+  %dir%/lsmInt.h
+  %dir%/lsm_vtab.c
+  %dir%/lsm_ckpt.c
+  %dir%/lsm_file.c
+  %dir%/lsm_log.c
+  %dir%/lsm_main.c
+  %dir%/lsm_mem.c
+  %dir%/lsm_mutex.c
+  %dir%/lsm_shared.c
+  %dir%/lsm_sorted.c
+  %dir%/lsm_str.c
+  %dir%/lsm_tree.c
+  %dir%/lsm_unix.c
+  %dir%/lsm_varint.c
+  %dir%/lsm_win32.c
+}]
+
+set G(hdr) {
+
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_LSM1) 
+
+#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) 
+# define NDEBUG 1
+#endif
+#if defined(NDEBUG) && defined(SQLITE_DEBUG)
+# undef NDEBUG
+#endif
+
+}
+
+set G(footer) {
+    
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_LSM1) */
+}
+
+#-------------------------------------------------------------------------
+# Read and return the entire contents of text file $zFile from disk.
+#
+proc readfile {zFile} {
+  set fd [open $zFile]
+  set data [read $fd]
+  close $fd
+  return $data
+}
+
+proc lsm1c_init {zOut} {
+  global G
+  set G(fd) stdout
+  set G(fd) [open $zOut w]
+
+  puts -nonewline $G(fd) $G(hdr)
+}
+
+proc lsm1c_printfile {zIn} {
+  global G
+  set data [readfile $zIn]
+  set zTail [file tail $zIn]
+  puts $G(fd) "#line 1 \"$zTail\""
+
+  foreach line [split $data "\n"] {
+    if {[regexp {^# *include.*lsm} $line]} {
+      set line "/* $line */"
+    } elseif { [regexp {^(const )?[a-zA-Z][a-zA-Z0-9]* [*]?lsm[^_]} $line] } {
+      set line "static $line"
+    }
+    puts $G(fd) $line
+  }
+}
+
+proc lsm1c_close {} {
+  global G
+  puts -nonewline $G(fd) $G(footer)
+  if {$G(fd)!="stdout"} {
+    close $G(fd)
+  }
+}
+
+
+lsm1c_init lsm1.c
+foreach f $G(src) { lsm1c_printfile $f }
+lsm1c_close

ext/misc/README.md

@@ -14,11 +14,20 @@ as follows:
      It is a good example of how to go about implementing a custom
      [table-valued function](https://www.sqlite.org/vtab.html#tabfunc2).
 
+  *  **csv.c** —  A [virtual table](https://sqlite.org/vtab.html)
+     for reading 
+     [Comma-Separated-Value (CSV) files](https://en.wikipedia.org/wiki/Comma-separated_values).
+
   *  **dbdump.c** —  This is not actually a loadable extension, but
      rather a library that implements an approximate equivalent to the
      ".dump" command of the
      [command-line shell](https://www.sqlite.org/cli.html).
 
+  *  **json1.c** —  Various SQL functions and table-valued functions
+     for processing JSON.  This extension is already built into the
+     [SQLite amalgamation](https://sqlite.org/amalgamation.html).  See
+     <https://sqlite.org/json1.html> for additional information.
+
   *  **memvfs.c** &mdash;  This file implements a custom
      [VFS](https://www.sqlite.org/vfs.html) that stores an entire database
      file in a single block of RAM.  It serves as a good example of how
@@ -38,3 +47,14 @@ as follows:
      on the source filename with digits removed, so if we used the name
      "sha3.c" then the entry point would conflict with the prior "sha1.c"
      extension.
+
+  *  **unionvtab.c** &mdash; Implementation of the unionvtab and
+     [swarmvtab](https://sqlite.org/swarmvtab.html) virtual tables.
+     These virtual tables allow a single
+     large table to be spread out across multiple database files.  In the
+     case of swarmvtab, the individual database files can be attached on
+     demand.
+
+  *  **zipfile.c** &mdash;  A [virtual table](https://sqlite.org/vtab.html)
+     that can read and write a 
+     [ZIP archive](https://en.wikipedia.org/wiki/Zip_%28file_format%29).

ext/misc/appendvfs.c

@@ -0,0 +1,565 @@
+/*
+** 2017-10-20
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This file implements a VFS shim that allows an SQLite database to be
+** appended onto the end of some other file, such as an executable.
+**
+** A special record must appear at the end of the file that identifies the
+** file as an appended database and provides an offset to page 1.  For
+** best performance page 1 should be located at a disk page boundary, though
+** that is not required.
+**
+** When opening a database using this VFS, the connection might treat
+** the file as an ordinary SQLite database, or it might treat is as a
+** database appended onto some other file.  Here are the rules:
+**
+**  (1)  When opening a new empty file, that file is treated as an ordinary
+**       database.
+**
+**  (2)  When opening a file that begins with the standard SQLite prefix
+**       string "SQLite format 3", that file is treated as an ordinary
+**       database.
+**
+**  (3)  When opening a file that ends with the appendvfs trailer string
+**       "Start-Of-SQLite3-NNNNNNNN" that file is treated as an appended
+**       database.
+**
+**  (4)  If none of the above apply and the SQLITE_OPEN_CREATE flag is
+**       set, then a new database is appended to the already existing file.
+**
+**  (5)  Otherwise, SQLITE_CANTOPEN is returned.
+**
+** To avoid unnecessary complications with the PENDING_BYTE, the size of
+** the file containing the database is limited to 1GB.  This VFS will refuse
+** to read or write past the 1GB mark.  This restriction might be lifted in
+** future versions.  For now, if you need a large database, then keep the
+** database in a separate file.
+**
+** If the file being opened is not an appended database, then this shim is
+** a pass-through into the default underlying VFS.
+**/
+#include "sqlite3ext.h"
+SQLITE_EXTENSION_INIT1
+#include <string.h>
+#include <assert.h>
+
+/* The append mark at the end of the database is:
+**
+**     Start-Of-SQLite3-NNNNNNNN
+**     123456789 123456789 12345
+**
+** The NNNNNNNN represents a 64-bit big-endian unsigned integer which is
+** the offset to page 1.
+*/
+#define APND_MARK_PREFIX     "Start-Of-SQLite3-"
+#define APND_MARK_PREFIX_SZ  17
+#define APND_MARK_SIZE       25
+
+/*
+** Maximum size of the combined prefix + database + append-mark.  This
+** must be less than 0x40000000 to avoid locking issues on Windows.
+*/
+#define APND_MAX_SIZE  (65536*15259)
+
+/*
+** Forward declaration of objects used by this utility
+*/
+typedef struct sqlite3_vfs ApndVfs;
+typedef struct ApndFile ApndFile;
+
+/* Access to a lower-level VFS that (might) implement dynamic loading,
+** access to randomness, etc.
+*/
+#define ORIGVFS(p)  ((sqlite3_vfs*)((p)->pAppData))
+#define ORIGFILE(p) ((sqlite3_file*)(((ApndFile*)(p))+1))
+
+/* An open file */
+struct ApndFile {
+  sqlite3_file base;              /* IO methods */
+  sqlite3_int64 iPgOne;           /* File offset to page 1 */
+  sqlite3_int64 iMark;            /* Start of the append-mark */
+};
+
+/*
+** Methods for ApndFile
+*/
+static int apndClose(sqlite3_file*);
+static int apndRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst);
+static int apndWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst);
+static int apndTruncate(sqlite3_file*, sqlite3_int64 size);
+static int apndSync(sqlite3_file*, int flags);
+static int apndFileSize(sqlite3_file*, sqlite3_int64 *pSize);
+static int apndLock(sqlite3_file*, int);
+static int apndUnlock(sqlite3_file*, int);
+static int apndCheckReservedLock(sqlite3_file*, int *pResOut);
+static int apndFileControl(sqlite3_file*, int op, void *pArg);
+static int apndSectorSize(sqlite3_file*);
+static int apndDeviceCharacteristics(sqlite3_file*);
+static int apndShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**);
+static int apndShmLock(sqlite3_file*, int offset, int n, int flags);
+static void apndShmBarrier(sqlite3_file*);
+static int apndShmUnmap(sqlite3_file*, int deleteFlag);
+static int apndFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp);
+static int apndUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p);
+
+/*
+** Methods for ApndVfs
+*/
+static int apndOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *);
+static int apndDelete(sqlite3_vfs*, const char *zName, int syncDir);
+static int apndAccess(sqlite3_vfs*, const char *zName, int flags, int *);
+static int apndFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut);
+static void *apndDlOpen(sqlite3_vfs*, const char *zFilename);
+static void apndDlError(sqlite3_vfs*, int nByte, char *zErrMsg);
+static void (*apndDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void);
+static void apndDlClose(sqlite3_vfs*, void*);
+static int apndRandomness(sqlite3_vfs*, int nByte, char *zOut);
+static int apndSleep(sqlite3_vfs*, int microseconds);
+static int apndCurrentTime(sqlite3_vfs*, double*);
+static int apndGetLastError(sqlite3_vfs*, int, char *);
+static int apndCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*);
+static int apndSetSystemCall(sqlite3_vfs*, const char*,sqlite3_syscall_ptr);
+static sqlite3_syscall_ptr apndGetSystemCall(sqlite3_vfs*, const char *z);
+static const char *apndNextSystemCall(sqlite3_vfs*, const char *zName);
+
+static sqlite3_vfs apnd_vfs = {
+  3,                            /* iVersion (set when registered) */
+  0,                            /* szOsFile (set when registered) */
+  1024,                         /* mxPathname */
+  0,                            /* pNext */
+  "apndvfs",                    /* zName */
+  0,                            /* pAppData (set when registered) */ 
+  apndOpen,                     /* xOpen */
+  apndDelete,                   /* xDelete */
+  apndAccess,                   /* xAccess */
+  apndFullPathname,             /* xFullPathname */
+  apndDlOpen,                   /* xDlOpen */
+  apndDlError,                  /* xDlError */
+  apndDlSym,                    /* xDlSym */
+  apndDlClose,                  /* xDlClose */
+  apndRandomness,               /* xRandomness */
+  apndSleep,                    /* xSleep */
+  apndCurrentTime,              /* xCurrentTime */
+  apndGetLastError,             /* xGetLastError */
+  apndCurrentTimeInt64,         /* xCurrentTimeInt64 */
+  apndSetSystemCall,            /* xSetSystemCall */
+  apndGetSystemCall,            /* xGetSystemCall */
+  apndNextSystemCall            /* xNextSystemCall */
+};
+
+static const sqlite3_io_methods apnd_io_methods = {
+  3,                              /* iVersion */
+  apndClose,                      /* xClose */
+  apndRead,                       /* xRead */
+  apndWrite,                      /* xWrite */
+  apndTruncate,                   /* xTruncate */
+  apndSync,                       /* xSync */
+  apndFileSize,                   /* xFileSize */
+  apndLock,                       /* xLock */
+  apndUnlock,                     /* xUnlock */
+  apndCheckReservedLock,          /* xCheckReservedLock */
+  apndFileControl,                /* xFileControl */
+  apndSectorSize,                 /* xSectorSize */
+  apndDeviceCharacteristics,      /* xDeviceCharacteristics */
+  apndShmMap,                     /* xShmMap */
+  apndShmLock,                    /* xShmLock */
+  apndShmBarrier,                 /* xShmBarrier */
+  apndShmUnmap,                   /* xShmUnmap */
+  apndFetch,                      /* xFetch */
+  apndUnfetch                     /* xUnfetch */
+};
+
+
+
+/*
+** Close an apnd-file.
+*/
+static int apndClose(sqlite3_file *pFile){
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xClose(pFile);
+}
+
+/*
+** Read data from an apnd-file.
+*/
+static int apndRead(
+  sqlite3_file *pFile, 
+  void *zBuf, 
+  int iAmt, 
+  sqlite_int64 iOfst
+){
+  ApndFile *p = (ApndFile *)pFile;
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xRead(pFile, zBuf, iAmt, iOfst+p->iPgOne);
+}
+
+/*
+** Add the append-mark onto the end of the file.
+*/
+static int apndWriteMark(ApndFile *p, sqlite3_file *pFile){
+  int i;
+  unsigned char a[APND_MARK_SIZE];
+  memcpy(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ);
+  for(i=0; i<8; i++){
+    a[APND_MARK_PREFIX_SZ+i] = (p->iPgOne >> (56 - i*8)) & 0xff;
+  }
+  return pFile->pMethods->xWrite(pFile, a, APND_MARK_SIZE, p->iMark);
+}
+
+/*
+** Write data to an apnd-file.
+*/
+static int apndWrite(
+  sqlite3_file *pFile,
+  const void *zBuf,
+  int iAmt,
+  sqlite_int64 iOfst
+){
+  int rc;
+  ApndFile *p = (ApndFile *)pFile;
+  pFile = ORIGFILE(pFile);
+  if( iOfst+iAmt>=APND_MAX_SIZE ) return SQLITE_FULL;
+  rc = pFile->pMethods->xWrite(pFile, zBuf, iAmt, iOfst+p->iPgOne);
+  if( rc==SQLITE_OK &&  iOfst + iAmt + p->iPgOne > p->iMark ){
+    sqlite3_int64 sz = 0;
+    rc = pFile->pMethods->xFileSize(pFile, &sz);
+    if( rc==SQLITE_OK ){
+      p->iMark = sz - APND_MARK_SIZE;
+      if( iOfst + iAmt + p->iPgOne > p->iMark ){
+        p->iMark = p->iPgOne + iOfst + iAmt;
+        rc = apndWriteMark(p, pFile);
+      }
+    }
+  }
+  return rc;
+}
+
+/*
+** Truncate an apnd-file.
+*/
+static int apndTruncate(sqlite3_file *pFile, sqlite_int64 size){
+  int rc;
+  ApndFile *p = (ApndFile *)pFile;
+  pFile = ORIGFILE(pFile);
+  rc = pFile->pMethods->xTruncate(pFile, size+p->iPgOne+APND_MARK_SIZE);
+  if( rc==SQLITE_OK ){
+    p->iMark = p->iPgOne+size;
+    rc = apndWriteMark(p, pFile);
+  }
+  return rc;
+}
+
+/*
+** Sync an apnd-file.
+*/
+static int apndSync(sqlite3_file *pFile, int flags){
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xSync(pFile, flags);
+}
+
+/*
+** Return the current file-size of an apnd-file.
+*/
+static int apndFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){
+  ApndFile *p = (ApndFile *)pFile;
+  int rc;
+  pFile = ORIGFILE(p);
+  rc = pFile->pMethods->xFileSize(pFile, pSize);
+  if( rc==SQLITE_OK && p->iPgOne ){
+    *pSize -= p->iPgOne + APND_MARK_SIZE;
+  }
+  return rc;
+}
+
+/*
+** Lock an apnd-file.
+*/
+static int apndLock(sqlite3_file *pFile, int eLock){
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xLock(pFile, eLock);
+}
+
+/*
+** Unlock an apnd-file.
+*/
+static int apndUnlock(sqlite3_file *pFile, int eLock){
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xUnlock(pFile, eLock);
+}
+
+/*
+** Check if another file-handle holds a RESERVED lock on an apnd-file.
+*/
+static int apndCheckReservedLock(sqlite3_file *pFile, int *pResOut){
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xCheckReservedLock(pFile, pResOut);
+}
+
+/*
+** File control method. For custom operations on an apnd-file.
+*/
+static int apndFileControl(sqlite3_file *pFile, int op, void *pArg){
+  ApndFile *p = (ApndFile *)pFile;
+  int rc;
+  pFile = ORIGFILE(pFile);
+  rc = pFile->pMethods->xFileControl(pFile, op, pArg);
+  if( rc==SQLITE_OK && op==SQLITE_FCNTL_VFSNAME ){
+    *(char**)pArg = sqlite3_mprintf("apnd(%lld)/%z", p->iPgOne, *(char**)pArg);
+  }
+  return rc;
+}
+
+/*
+** Return the sector-size in bytes for an apnd-file.
+*/
+static int apndSectorSize(sqlite3_file *pFile){
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xSectorSize(pFile);
+}
+
+/*
+** Return the device characteristic flags supported by an apnd-file.
+*/
+static int apndDeviceCharacteristics(sqlite3_file *pFile){
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xDeviceCharacteristics(pFile);
+}
+
+/* Create a shared memory file mapping */
+static int apndShmMap(
+  sqlite3_file *pFile,
+  int iPg,
+  int pgsz,
+  int bExtend,
+  void volatile **pp
+){
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xShmMap(pFile,iPg,pgsz,bExtend,pp);
+}
+
+/* Perform locking on a shared-memory segment */
+static int apndShmLock(sqlite3_file *pFile, int offset, int n, int flags){
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xShmLock(pFile,offset,n,flags);
+}
+
+/* Memory barrier operation on shared memory */
+static void apndShmBarrier(sqlite3_file *pFile){
+  pFile = ORIGFILE(pFile);
+  pFile->pMethods->xShmBarrier(pFile);
+}
+
+/* Unmap a shared memory segment */
+static int apndShmUnmap(sqlite3_file *pFile, int deleteFlag){
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xShmUnmap(pFile,deleteFlag);
+}
+
+/* Fetch a page of a memory-mapped file */
+static int apndFetch(
+  sqlite3_file *pFile,
+  sqlite3_int64 iOfst,
+  int iAmt,
+  void **pp
+){
+  ApndFile *p = (ApndFile *)pFile;
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xFetch(pFile, iOfst+p->iPgOne, iAmt, pp);
+}
+
+/* Release a memory-mapped page */
+static int apndUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){
+  ApndFile *p = (ApndFile *)pFile;
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xUnfetch(pFile, iOfst+p->iPgOne, pPage);
+}
+
+/*
+** Check to see if the file is an ordinary SQLite database file.
+*/
+static int apndIsOrdinaryDatabaseFile(sqlite3_int64 sz, sqlite3_file *pFile){
+  int rc;
+  char zHdr[16];
+  static const char aSqliteHdr[] = "SQLite format 3";
+  if( sz<512 ) return 0;
+  rc = pFile->pMethods->xRead(pFile, zHdr, sizeof(zHdr), 0);
+  if( rc ) return 0;
+  return memcmp(zHdr, aSqliteHdr, sizeof(zHdr))==0;
+}
+
+/*
+** Try to read the append-mark off the end of a file.  Return the
+** start of the appended database if the append-mark is present.  If
+** there is no append-mark, return -1;
+*/
+static sqlite3_int64 apndReadMark(sqlite3_int64 sz, sqlite3_file *pFile){
+  int rc, i;
+  sqlite3_int64 iMark;
+  unsigned char a[APND_MARK_SIZE];
+
+  if( sz<=APND_MARK_SIZE ) return -1;
+  rc = pFile->pMethods->xRead(pFile, a, APND_MARK_SIZE, sz-APND_MARK_SIZE);
+  if( rc ) return -1;
+  if( memcmp(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ)!=0 ) return -1;
+  iMark = ((sqlite3_int64)(a[APND_MARK_PREFIX_SZ]&0x7f))<<56;
+  for(i=1; i<8; i++){    
+    iMark += (sqlite3_int64)a[APND_MARK_PREFIX_SZ+i]<<(56-8*i);
+  }
+  return iMark;
+}
+
+/*
+** Open an apnd file handle.
+*/
+static int apndOpen(
+  sqlite3_vfs *pVfs,
+  const char *zName,
+  sqlite3_file *pFile,
+  int flags,
+  int *pOutFlags
+){
+  ApndFile *p;
+  sqlite3_file *pSubFile;
+  sqlite3_vfs *pSubVfs;
+  int rc;
+  sqlite3_int64 sz;
+  pSubVfs = ORIGVFS(pVfs);
+  if( (flags & SQLITE_OPEN_MAIN_DB)==0 ){
+    return pSubVfs->xOpen(pSubVfs, zName, pFile, flags, pOutFlags);
+  }
+  p = (ApndFile*)pFile;
+  memset(p, 0, sizeof(*p));
+  pSubFile = ORIGFILE(pFile);
+  p->base.pMethods = &apnd_io_methods;
+  rc = pSubVfs->xOpen(pSubVfs, zName, pSubFile, flags, pOutFlags);
+  if( rc ) goto apnd_open_done;
+  rc = pSubFile->pMethods->xFileSize(pSubFile, &sz);
+  if( rc ){
+    pSubFile->pMethods->xClose(pSubFile);
+    goto apnd_open_done;
+  }
+  if( apndIsOrdinaryDatabaseFile(sz, pSubFile) ){
+    memmove(pFile, pSubFile, pSubVfs->szOsFile);
+    return SQLITE_OK;
+  }
+  p->iMark = 0;
+  p->iPgOne = apndReadMark(sz, pFile);
+  if( p->iPgOne>0 ){
+    return SQLITE_OK;
+  }
+  if( (flags & SQLITE_OPEN_CREATE)==0 ){
+    pSubFile->pMethods->xClose(pSubFile);
+    rc = SQLITE_CANTOPEN;
+  }
+  p->iPgOne = (sz+0xfff) & ~(sqlite3_int64)0xfff;
+apnd_open_done:
+  if( rc ) pFile->pMethods = 0;
+  return rc;
+}
+
+/*
+** All other VFS methods are pass-thrus.
+*/
+static int apndDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){
+  return ORIGVFS(pVfs)->xDelete(ORIGVFS(pVfs), zPath, dirSync);
+}
+static int apndAccess(
+  sqlite3_vfs *pVfs, 
+  const char *zPath, 
+  int flags, 
+  int *pResOut
+){
+  return ORIGVFS(pVfs)->xAccess(ORIGVFS(pVfs), zPath, flags, pResOut);
+}
+static int apndFullPathname(
+  sqlite3_vfs *pVfs, 
+  const char *zPath, 
+  int nOut, 
+  char *zOut
+){
+  return ORIGVFS(pVfs)->xFullPathname(ORIGVFS(pVfs),zPath,nOut,zOut);
+}
+static void *apndDlOpen(sqlite3_vfs *pVfs, const char *zPath){
+  return ORIGVFS(pVfs)->xDlOpen(ORIGVFS(pVfs), zPath);
+}
+static void apndDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){
+  ORIGVFS(pVfs)->xDlError(ORIGVFS(pVfs), nByte, zErrMsg);
+}
+static void (*apndDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){
+  return ORIGVFS(pVfs)->xDlSym(ORIGVFS(pVfs), p, zSym);
+}
+static void apndDlClose(sqlite3_vfs *pVfs, void *pHandle){
+  ORIGVFS(pVfs)->xDlClose(ORIGVFS(pVfs), pHandle);
+}
+static int apndRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){
+  return ORIGVFS(pVfs)->xRandomness(ORIGVFS(pVfs), nByte, zBufOut);
+}
+static int apndSleep(sqlite3_vfs *pVfs, int nMicro){
+  return ORIGVFS(pVfs)->xSleep(ORIGVFS(pVfs), nMicro);
+}
+static int apndCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){
+  return ORIGVFS(pVfs)->xCurrentTime(ORIGVFS(pVfs), pTimeOut);
+}
+static int apndGetLastError(sqlite3_vfs *pVfs, int a, char *b){
+  return ORIGVFS(pVfs)->xGetLastError(ORIGVFS(pVfs), a, b);
+}
+static int apndCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){
+  return ORIGVFS(pVfs)->xCurrentTimeInt64(ORIGVFS(pVfs), p);
+}
+static int apndSetSystemCall(
+  sqlite3_vfs *pVfs,
+  const char *zName,
+  sqlite3_syscall_ptr pCall
+){
+  return ORIGVFS(pVfs)->xSetSystemCall(ORIGVFS(pVfs),zName,pCall);
+}
+static sqlite3_syscall_ptr apndGetSystemCall(
+  sqlite3_vfs *pVfs,
+  const char *zName
+){
+  return ORIGVFS(pVfs)->xGetSystemCall(ORIGVFS(pVfs),zName);
+}
+static const char *apndNextSystemCall(sqlite3_vfs *pVfs, const char *zName){
+  return ORIGVFS(pVfs)->xNextSystemCall(ORIGVFS(pVfs), zName);
+}
+
+  
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+/* 
+** This routine is called when the extension is loaded.
+** Register the new VFS.
+*/
+int sqlite3_appendvfs_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  int rc = SQLITE_OK;
+  sqlite3_vfs *pOrig;
+  SQLITE_EXTENSION_INIT2(pApi);
+  (void)pzErrMsg;
+  (void)db;
+  pOrig = sqlite3_vfs_find(0);
+  apnd_vfs.iVersion = pOrig->iVersion;
+  apnd_vfs.pAppData = pOrig;
+  apnd_vfs.szOsFile = pOrig->szOsFile + sizeof(ApndFile);
+  rc = sqlite3_vfs_register(&apnd_vfs, 0);
+#ifdef APPENDVFS_TEST
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_auto_extension((void(*)(void))apndvfsRegister);
+  }
+#endif
+  if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY;
+  return rc;
+}

ext/misc/btreeinfo.c

@@ -0,0 +1,428 @@
+/*
+** 2017-10-24
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This file contains an implementation of the "sqlite_btreeinfo" virtual table.
+**
+** The sqlite_btreeinfo virtual table is a read-only eponymous-only virtual
+** table that shows information about all btrees in an SQLite database file.
+** The schema is like this:
+**
+** CREATE TABLE sqlite_btreeinfo(
+**    type TEXT,                   -- "table" or "index"
+**    name TEXT,                   -- Name of table or index for this btree.
+**    tbl_name TEXT,               -- Associated table
+**    rootpage INT,                -- The root page of the btree
+**    sql TEXT,                    -- SQL for this btree - from sqlite_master
+**    hasRowid BOOLEAN,            -- True if the btree has a rowid
+**    nEntry INT,                  -- Estimated number of enteries
+**    nPage INT,                   -- Estimated number of pages
+**    depth INT,                   -- Depth of the btree
+**    szPage INT,                  -- Size of each page in bytes
+**    zSchema TEXT HIDDEN          -- The schema to which this btree belongs
+** );
+**
+** The first 5 fields are taken directly from the sqlite_master table.
+** Considering only the first 5 fields, the only difference between 
+** this virtual table and the sqlite_master table is that this virtual
+** table omits all entries that have a 0 or NULL rowid - in other words
+** it omits triggers and views.
+**
+** The value added by this table comes in the next 5 fields.
+**
+** Note that nEntry and nPage are *estimated*.  They are computed doing
+** a single search from the root to a leaf, counting the number of cells
+** at each level, and assuming that unvisited pages have a similar number
+** of cells.
+**
+** The sqlite_dbpage virtual table must be available for this virtual table
+** to operate.
+**
+** USAGE EXAMPLES:
+**
+** Show the table btrees in a schema order with the tables with the most
+** rows occuring first:
+**
+**      SELECT name, nEntry
+**        FROM sqlite_btreeinfo
+**       WHERE type='table'
+**       ORDER BY nEntry DESC, name;
+**
+** Show the names of all WITHOUT ROWID tables: 
+**
+**      SELECT name FROM sqlite_btreeinfo
+**       WHERE type='table' AND NOT hasRowid;
+*/
+#if !defined(SQLITEINT_H)
+#include "sqlite3ext.h"
+#endif
+SQLITE_EXTENSION_INIT1
+#include <string.h>
+#include <assert.h>
+
+/* Columns available in this virtual table */
+#define BINFO_COLUMN_TYPE         0
+#define BINFO_COLUMN_NAME         1
+#define BINFO_COLUMN_TBL_NAME     2
+#define BINFO_COLUMN_ROOTPAGE     3
+#define BINFO_COLUMN_SQL          4
+#define BINFO_COLUMN_HASROWID     5
+#define BINFO_COLUMN_NENTRY       6
+#define BINFO_COLUMN_NPAGE        7
+#define BINFO_COLUMN_DEPTH        8
+#define BINFO_COLUMN_SZPAGE       9
+#define BINFO_COLUMN_SCHEMA      10
+
+/* Forward declarations */
+typedef struct BinfoTable BinfoTable;
+typedef struct BinfoCursor BinfoCursor;
+
+/* A cursor for the sqlite_btreeinfo table */
+struct BinfoCursor {
+  sqlite3_vtab_cursor base;       /* Base class.  Must be first */
+  sqlite3_stmt *pStmt;            /* Query against sqlite_master */
+  int rc;                         /* Result of previous sqlite_step() call */
+  int hasRowid;                   /* hasRowid value.  Negative if unknown. */
+  sqlite3_int64 nEntry;           /* nEntry value */
+  int nPage;                      /* nPage value */
+  int depth;                      /* depth value */
+  int szPage;                     /* size of a btree page.  0 if unknown */
+  char *zSchema;                  /* Schema being interrogated */
+};
+
+/* The sqlite_btreeinfo table */
+struct BinfoTable {
+  sqlite3_vtab base;              /* Base class.  Must be first */
+  sqlite3 *db;                    /* The databse connection */
+};
+
+/*
+** Connect to the sqlite_btreeinfo virtual table.
+*/
+static int binfoConnect(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
+){
+  BinfoTable *pTab = 0;
+  int rc = SQLITE_OK;
+  rc = sqlite3_declare_vtab(db, 
+      "CREATE TABLE x(\n"
+      " type TEXT,\n"
+      " name TEXT,\n"
+      " tbl_name TEXT,\n"
+      " rootpage INT,\n"
+      " sql TEXT,\n"
+      " hasRowid BOOLEAN,\n"
+      " nEntry INT,\n"
+      " nPage INT,\n"
+      " depth INT,\n"
+      " szPage INT,\n"
+      " zSchema TEXT HIDDEN\n"
+      ")");
+  if( rc==SQLITE_OK ){
+    pTab = (BinfoTable *)sqlite3_malloc64(sizeof(BinfoTable));
+    if( pTab==0 ) rc = SQLITE_NOMEM;
+  }
+  assert( rc==SQLITE_OK || pTab==0 );
+  if( pTab ){
+    pTab->db = db;
+  }
+  *ppVtab = (sqlite3_vtab*)pTab;
+  return rc;
+}
+
+/*
+** Disconnect from or destroy a btreeinfo virtual table.
+*/
+static int binfoDisconnect(sqlite3_vtab *pVtab){
+  sqlite3_free(pVtab);
+  return SQLITE_OK;
+}
+
+/*
+** idxNum:
+**
+**     0     Use "main" for the schema
+**     1     Schema identified by parameter ?1
+*/
+static int binfoBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
+  int i;
+  pIdxInfo->estimatedCost = 10000.0;  /* Cost estimate */
+  pIdxInfo->estimatedRows = 100;
+  for(i=0; i<pIdxInfo->nConstraint; i++){
+    struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[i];
+    if( p->usable
+     && p->iColumn==BINFO_COLUMN_SCHEMA
+     && p->op==SQLITE_INDEX_CONSTRAINT_EQ
+    ){
+      pIdxInfo->estimatedCost = 1000.0;
+      pIdxInfo->idxNum = 1;
+      pIdxInfo->aConstraintUsage[i].argvIndex = 1;
+      pIdxInfo->aConstraintUsage[i].omit = 1;
+      break;
+    }
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Open a new btreeinfo cursor.
+*/
+static int binfoOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
+  BinfoCursor *pCsr;
+
+  pCsr = (BinfoCursor *)sqlite3_malloc64(sizeof(BinfoCursor));
+  if( pCsr==0 ){
+    return SQLITE_NOMEM;
+  }else{
+    memset(pCsr, 0, sizeof(BinfoCursor));
+    pCsr->base.pVtab = pVTab;
+  }
+
+  *ppCursor = (sqlite3_vtab_cursor *)pCsr;
+  return SQLITE_OK;
+}
+
+/*
+** Close a btreeinfo cursor.
+*/
+static int binfoClose(sqlite3_vtab_cursor *pCursor){
+  BinfoCursor *pCsr = (BinfoCursor *)pCursor;
+  sqlite3_finalize(pCsr->pStmt);
+  sqlite3_free(pCsr->zSchema);
+  sqlite3_free(pCsr);
+  return SQLITE_OK;
+}
+
+/*
+** Move a btreeinfo cursor to the next entry in the file.
+*/
+static int binfoNext(sqlite3_vtab_cursor *pCursor){
+  BinfoCursor *pCsr = (BinfoCursor *)pCursor;
+  pCsr->rc = sqlite3_step(pCsr->pStmt);
+  pCsr->hasRowid = -1;
+  return pCsr->rc==SQLITE_ERROR ? SQLITE_ERROR : SQLITE_OK;
+}
+
+/* We have reached EOF if previous sqlite3_step() returned
+** anything other than SQLITE_ROW;
+*/
+static int binfoEof(sqlite3_vtab_cursor *pCursor){
+  BinfoCursor *pCsr = (BinfoCursor *)pCursor;
+  return pCsr->rc!=SQLITE_ROW;
+}
+
+/* Position a cursor back to the beginning.
+*/
+static int binfoFilter(
+  sqlite3_vtab_cursor *pCursor, 
+  int idxNum, const char *idxStr,
+  int argc, sqlite3_value **argv
+){
+  BinfoCursor *pCsr = (BinfoCursor *)pCursor;
+  BinfoTable *pTab = (BinfoTable *)pCursor->pVtab;
+  char *zSql;
+  int rc;
+
+  sqlite3_free(pCsr->zSchema);
+  if( idxNum==1 && sqlite3_value_type(argv[0])!=SQLITE_NULL ){
+    pCsr->zSchema = sqlite3_mprintf("%s", sqlite3_value_text(argv[0]));
+  }else{
+    pCsr->zSchema = sqlite3_mprintf("main");
+  }
+  zSql = sqlite3_mprintf(
+      "SELECT 0, 'table','sqlite_master','sqlite_master',1,NULL "
+      "UNION ALL "
+      "SELECT rowid, type, name, tbl_name, rootpage, sql"
+      " FROM \"%w\".sqlite_master WHERE rootpage>=1",
+       pCsr->zSchema);
+  sqlite3_finalize(pCsr->pStmt);
+  pCsr->pStmt = 0;
+  pCsr->hasRowid = -1;
+  rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0);
+  sqlite3_free(zSql);
+  if( rc==SQLITE_OK ){
+    rc = binfoNext(pCursor);
+  }
+  return rc;
+}
+
+/* Decode big-endian integers */
+static unsigned int get_uint16(unsigned char *a){
+  return (a[0]<<8)|a[1];
+}
+static unsigned int get_uint32(unsigned char *a){
+  return (a[0]<<24)|(a[1]<<16)|(a[2]<<8)|a[3];
+}
+
+/* Examine the b-tree rooted at pgno and estimate its size.
+** Return non-zero if anything goes wrong.
+*/
+static int binfoCompute(sqlite3 *db, int pgno, BinfoCursor *pCsr){
+  sqlite3_int64 nEntry = 1;
+  int nPage = 1;
+  unsigned char *aData;
+  sqlite3_stmt *pStmt = 0;
+  int rc = SQLITE_OK;
+  int pgsz = 0;
+  int nCell;
+  int iCell;
+
+  rc = sqlite3_prepare_v2(db, 
+           "SELECT data FROM sqlite_dbpage('main') WHERE pgno=?1", -1,
+           &pStmt, 0);
+  if( rc ) return rc;
+  pCsr->depth = 1;
+  while(1){
+    sqlite3_bind_int(pStmt, 1, pgno);
+    rc = sqlite3_step(pStmt);
+    if( rc!=SQLITE_ROW ){
+      rc = SQLITE_ERROR;
+      break;
+    }
+    pCsr->szPage = pgsz = sqlite3_column_bytes(pStmt, 0);
+    aData = (unsigned char*)sqlite3_column_blob(pStmt, 0);
+    if( aData==0 ){    
+      rc = SQLITE_NOMEM;
+      break;
+    }
+    if( pgno==1 ){
+      aData += 100;
+      pgsz -= 100;
+    }
+    pCsr->hasRowid = aData[0]!=2 && aData[0]!=10;
+    nCell = get_uint16(aData+3);
+    nEntry *= (nCell+1);
+    if( aData[0]==10 || aData[0]==13 ) break;
+    nPage *= (nCell+1);
+    if( nCell<=1 ){
+      pgno = get_uint32(aData+8);
+    }else{
+      iCell = get_uint16(aData+12+2*(nCell/2));
+      if( pgno==1 ) iCell -= 100;
+      if( iCell<=12 || iCell>=pgsz-4 ){
+        rc = SQLITE_CORRUPT;
+        break;
+      }
+      pgno = get_uint32(aData+iCell);
+    }
+    pCsr->depth++;
+    sqlite3_reset(pStmt);
+  }
+  sqlite3_finalize(pStmt);
+  pCsr->nPage = nPage;
+  pCsr->nEntry = nEntry;
+  if( rc==SQLITE_ROW ) rc = SQLITE_OK;
+  return rc;
+}
+
+/* Return a column for the sqlite_btreeinfo table */
+static int binfoColumn(
+  sqlite3_vtab_cursor *pCursor, 
+  sqlite3_context *ctx, 
+  int i
+){
+  BinfoCursor *pCsr = (BinfoCursor *)pCursor;
+  if( i>=BINFO_COLUMN_HASROWID && i<=BINFO_COLUMN_SZPAGE && pCsr->hasRowid<0 ){
+    int pgno = sqlite3_column_int(pCsr->pStmt, BINFO_COLUMN_ROOTPAGE+1);
+    sqlite3 *db = sqlite3_context_db_handle(ctx);
+    int rc = binfoCompute(db, pgno, pCsr);
+    if( rc ){
+      pCursor->pVtab->zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
+      return SQLITE_ERROR;
+    }
+  }
+  switch( i ){
+    case BINFO_COLUMN_NAME:
+    case BINFO_COLUMN_TYPE:
+    case BINFO_COLUMN_TBL_NAME:
+    case BINFO_COLUMN_ROOTPAGE:
+    case BINFO_COLUMN_SQL: {
+      sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pStmt, i+1));
+      break;
+    }
+    case BINFO_COLUMN_HASROWID: {
+      sqlite3_result_int(ctx, pCsr->hasRowid);
+      break;
+    }
+    case BINFO_COLUMN_NENTRY: {
+      sqlite3_result_int64(ctx, pCsr->nEntry);
+      break;
+    }
+    case BINFO_COLUMN_NPAGE: {
+      sqlite3_result_int(ctx, pCsr->nPage);
+      break;
+    }
+    case BINFO_COLUMN_DEPTH: {
+      sqlite3_result_int(ctx, pCsr->depth);
+      break;
+    }
+    case BINFO_COLUMN_SCHEMA: {
+      sqlite3_result_text(ctx, pCsr->zSchema, -1, SQLITE_STATIC);
+      break;
+    }
+  }
+  return SQLITE_OK;
+}
+
+/* Return the ROWID for the sqlite_btreeinfo table */
+static int binfoRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
+  BinfoCursor *pCsr = (BinfoCursor *)pCursor;
+  *pRowid = sqlite3_column_int64(pCsr->pStmt, 0);
+  return SQLITE_OK;
+}
+
+/*
+** Invoke this routine to register the "sqlite_btreeinfo" virtual table module
+*/
+int sqlite3BinfoRegister(sqlite3 *db){
+  static sqlite3_module binfo_module = {
+    0,                           /* iVersion */
+    0,                           /* xCreate */
+    binfoConnect,                /* xConnect */
+    binfoBestIndex,              /* xBestIndex */
+    binfoDisconnect,             /* xDisconnect */
+    0,                           /* xDestroy */
+    binfoOpen,                   /* xOpen - open a cursor */
+    binfoClose,                  /* xClose - close a cursor */
+    binfoFilter,                 /* xFilter - configure scan constraints */
+    binfoNext,                   /* xNext - advance a cursor */
+    binfoEof,                    /* xEof - check for end of scan */
+    binfoColumn,                 /* xColumn - read data */
+    binfoRowid,                  /* xRowid - read data */
+    0,                           /* xUpdate */
+    0,                           /* xBegin */
+    0,                           /* xSync */
+    0,                           /* xCommit */
+    0,                           /* xRollback */
+    0,                           /* xFindMethod */
+    0,                           /* xRename */
+    0,                           /* xSavepoint */
+    0,                           /* xRelease */
+    0,                           /* xRollbackTo */
+  };
+  return sqlite3_create_module(db, "sqlite_btreeinfo", &binfo_module, 0);
+}
+
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_btreeinfo_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  SQLITE_EXTENSION_INIT2(pApi);
+  return sqlite3BinfoRegister(db);
+}

ext/misc/closure.c

@@ -826,17 +826,12 @@ static int closureBestIndex(
   int iPlan = 0;
   int i;
   int idx = 1;
-  int seenMatch = 0;
   const struct sqlite3_index_constraint *pConstraint;
   closure_vtab *pVtab = (closure_vtab*)pTab;
   double rCost = 10000000.0;
 
   pConstraint = pIdxInfo->aConstraint;
   for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){
-    if( pConstraint->iColumn==CLOSURE_COL_ROOT
-     && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){
-      seenMatch = 1;
-    }
     if( pConstraint->usable==0 ) continue;
     if( (iPlan & 1)==0 
      && pConstraint->iColumn==CLOSURE_COL_ROOT
@@ -893,6 +888,18 @@ static int closureBestIndex(
     ** or else the result is an empty set. */
     iPlan = 0;
   }
+  if( (iPlan&1)==0 ){
+    /* If there is no usable "root=?" term, then set the index-type to 0.
+    ** Also clear any argvIndex variables already set. This is necessary
+    ** to prevent the core from throwing an "xBestIndex malfunction error"
+    ** error (because the argvIndex values are not contiguously assigned
+    ** starting from 1).  */
+    rCost *= 1e30;
+    for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){
+      pIdxInfo->aConstraintUsage[i].argvIndex = 0;
+    }
+    iPlan = 0;
+  }
   pIdxInfo->idxNum = iPlan;
   if( pIdxInfo->nOrderBy==1
    && pIdxInfo->aOrderBy[0].iColumn==CLOSURE_COL_ID
@@ -900,7 +907,6 @@ static int closureBestIndex(
   ){
     pIdxInfo->orderByConsumed = 1;
   }
-  if( seenMatch && (iPlan&1)==0 ) rCost *= 1e30;
   pIdxInfo->estimatedCost = rCost;
    
   return SQLITE_OK;

ext/misc/completion.c

@@ -62,6 +62,7 @@ struct completion_cursor {
   char *zPrefix;             /* The prefix for the word we want to complete */
   char *zLine;               /* The whole that we want to complete */
   const char *zCurrentRow;   /* Current output row */
+  int szRow;                 /* Length of the zCurrentRow string */
   sqlite3_stmt *pStmt;       /* Current statement */
   sqlite3_int64 iRowid;      /* The rowid */
   int ePhase;                /* Current phase */
@@ -78,7 +79,7 @@ struct completion_cursor {
 #define COMPLETION_INDEXES       5
 #define COMPLETION_TRIGGERS      6
 #define COMPLETION_DATABASES     7
-#define COMPLETION_TABLES        8
+#define COMPLETION_TABLES        8    /* Also VIEWs and TRIGGERs */
 #define COMPLETION_COLUMNS       9
 #define COMPLETION_MODULES       10
 #define COMPLETION_EOF           11
@@ -174,32 +175,6 @@ static int completionClose(sqlite3_vtab_cursor *cur){
   return SQLITE_OK;
 }
 
-/*
-** All SQL keywords understood by SQLite
-*/
-static const char *completionKwrds[] = {
-  "ABORT", "ACTION", "ADD", "AFTER", "ALL", "ALTER", "ANALYZE", "AND", "AS",
-  "ASC", "ATTACH", "AUTOINCREMENT", "BEFORE", "BEGIN", "BETWEEN", "BY",
-  "CASCADE", "CASE", "CAST", "CHECK", "COLLATE", "COLUMN", "COMMIT",
-  "CONFLICT", "CONSTRAINT", "CREATE", "CROSS", "CURRENT_DATE",
-  "CURRENT_TIME", "CURRENT_TIMESTAMP", "DATABASE", "DEFAULT", "DEFERRABLE",
-  "DEFERRED", "DELETE", "DESC", "DETACH", "DISTINCT", "DROP", "EACH",
-  "ELSE", "END", "ESCAPE", "EXCEPT", "EXCLUSIVE", "EXISTS", "EXPLAIN",
-  "FAIL", "FOR", "FOREIGN", "FROM", "FULL", "GLOB", "GROUP", "HAVING", "IF",
-  "IGNORE", "IMMEDIATE", "IN", "INDEX", "INDEXED", "INITIALLY", "INNER",
-  "INSERT", "INSTEAD", "INTERSECT", "INTO", "IS", "ISNULL", "JOIN", "KEY",
-  "LEFT", "LIKE", "LIMIT", "MATCH", "NATURAL", "NO", "NOT", "NOTNULL",
-  "NULL", "OF", "OFFSET", "ON", "OR", "ORDER", "OUTER", "PLAN", "PRAGMA",
-  "PRIMARY", "QUERY", "RAISE", "RECURSIVE", "REFERENCES", "REGEXP",
-  "REINDEX", "RELEASE", "RENAME", "REPLACE", "RESTRICT", "RIGHT",
-  "ROLLBACK", "ROW", "SAVEPOINT", "SELECT", "SET", "TABLE", "TEMP",
-  "TEMPORARY", "THEN", "TO", "TRANSACTION", "TRIGGER", "UNION", "UNIQUE",
-  "UPDATE", "USING", "VACUUM", "VALUES", "VIEW", "VIRTUAL", "WHEN", "WHERE",
-  "WITH", "WITHOUT",
-};
-#define completionKwCount \
-   (int)(sizeof(completionKwrds)/sizeof(completionKwrds[0]))
-
 /*
 ** Advance a completion_cursor to its next row of output.
 **
@@ -222,11 +197,11 @@ static int completionNext(sqlite3_vtab_cursor *cur){
   while( pCur->ePhase!=COMPLETION_EOF ){
     switch( pCur->ePhase ){
       case COMPLETION_KEYWORDS: {
-        if( pCur->j >= completionKwCount ){
+        if( pCur->j >= sqlite3_keyword_count() ){
           pCur->zCurrentRow = 0;
           pCur->ePhase = COMPLETION_DATABASES;
         }else{
-          pCur->zCurrentRow = completionKwrds[pCur->j++];
+          sqlite3_keyword_name(pCur->j++, &pCur->zCurrentRow, &pCur->szRow);
         }
         iCol = -1;
         break;
@@ -250,8 +225,7 @@ static int completionNext(sqlite3_vtab_cursor *cur){
             const char *zDb = (const char*)sqlite3_column_text(pS2, 1);
             zSql = sqlite3_mprintf(
                "%z%s"
-               "SELECT name FROM \"%w\".sqlite_master"
+               "SELECT name FROM \"%w\".sqlite_master",
-               " WHERE type='table'",
                zSql, zSep, zDb
             );
             if( zSql==0 ) return SQLITE_NOMEM;
@@ -299,6 +273,7 @@ static int completionNext(sqlite3_vtab_cursor *cur){
       if( sqlite3_step(pCur->pStmt)==SQLITE_ROW ){
         /* Extract the next row of content */
         pCur->zCurrentRow = (const char*)sqlite3_column_text(pCur->pStmt, iCol);
+        pCur->szRow = sqlite3_column_bytes(pCur->pStmt, iCol);
       }else{
         /* When all rows are finished, advance to the next phase */
         sqlite3_finalize(pCur->pStmt);
@@ -308,7 +283,9 @@ static int completionNext(sqlite3_vtab_cursor *cur){
       }
     }
     if( pCur->nPrefix==0 ) break;
-    if( sqlite3_strnicmp(pCur->zPrefix, pCur->zCurrentRow, pCur->nPrefix)==0 ){
+    if( pCur->nPrefix<=pCur->szRow
+     && sqlite3_strnicmp(pCur->zPrefix, pCur->zCurrentRow, pCur->nPrefix)==0
+    ){
       break;
     }
   }
@@ -328,7 +305,7 @@ static int completionColumn(
   completion_cursor *pCur = (completion_cursor*)cur;
   switch( i ){
     case COMPLETION_COLUMN_CANDIDATE: {
-      sqlite3_result_text(ctx, pCur->zCurrentRow, -1, SQLITE_TRANSIENT);
+      sqlite3_result_text(ctx, pCur->zCurrentRow, pCur->szRow,SQLITE_TRANSIENT);
       break;
     }
     case COMPLETION_COLUMN_PREFIX: {
@@ -388,7 +365,7 @@ static int completionFilter(
       pCur->zPrefix = sqlite3_mprintf("%s", sqlite3_value_text(argv[iArg]));
       if( pCur->zPrefix==0 ) return SQLITE_NOMEM;
     }
-    iArg++;
+    iArg = 1;
   }
   if( idxNum & 2 ){
     pCur->nLine = sqlite3_value_bytes(argv[iArg]);
@@ -396,7 +373,6 @@ static int completionFilter(
       pCur->zLine = sqlite3_mprintf("%s", sqlite3_value_text(argv[iArg]));
       if( pCur->zLine==0 ) return SQLITE_NOMEM;
     }
-    iArg++;
   }
   if( pCur->zLine!=0 && pCur->zPrefix==0 ){
     int i = pCur->nLine;

ext/misc/compress.c

@@ -27,6 +27,21 @@ SQLITE_EXTENSION_INIT1
 ** seven bits per integer stored in the lower seven bits of each byte.
 ** More significant bits occur first.  The most significant bit (0x80)
 ** is a flag to indicate the end of the integer.
+**
+** This function, SQLAR, and ZIP all use the same "deflate" compression
+** algorithm, but each is subtly different:
+**
+**   *  ZIP uses raw deflate.
+**
+**   *  SQLAR uses the "zlib format" which is raw deflate with a two-byte
+**      algorithm-identification header and a four-byte checksum at the end.
+**
+**   *  This utility uses the "zlib format" like SQLAR, but adds the variable-
+**      length integer uncompressed size value at the beginning.
+**
+** This function might be extended in the future to support compression
+** formats other than deflate, by providing a different algorithm-id
+** mark following the variable-length integer size parameter.
 */
 static void compressFunc(
   sqlite3_context *context,

ext/misc/csv.c

@@ -78,7 +78,7 @@ struct CsvReader {
   int nAlloc;            /* Space allocated for z[] */
   int nLine;             /* Current line number */
   int bNotFirst;         /* True if prior text has been seen */
-  char cTerm;            /* Character that terminated the most recent field */
+  int cTerm;             /* Character that terminated the most recent field */
   size_t iIn;            /* Next unread character in the input buffer */
   size_t nIn;            /* Number of characters in the input buffer */
   char *zIn;             /* The input buffer */
@@ -132,6 +132,7 @@ static int csv_reader_open(
     }
     p->in = fopen(zFilename, "rb");
     if( p->in==0 ){
+      sqlite3_free(p->zIn);
       csv_reader_reset(p);
       csv_errmsg(p, "cannot open '%s' for reading", zFilename);
       return 1;
@@ -166,7 +167,7 @@ static int csv_getc(CsvReader *p){
     if( p->in!=0 ) return csv_getc_refill(p);
     return EOF;
   }
-  return p->zIn[p->iIn++];
+  return ((unsigned char*)p->zIn)[p->iIn++];
 }
 
 /* Increase the size of p->z and append character c to the end. 
@@ -204,7 +205,8 @@ static int csv_append(CsvReader *p, char c){
 **   +  Store the character that terminates the field in p->cTerm.  Store
 **      EOF on end-of-file.
 **
-** Return "" at EOF.  Return 0 on an OOM error.
+** Return 0 at EOF or on OOM.  On EOF, the p->cTerm character will have
+** been set to EOF.
 */
 static char *csv_read_one_field(CsvReader *p){
   int c;
@@ -212,7 +214,7 @@ static char *csv_read_one_field(CsvReader *p){
   c = csv_getc(p);
   if( c==EOF ){
     p->cTerm = EOF;
-    return "";
+    return 0;
   }
   if( c=='"' ){
     int pc, ppc;
@@ -543,8 +545,7 @@ static int csvtabConnect(
     pNew->nCol = nCol;
   }else{
     do{
-      const char *z = csv_read_one_field(&sRdr);
+      csv_read_one_field(&sRdr);
-      if( z==0 ) goto csvtab_connect_oom;
       pNew->nCol++;
     }while( sRdr.cTerm==',' );
   }
@@ -662,7 +663,6 @@ static int csvtabNext(sqlite3_vtab_cursor *cur){
   do{
     z = csv_read_one_field(&pCur->rdr);
     if( z==0 ){
-      csv_xfer_error(pTab, &pCur->rdr);
       break;
     }
     if( i<pTab->nCol ){

ext/misc/dbdump.c

@@ -141,45 +141,12 @@ static void appendText(DText *p, char const *zAppend, char quote){
 ** Return '"' if quoting is required.  Return 0 if no quoting is required.
 */
 static char quoteChar(const char *zName){
-  /* All SQLite keywords, in alphabetical order */
+  int i;
-  static const char *azKeywords[] = {
-    "ABORT", "ACTION", "ADD", "AFTER", "ALL", "ALTER", "ANALYZE", "AND", "AS",
-    "ASC", "ATTACH", "AUTOINCREMENT", "BEFORE", "BEGIN", "BETWEEN", "BY",
-    "CASCADE", "CASE", "CAST", "CHECK", "COLLATE", "COLUMN", "COMMIT",
-    "CONFLICT", "CONSTRAINT", "CREATE", "CROSS", "CURRENT_DATE",
-    "CURRENT_TIME", "CURRENT_TIMESTAMP", "DATABASE", "DEFAULT", "DEFERRABLE",
-    "DEFERRED", "DELETE", "DESC", "DETACH", "DISTINCT", "DROP", "EACH",
-    "ELSE", "END", "ESCAPE", "EXCEPT", "EXCLUSIVE", "EXISTS", "EXPLAIN",
-    "FAIL", "FOR", "FOREIGN", "FROM", "FULL", "GLOB", "GROUP", "HAVING", "IF",
-    "IGNORE", "IMMEDIATE", "IN", "INDEX", "INDEXED", "INITIALLY", "INNER",
-    "INSERT", "INSTEAD", "INTERSECT", "INTO", "IS", "ISNULL", "JOIN", "KEY",
-    "LEFT", "LIKE", "LIMIT", "MATCH", "NATURAL", "NO", "NOT", "NOTNULL",
-    "NULL", "OF", "OFFSET", "ON", "OR", "ORDER", "OUTER", "PLAN", "PRAGMA",
-    "PRIMARY", "QUERY", "RAISE", "RECURSIVE", "REFERENCES", "REGEXP",
-    "REINDEX", "RELEASE", "RENAME", "REPLACE", "RESTRICT", "RIGHT",
-    "ROLLBACK", "ROW", "SAVEPOINT", "SELECT", "SET", "TABLE", "TEMP",
-    "TEMPORARY", "THEN", "TO", "TRANSACTION", "TRIGGER", "UNION", "UNIQUE",
-    "UPDATE", "USING", "VACUUM", "VALUES", "VIEW", "VIRTUAL", "WHEN", "WHERE",
-    "WITH", "WITHOUT",
-  };
-  int i, lwr, upr, mid, c;
   if( !isalpha((unsigned char)zName[0]) && zName[0]!='_' ) return '"';
   for(i=0; zName[i]; i++){
     if( !isalnum((unsigned char)zName[i]) && zName[i]!='_' ) return '"';
   }
-  lwr = 0;
+  return sqlite3_keyword_check(zName, i) ? '"' : 0;
-  upr = sizeof(azKeywords)/sizeof(azKeywords[0]) - 1;
-  while( lwr<=upr ){
-    mid = (lwr+upr)/2;
-    c = sqlite3_stricmp(azKeywords[mid], zName);
-    if( c==0 ) return '"';
-    if( c<0 ){
-      lwr = mid+1;
-    }else{
-      upr = mid-1;
-    }
-  }
-  return 0;
 }
 
 
@@ -293,7 +260,6 @@ static char **tableColumnList(DState *p, const char *zTab){
         ** ordinary column in the table.  Verify that azRowid[j] is a valid
         ** name for the rowid before adding it to azCol[0].  WITHOUT ROWID
         ** tables will fail this last check */
-        int rc;
         rc = sqlite3_table_column_metadata(p->db,0,zTab,azRowid[j],0,0,0,0,0);
         if( rc==SQLITE_OK ) azCol[0] = azRowid[j];
         break;
@@ -455,12 +421,12 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){
   if( strcmp(zType, "table")==0 ){
     DText sSelect;
     DText sTable;
-    char **azCol;
+    char **azTCol;
     int i;
     int nCol;
 
-    azCol = tableColumnList(p, zTable);
+    azTCol = tableColumnList(p, zTable);
-    if( azCol==0 ) return 0;
+    if( azTCol==0 ) return 0;
 
     initText(&sTable);
     appendText(&sTable, "INSERT INTO ", 0);
@@ -473,12 +439,12 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){
     ** In other words:  "INSERT INTO tab(rowid,a,b,c,...) VALUES(...)"
     ** instead of the usual "INSERT INTO tab VALUES(...)".
     */
-    if( azCol[0] ){
+    if( azTCol[0] ){
       appendText(&sTable, "(", 0);
-      appendText(&sTable, azCol[0], 0);
+      appendText(&sTable, azTCol[0], 0);
-      for(i=1; azCol[i]; i++){
+      for(i=1; azTCol[i]; i++){
         appendText(&sTable, ",", 0);
-        appendText(&sTable, azCol[i], quoteChar(azCol[i]));
+        appendText(&sTable, azTCol[i], quoteChar(azTCol[i]));
       }
       appendText(&sTable, ")", 0);
     }
@@ -487,19 +453,19 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){
     /* Build an appropriate SELECT statement */
     initText(&sSelect);
     appendText(&sSelect, "SELECT ", 0);
-    if( azCol[0] ){
+    if( azTCol[0] ){
-      appendText(&sSelect, azCol[0], 0);
+      appendText(&sSelect, azTCol[0], 0);
       appendText(&sSelect, ",", 0);
     }
-    for(i=1; azCol[i]; i++){
+    for(i=1; azTCol[i]; i++){
-      appendText(&sSelect, azCol[i], quoteChar(azCol[i]));
+      appendText(&sSelect, azTCol[i], quoteChar(azTCol[i]));
-      if( azCol[i+1] ){
+      if( azTCol[i+1] ){
         appendText(&sSelect, ",", 0);
       }
     }
     nCol = i;
-    if( azCol[0]==0 ) nCol--;
+    if( azTCol[0]==0 ) nCol--;
-    freeColumnList(azCol);
+    freeColumnList(azTCol);
     appendText(&sSelect, " FROM ", 0);
     appendText(&sSelect, zTable, quoteChar(zTable));
 
@@ -519,7 +485,15 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){
             }
             case SQLITE_FLOAT: {
               double r = sqlite3_column_double(pStmt,i);
-              output_formatted(p, "%!.20g", r);
+              sqlite3_uint64 ur;
+              memcpy(&ur,&r,sizeof(r));
+              if( ur==0x7ff0000000000000LL ){
+                p->xCallback("1e999", p->pArg);
+              }else if( ur==0xfff0000000000000LL ){
+                p->xCallback("-1e999", p->pArg);
+              }else{
+                output_formatted(p, "%!.20g", r);
+              }
               break;
             }
             case SQLITE_NULL: {

ext/misc/eval.c

@@ -34,6 +34,7 @@ struct EvalResult {
 static int callback(void *pCtx, int argc, char **argv, char **colnames){
   struct EvalResult *p = (struct EvalResult*)pCtx;
   int i; 
+  if( argv==0 ) return 0;
   for(i=0; i<argc; i++){
     const char *z = argv[i] ? argv[i] : "";
     size_t sz = strlen(z);

ext/misc/explain.c

@@ -0,0 +1,309 @@
+/*
+** 2018-09-16
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** This file demonstrates an eponymous virtual table that returns the
+** EXPLAIN output from an SQL statement.
+**
+** Usage example:
+**
+**     .load ./explain
+**     SELECT p2 FROM explain('SELECT * FROM sqlite_master')
+**      WHERE opcode='OpenRead';
+*/
+#if !defined(SQLITEINT_H)
+#include "sqlite3ext.h"
+#endif
+SQLITE_EXTENSION_INIT1
+#include <assert.h>
+#include <string.h>
+
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+
+/* explain_vtab is a subclass of sqlite3_vtab which will
+** serve as the underlying representation of a explain virtual table
+*/
+typedef struct explain_vtab explain_vtab;
+struct explain_vtab {
+  sqlite3_vtab base;  /* Base class - must be first */
+  sqlite3 *db;        /* Database connection for this explain vtab */
+};
+
+/* explain_cursor is a subclass of sqlite3_vtab_cursor which will
+** serve as the underlying representation of a cursor that scans
+** over rows of the result from an EXPLAIN operation.
+*/
+typedef struct explain_cursor explain_cursor;
+struct explain_cursor {
+  sqlite3_vtab_cursor base;  /* Base class - must be first */
+  sqlite3 *db;               /* Database connection for this cursor */
+  char *zSql;                /* Value for the EXPLN_COLUMN_SQL column */
+  sqlite3_stmt *pExplain;    /* Statement being explained */
+  int rc;                    /* Result of last sqlite3_step() on pExplain */
+};
+
+/*
+** The explainConnect() method is invoked to create a new
+** explain_vtab that describes the explain virtual table.
+**
+** Think of this routine as the constructor for explain_vtab objects.
+**
+** All this routine needs to do is:
+**
+**    (1) Allocate the explain_vtab object and initialize all fields.
+**
+**    (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the
+**        result set of queries against explain will look like.
+*/
+static int explainConnect(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
+){
+  explain_vtab *pNew;
+  int rc;
+
+/* Column numbers */
+#define EXPLN_COLUMN_ADDR     0   /* Instruction address */
+#define EXPLN_COLUMN_OPCODE   1   /* Opcode */
+#define EXPLN_COLUMN_P1       2   /* Operand 1 */
+#define EXPLN_COLUMN_P2       3   /* Operand 2 */
+#define EXPLN_COLUMN_P3       4   /* Operand 3 */
+#define EXPLN_COLUMN_P4       5   /* Operand 4 */
+#define EXPLN_COLUMN_P5       6   /* Operand 5 */
+#define EXPLN_COLUMN_COMMENT  7   /* Comment */
+#define EXPLN_COLUMN_SQL      8   /* SQL that is being explained */
+
+
+  rc = sqlite3_declare_vtab(db,
+     "CREATE TABLE x(addr,opcode,p1,p2,p3,p4,p5,comment,sql HIDDEN)");
+  if( rc==SQLITE_OK ){
+    pNew = sqlite3_malloc( sizeof(*pNew) );
+    *ppVtab = (sqlite3_vtab*)pNew;
+    if( pNew==0 ) return SQLITE_NOMEM;
+    memset(pNew, 0, sizeof(*pNew));
+    pNew->db = db;
+  }
+  return rc;
+}
+
+/*
+** This method is the destructor for explain_cursor objects.
+*/
+static int explainDisconnect(sqlite3_vtab *pVtab){
+  sqlite3_free(pVtab);
+  return SQLITE_OK;
+}
+
+/*
+** Constructor for a new explain_cursor object.
+*/
+static int explainOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
+  explain_cursor *pCur;
+  pCur = sqlite3_malloc( sizeof(*pCur) );
+  if( pCur==0 ) return SQLITE_NOMEM;
+  memset(pCur, 0, sizeof(*pCur));
+  pCur->db = ((explain_vtab*)p)->db;
+  *ppCursor = &pCur->base;
+  return SQLITE_OK;
+}
+
+/*
+** Destructor for a explain_cursor.
+*/
+static int explainClose(sqlite3_vtab_cursor *cur){
+  explain_cursor *pCur = (explain_cursor*)cur;
+  sqlite3_finalize(pCur->pExplain);
+  sqlite3_free(pCur->zSql);
+  sqlite3_free(pCur);
+  return SQLITE_OK;
+}
+
+
+/*
+** Advance a explain_cursor to its next row of output.
+*/
+static int explainNext(sqlite3_vtab_cursor *cur){
+  explain_cursor *pCur = (explain_cursor*)cur;
+  pCur->rc = sqlite3_step(pCur->pExplain);
+  if( pCur->rc!=SQLITE_DONE && pCur->rc!=SQLITE_ROW ) return pCur->rc;
+  return SQLITE_OK;
+}
+
+/*
+** Return values of columns for the row at which the explain_cursor
+** is currently pointing.
+*/
+static int explainColumn(
+  sqlite3_vtab_cursor *cur,   /* The cursor */
+  sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
+  int i                       /* Which column to return */
+){
+  explain_cursor *pCur = (explain_cursor*)cur;
+  if( i==EXPLN_COLUMN_SQL ){
+    sqlite3_result_text(ctx, pCur->zSql, -1, SQLITE_TRANSIENT);
+  }else{
+    sqlite3_result_value(ctx, sqlite3_column_value(pCur->pExplain, i));
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Return the rowid for the current row.  In this implementation, the
+** rowid is the same as the output value.
+*/
+static int explainRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
+  explain_cursor *pCur = (explain_cursor*)cur;
+  *pRowid = sqlite3_column_int64(pCur->pExplain, 0);
+  return SQLITE_OK;
+}
+
+/*
+** Return TRUE if the cursor has been moved off of the last
+** row of output.
+*/
+static int explainEof(sqlite3_vtab_cursor *cur){
+  explain_cursor *pCur = (explain_cursor*)cur;
+  return pCur->rc!=SQLITE_ROW;
+}
+
+/*
+** This method is called to "rewind" the explain_cursor object back
+** to the first row of output.  This method is always called at least
+** once prior to any call to explainColumn() or explainRowid() or 
+** explainEof().
+**
+** The argv[0] is the SQL statement that is to be explained.
+*/
+static int explainFilter(
+  sqlite3_vtab_cursor *pVtabCursor, 
+  int idxNum, const char *idxStr,
+  int argc, sqlite3_value **argv
+){
+  explain_cursor *pCur = (explain_cursor *)pVtabCursor;
+  char *zSql = 0;
+  int rc;
+  sqlite3_finalize(pCur->pExplain);
+  pCur->pExplain = 0;
+  if( sqlite3_value_type(argv[0])!=SQLITE_TEXT ){
+    pCur->rc = SQLITE_DONE;
+    return SQLITE_OK;
+  }
+  sqlite3_free(pCur->zSql);
+  pCur->zSql = sqlite3_mprintf("%s", sqlite3_value_text(argv[0]));
+  if( pCur->zSql ){
+    zSql = sqlite3_mprintf("EXPLAIN %s", pCur->zSql);
+  }
+  if( zSql==0 ){
+    rc = SQLITE_NOMEM;
+  }else{
+    rc = sqlite3_prepare_v2(pCur->db, zSql, -1, &pCur->pExplain, 0);
+    sqlite3_free(zSql);
+  }
+  if( rc ){
+    sqlite3_finalize(pCur->pExplain);
+    pCur->pExplain = 0;
+    sqlite3_free(pCur->zSql);
+    pCur->zSql = 0;
+  }else{
+    pCur->rc = sqlite3_step(pCur->pExplain);
+    rc = (pCur->rc==SQLITE_DONE || pCur->rc==SQLITE_ROW) ? SQLITE_OK : pCur->rc;
+  }
+  return rc;
+}
+
+/*
+** SQLite will invoke this method one or more times while planning a query
+** that uses the explain virtual table.  This routine needs to create
+** a query plan for each invocation and compute an estimated cost for that
+** plan.
+*/
+static int explainBestIndex(
+  sqlite3_vtab *tab,
+  sqlite3_index_info *pIdxInfo
+){
+  int i;
+
+  pIdxInfo->estimatedCost = (double)1000000;
+  pIdxInfo->estimatedRows = 500;
+  for(i=0; i<pIdxInfo->nConstraint; i++){
+    struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[i];
+    if( p->usable
+     && p->iColumn==EXPLN_COLUMN_SQL
+     && p->op==SQLITE_INDEX_CONSTRAINT_EQ
+    ){
+      pIdxInfo->estimatedCost = 10.0;
+      pIdxInfo->idxNum = 1;
+      pIdxInfo->aConstraintUsage[i].argvIndex = 1;
+      pIdxInfo->aConstraintUsage[i].omit = 1;
+      break;
+    }
+  }
+  return SQLITE_OK;
+}
+
+/*
+** This following structure defines all the methods for the 
+** explain virtual table.
+*/
+static sqlite3_module explainModule = {
+  0,                         /* iVersion */
+  0,                         /* xCreate */
+  explainConnect,            /* xConnect */
+  explainBestIndex,          /* xBestIndex */
+  explainDisconnect,         /* xDisconnect */
+  0,                         /* xDestroy */
+  explainOpen,               /* xOpen - open a cursor */
+  explainClose,              /* xClose - close a cursor */
+  explainFilter,             /* xFilter - configure scan constraints */
+  explainNext,               /* xNext - advance a cursor */
+  explainEof,                /* xEof - check for end of scan */
+  explainColumn,             /* xColumn - read data */
+  explainRowid,              /* xRowid - read data */
+  0,                         /* xUpdate */
+  0,                         /* xBegin */
+  0,                         /* xSync */
+  0,                         /* xCommit */
+  0,                         /* xRollback */
+  0,                         /* xFindMethod */
+  0,                         /* xRename */
+  0,                         /* xSavepoint */
+  0,                         /* xRelease */
+  0,                         /* xRollbackTo */
+};
+
+#endif /* SQLITE_OMIT_VIRTUALTABLE */
+
+int sqlite3ExplainVtabInit(sqlite3 *db){
+  int rc = SQLITE_OK;
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+  rc = sqlite3_create_module(db, "explain", &explainModule, 0);
+#endif
+  return rc;
+}
+
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_explain_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  int rc = SQLITE_OK;
+  SQLITE_EXTENSION_INIT2(pApi);
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+  rc = sqlite3ExplainVtabInit(db);
+#endif
+  return rc;
+}

ext/misc/fileio.c

@@ -11,11 +11,125 @@
 ******************************************************************************
 **
 ** This SQLite extension implements SQL functions readfile() and
-** writefile().
+** writefile(), and eponymous virtual type "fsdir".
+**
+** WRITEFILE(FILE, DATA [, MODE [, MTIME]]):
+**
+**   If neither of the optional arguments is present, then this UDF
+**   function writes blob DATA to file FILE. If successful, the number
+**   of bytes written is returned. If an error occurs, NULL is returned.
+**
+**   If the first option argument - MODE - is present, then it must
+**   be passed an integer value that corresponds to a POSIX mode
+**   value (file type + permissions, as returned in the stat.st_mode
+**   field by the stat() system call). Three types of files may
+**   be written/created:
+**
+**     regular files:  (mode & 0170000)==0100000
+**     symbolic links: (mode & 0170000)==0120000
+**     directories:    (mode & 0170000)==0040000
+**
+**   For a directory, the DATA is ignored. For a symbolic link, it is
+**   interpreted as text and used as the target of the link. For a
+**   regular file, it is interpreted as a blob and written into the
+**   named file. Regardless of the type of file, its permissions are
+**   set to (mode & 0777) before returning.
+**
+**   If the optional MTIME argument is present, then it is interpreted
+**   as an integer - the number of seconds since the unix epoch. The
+**   modification-time of the target file is set to this value before
+**   returning.
+**
+**   If three or more arguments are passed to this function and an
+**   error is encountered, an exception is raised.
+**
+** READFILE(FILE):
+**
+**   Read and return the contents of file FILE (type blob) from disk.
+**
+** FSDIR:
+**
+**   Used as follows:
+**
+**     SELECT * FROM fsdir($path [, $dir]);
+**
+**   Parameter $path is an absolute or relative pathname. If the file that it
+**   refers to does not exist, it is an error. If the path refers to a regular
+**   file or symbolic link, it returns a single row. Or, if the path refers
+**   to a directory, it returns one row for the directory, and one row for each
+**   file within the hierarchy rooted at $path.
+**
+**   Each row has the following columns:
+**
+**     name:  Path to file or directory (text value).
+**     mode:  Value of stat.st_mode for directory entry (an integer).
+**     mtime: Value of stat.st_mtime for directory entry (an integer).
+**     data:  For a regular file, a blob containing the file data. For a
+**            symlink, a text value containing the text of the link. For a
+**            directory, NULL.
+**
+**   If a non-NULL value is specified for the optional $dir parameter and
+**   $path is a relative path, then $path is interpreted relative to $dir. 
+**   And the paths returned in the "name" column of the table are also 
+**   relative to directory $dir.
 */
 #include "sqlite3ext.h"
 SQLITE_EXTENSION_INIT1
 #include <stdio.h>
+#include <string.h>
+#include <assert.h>
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#if !defined(_WIN32) && !defined(WIN32)
+#  include <unistd.h>
+#  include <dirent.h>
+#  include <utime.h>
+#  include <sys/time.h>
+#else
+#  include "windows.h"
+#  include <io.h>
+#  include <direct.h>
+#  include "test_windirent.h"
+#  define dirent DIRENT
+#  ifndef chmod
+#    define chmod _chmod
+#  endif
+#  ifndef stat
+#    define stat _stat
+#  endif
+#  define mkdir(path,mode) _mkdir(path)
+#  define lstat(path,buf) stat(path,buf)
+#endif
+#include <time.h>
+#include <errno.h>
+
+
+#define FSDIR_SCHEMA "(name,mode,mtime,data,path HIDDEN,dir HIDDEN)"
+
+/*
+** Set the result stored by context ctx to a blob containing the 
+** contents of file zName.
+*/
+static void readFileContents(sqlite3_context *ctx, const char *zName){
+  FILE *in;
+  long nIn;
+  void *pBuf;
+
+  in = fopen(zName, "rb");
+  if( in==0 ) return;
+  fseek(in, 0, SEEK_END);
+  nIn = ftell(in);
+  rewind(in);
+  pBuf = sqlite3_malloc( nIn );
+  if( pBuf && 1==fread(pBuf, nIn, 1, in) ){
+    sqlite3_result_blob(ctx, pBuf, nIn, sqlite3_free);
+  }else{
+    sqlite3_free(pBuf);
+  }
+  fclose(in);
+}
 
 /*
 ** Implementation of the "readfile(X)" SQL function.  The entire content
@@ -28,58 +142,761 @@ static void readfileFunc(
   sqlite3_value **argv
 ){
   const char *zName;
-  FILE *in;
-  long nIn;
-  void *pBuf;
-
   (void)(argc);  /* Unused parameter */
   zName = (const char*)sqlite3_value_text(argv[0]);
   if( zName==0 ) return;
-  in = fopen(zName, "rb");
+  readFileContents(context, zName);
-  if( in==0 ) return;
-  fseek(in, 0, SEEK_END);
-  nIn = ftell(in);
-  rewind(in);
-  pBuf = sqlite3_malloc( nIn );
-  if( pBuf && 1==fread(pBuf, nIn, 1, in) ){
-    sqlite3_result_blob(context, pBuf, nIn, sqlite3_free);
-  }else{
-    sqlite3_free(pBuf);
-  }
-  fclose(in);
 }
 
 /*
-** Implementation of the "writefile(X,Y)" SQL function.  The argument Y
+** Set the error message contained in context ctx to the results of
-** is written into file X.  The number of bytes written is returned.  Or
+** vprintf(zFmt, ...).
-** NULL is returned if something goes wrong, such as being unable to open
+*/
-** file X for writing.
+static void ctxErrorMsg(sqlite3_context *ctx, const char *zFmt, ...){
+  char *zMsg = 0;
+  va_list ap;
+  va_start(ap, zFmt);
+  zMsg = sqlite3_vmprintf(zFmt, ap);
+  sqlite3_result_error(ctx, zMsg, -1);
+  sqlite3_free(zMsg);
+  va_end(ap);
+}
+
+#if defined(_WIN32)
+/*
+** This function is designed to convert a Win32 FILETIME structure into the
+** number of seconds since the Unix Epoch (1970-01-01 00:00:00 UTC).
+*/
+static sqlite3_uint64 fileTimeToUnixTime(
+  LPFILETIME pFileTime
+){
+  SYSTEMTIME epochSystemTime;
+  ULARGE_INTEGER epochIntervals;
+  FILETIME epochFileTime;
+  ULARGE_INTEGER fileIntervals;
+
+  memset(&epochSystemTime, 0, sizeof(SYSTEMTIME));
+  epochSystemTime.wYear = 1970;
+  epochSystemTime.wMonth = 1;
+  epochSystemTime.wDay = 1;
+  SystemTimeToFileTime(&epochSystemTime, &epochFileTime);
+  epochIntervals.LowPart = epochFileTime.dwLowDateTime;
+  epochIntervals.HighPart = epochFileTime.dwHighDateTime;
+
+  fileIntervals.LowPart = pFileTime->dwLowDateTime;
+  fileIntervals.HighPart = pFileTime->dwHighDateTime;
+
+  return (fileIntervals.QuadPart - epochIntervals.QuadPart) / 10000000;
+}
+
+/*
+** This function attempts to normalize the time values found in the stat()
+** buffer to UTC.  This is necessary on Win32, where the runtime library
+** appears to return these values as local times.
+*/
+static void statTimesToUtc(
+  const char *zPath,
+  struct stat *pStatBuf
+){
+  HANDLE hFindFile;
+  WIN32_FIND_DATAW fd;
+  LPWSTR zUnicodeName;
+  extern LPWSTR sqlite3_win32_utf8_to_unicode(const char*);
+  zUnicodeName = sqlite3_win32_utf8_to_unicode(zPath);
+  if( zUnicodeName ){
+    memset(&fd, 0, sizeof(WIN32_FIND_DATAW));
+    hFindFile = FindFirstFileW(zUnicodeName, &fd);
+    if( hFindFile!=NULL ){
+      pStatBuf->st_ctime = (time_t)fileTimeToUnixTime(&fd.ftCreationTime);
+      pStatBuf->st_atime = (time_t)fileTimeToUnixTime(&fd.ftLastAccessTime);
+      pStatBuf->st_mtime = (time_t)fileTimeToUnixTime(&fd.ftLastWriteTime);
+      FindClose(hFindFile);
+    }
+    sqlite3_free(zUnicodeName);
+  }
+}
+#endif
+
+/*
+** This function is used in place of stat().  On Windows, special handling
+** is required in order for the included time to be returned as UTC.  On all
+** other systems, this function simply calls stat().
+*/
+static int fileStat(
+  const char *zPath,
+  struct stat *pStatBuf
+){
+#if defined(_WIN32)
+  int rc = stat(zPath, pStatBuf);
+  if( rc==0 ) statTimesToUtc(zPath, pStatBuf);
+  return rc;
+#else
+  return stat(zPath, pStatBuf);
+#endif
+}
+
+/*
+** This function is used in place of lstat().  On Windows, special handling
+** is required in order for the included time to be returned as UTC.  On all
+** other systems, this function simply calls lstat().
+*/
+static int fileLinkStat(
+  const char *zPath,
+  struct stat *pStatBuf
+){
+#if defined(_WIN32)
+  int rc = lstat(zPath, pStatBuf);
+  if( rc==0 ) statTimesToUtc(zPath, pStatBuf);
+  return rc;
+#else
+  return lstat(zPath, pStatBuf);
+#endif
+}
+
+/*
+** Argument zFile is the name of a file that will be created and/or written
+** by SQL function writefile(). This function ensures that the directory
+** zFile will be written to exists, creating it if required. The permissions
+** for any path components created by this function are set to (mode&0777).
+**
+** If an OOM condition is encountered, SQLITE_NOMEM is returned. Otherwise,
+** SQLITE_OK is returned if the directory is successfully created, or
+** SQLITE_ERROR otherwise.
+*/
+static int makeDirectory(
+  const char *zFile,
+  mode_t mode
+){
+  char *zCopy = sqlite3_mprintf("%s", zFile);
+  int rc = SQLITE_OK;
+
+  if( zCopy==0 ){
+    rc = SQLITE_NOMEM;
+  }else{
+    int nCopy = (int)strlen(zCopy);
+    int i = 1;
+
+    while( rc==SQLITE_OK ){
+      struct stat sStat;
+      int rc2;
+
+      for(; zCopy[i]!='/' && i<nCopy; i++);
+      if( i==nCopy ) break;
+      zCopy[i] = '\0';
+
+      rc2 = fileStat(zCopy, &sStat);
+      if( rc2!=0 ){
+        if( mkdir(zCopy, mode & 0777) ) rc = SQLITE_ERROR;
+      }else{
+        if( !S_ISDIR(sStat.st_mode) ) rc = SQLITE_ERROR;
+      }
+      zCopy[i] = '/';
+      i++;
+    }
+
+    sqlite3_free(zCopy);
+  }
+
+  return rc;
+}
+
+/*
+** This function does the work for the writefile() UDF. Refer to 
+** header comments at the top of this file for details.
+*/
+static int writeFile(
+  sqlite3_context *pCtx,          /* Context to return bytes written in */
+  const char *zFile,              /* File to write */
+  sqlite3_value *pData,           /* Data to write */
+  mode_t mode,                    /* MODE parameter passed to writefile() */
+  sqlite3_int64 mtime             /* MTIME parameter (or -1 to not set time) */
+){
+#if !defined(_WIN32) && !defined(WIN32)
+  if( S_ISLNK(mode) ){
+    const char *zTo = (const char*)sqlite3_value_text(pData);
+    if( symlink(zTo, zFile)<0 ) return 1;
+  }else
+#endif
+  {
+    if( S_ISDIR(mode) ){
+      if( mkdir(zFile, mode) ){
+        /* The mkdir() call to create the directory failed. This might not
+        ** be an error though - if there is already a directory at the same
+        ** path and either the permissions already match or can be changed
+        ** to do so using chmod(), it is not an error.  */
+        struct stat sStat;
+        if( errno!=EEXIST
+         || 0!=fileStat(zFile, &sStat)
+         || !S_ISDIR(sStat.st_mode)
+         || ((sStat.st_mode&0777)!=(mode&0777) && 0!=chmod(zFile, mode&0777))
+        ){
+          return 1;
+        }
+      }
+    }else{
+      sqlite3_int64 nWrite = 0;
+      const char *z;
+      int rc = 0;
+      FILE *out = fopen(zFile, "wb");
+      if( out==0 ) return 1;
+      z = (const char*)sqlite3_value_blob(pData);
+      if( z ){
+        sqlite3_int64 n = fwrite(z, 1, sqlite3_value_bytes(pData), out);
+        nWrite = sqlite3_value_bytes(pData);
+        if( nWrite!=n ){
+          rc = 1;
+        }
+      }
+      fclose(out);
+      if( rc==0 && mode && chmod(zFile, mode & 0777) ){
+        rc = 1;
+      }
+      if( rc ) return 2;
+      sqlite3_result_int64(pCtx, nWrite);
+    }
+  }
+
+  if( mtime>=0 ){
+#if defined(_WIN32)
+    /* Windows */
+    FILETIME lastAccess;
+    FILETIME lastWrite;
+    SYSTEMTIME currentTime;
+    LONGLONG intervals;
+    HANDLE hFile;
+    LPWSTR zUnicodeName;
+    extern LPWSTR sqlite3_win32_utf8_to_unicode(const char*);
+
+    GetSystemTime(&currentTime);
+    SystemTimeToFileTime(&currentTime, &lastAccess);
+    intervals = Int32x32To64(mtime, 10000000) + 116444736000000000;
+    lastWrite.dwLowDateTime = (DWORD)intervals;
+    lastWrite.dwHighDateTime = intervals >> 32;
+    zUnicodeName = sqlite3_win32_utf8_to_unicode(zFile);
+    if( zUnicodeName==0 ){
+      return 1;
+    }
+    hFile = CreateFileW(
+      zUnicodeName, FILE_WRITE_ATTRIBUTES, 0, NULL, OPEN_EXISTING,
+      FILE_FLAG_BACKUP_SEMANTICS, NULL
+    );
+    sqlite3_free(zUnicodeName);
+    if( hFile!=INVALID_HANDLE_VALUE ){
+      BOOL bResult = SetFileTime(hFile, NULL, &lastAccess, &lastWrite);
+      CloseHandle(hFile);
+      return !bResult;
+    }else{
+      return 1;
+    }
+#elif defined(AT_FDCWD) && 0 /* utimensat() is not universally available */
+    /* Recent unix */
+    struct timespec times[2];
+    times[0].tv_nsec = times[1].tv_nsec = 0;
+    times[0].tv_sec = time(0);
+    times[1].tv_sec = mtime;
+    if( utimensat(AT_FDCWD, zFile, times, AT_SYMLINK_NOFOLLOW) ){
+      return 1;
+    }
+#else
+    /* Legacy unix */
+    struct timeval times[2];
+    times[0].tv_usec = times[1].tv_usec = 0;
+    times[0].tv_sec = time(0);
+    times[1].tv_sec = mtime;
+    if( utimes(zFile, times) ){
+      return 1;
+    }
+#endif
+  }
+
+  return 0;
+}
+
+/*
+** Implementation of the "writefile(W,X[,Y[,Z]]])" SQL function.  
+** Refer to header comments at the top of this file for details.
 */
 static void writefileFunc(
   sqlite3_context *context,
   int argc,
   sqlite3_value **argv
 ){
-  FILE *out;
-  const char *z;
-  sqlite3_int64 rc;
   const char *zFile;
+  mode_t mode = 0;
+  int res;
+  sqlite3_int64 mtime = -1;
+
+  if( argc<2 || argc>4 ){
+    sqlite3_result_error(context, 
+        "wrong number of arguments to function writefile()", -1
+    );
+    return;
+  }
 
-  (void)(argc);  /* Unused parameter */
   zFile = (const char*)sqlite3_value_text(argv[0]);
   if( zFile==0 ) return;
-  out = fopen(zFile, "wb");
+  if( argc>=3 ){
-  if( out==0 ) return;
+    mode = (mode_t)sqlite3_value_int(argv[2]);
-  z = (const char*)sqlite3_value_blob(argv[1]);
+  }
-  if( z==0 ){
+  if( argc==4 ){
-    rc = 0;
+    mtime = sqlite3_value_int64(argv[3]);
-  }else{
+  }
-    rc = fwrite(z, 1, sqlite3_value_bytes(argv[1]), out);
+
+  res = writeFile(context, zFile, argv[1], mode, mtime);
+  if( res==1 && errno==ENOENT ){
+    if( makeDirectory(zFile, mode)==SQLITE_OK ){
+      res = writeFile(context, zFile, argv[1], mode, mtime);
+    }
+  }
+
+  if( argc>2 && res!=0 ){
+    if( S_ISLNK(mode) ){
+      ctxErrorMsg(context, "failed to create symlink: %s", zFile);
+    }else if( S_ISDIR(mode) ){
+      ctxErrorMsg(context, "failed to create directory: %s", zFile);
+    }else{
+      ctxErrorMsg(context, "failed to write file: %s", zFile);
+    }
   }
-  fclose(out);
-  sqlite3_result_int64(context, rc);
 }
 
+/*
+** SQL function:   lsmode(MODE)
+**
+** Given a numberic st_mode from stat(), convert it into a human-readable
+** text string in the style of "ls -l".
+*/
+static void lsModeFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  int i;
+  int iMode = sqlite3_value_int(argv[0]);
+  char z[16];
+  (void)argc;
+  if( S_ISLNK(iMode) ){
+    z[0] = 'l';
+  }else if( S_ISREG(iMode) ){
+    z[0] = '-';
+  }else if( S_ISDIR(iMode) ){
+    z[0] = 'd';
+  }else{
+    z[0] = '?';
+  }
+  for(i=0; i<3; i++){
+    int m = (iMode >> ((2-i)*3));
+    char *a = &z[1 + i*3];
+    a[0] = (m & 0x4) ? 'r' : '-';
+    a[1] = (m & 0x2) ? 'w' : '-';
+    a[2] = (m & 0x1) ? 'x' : '-';
+  }
+  z[10] = '\0';
+  sqlite3_result_text(context, z, -1, SQLITE_TRANSIENT);
+}
+
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+
+/* 
+** Cursor type for recursively iterating through a directory structure.
+*/
+typedef struct fsdir_cursor fsdir_cursor;
+typedef struct FsdirLevel FsdirLevel;
+
+struct FsdirLevel {
+  DIR *pDir;                 /* From opendir() */
+  char *zDir;                /* Name of directory (nul-terminated) */
+};
+
+struct fsdir_cursor {
+  sqlite3_vtab_cursor base;  /* Base class - must be first */
+
+  int nLvl;                  /* Number of entries in aLvl[] array */
+  int iLvl;                  /* Index of current entry */
+  FsdirLevel *aLvl;          /* Hierarchy of directories being traversed */
+
+  const char *zBase;
+  int nBase;
+
+  struct stat sStat;         /* Current lstat() results */
+  char *zPath;               /* Path to current entry */
+  sqlite3_int64 iRowid;      /* Current rowid */
+};
+
+typedef struct fsdir_tab fsdir_tab;
+struct fsdir_tab {
+  sqlite3_vtab base;         /* Base class - must be first */
+};
+
+/*
+** Construct a new fsdir virtual table object.
+*/
+static int fsdirConnect(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
+){
+  fsdir_tab *pNew = 0;
+  int rc;
+  (void)pAux;
+  (void)argc;
+  (void)argv;
+  (void)pzErr;
+  rc = sqlite3_declare_vtab(db, "CREATE TABLE x" FSDIR_SCHEMA);
+  if( rc==SQLITE_OK ){
+    pNew = (fsdir_tab*)sqlite3_malloc( sizeof(*pNew) );
+    if( pNew==0 ) return SQLITE_NOMEM;
+    memset(pNew, 0, sizeof(*pNew));
+  }
+  *ppVtab = (sqlite3_vtab*)pNew;
+  return rc;
+}
+
+/*
+** This method is the destructor for fsdir vtab objects.
+*/
+static int fsdirDisconnect(sqlite3_vtab *pVtab){
+  sqlite3_free(pVtab);
+  return SQLITE_OK;
+}
+
+/*
+** Constructor for a new fsdir_cursor object.
+*/
+static int fsdirOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
+  fsdir_cursor *pCur;
+  (void)p;
+  pCur = sqlite3_malloc( sizeof(*pCur) );
+  if( pCur==0 ) return SQLITE_NOMEM;
+  memset(pCur, 0, sizeof(*pCur));
+  pCur->iLvl = -1;
+  *ppCursor = &pCur->base;
+  return SQLITE_OK;
+}
+
+/*
+** Reset a cursor back to the state it was in when first returned
+** by fsdirOpen().
+*/
+static void fsdirResetCursor(fsdir_cursor *pCur){
+  int i;
+  for(i=0; i<=pCur->iLvl; i++){
+    FsdirLevel *pLvl = &pCur->aLvl[i];
+    if( pLvl->pDir ) closedir(pLvl->pDir);
+    sqlite3_free(pLvl->zDir);
+  }
+  sqlite3_free(pCur->zPath);
+  sqlite3_free(pCur->aLvl);
+  pCur->aLvl = 0;
+  pCur->zPath = 0;
+  pCur->zBase = 0;
+  pCur->nBase = 0;
+  pCur->nLvl = 0;
+  pCur->iLvl = -1;
+  pCur->iRowid = 1;
+}
+
+/*
+** Destructor for an fsdir_cursor.
+*/
+static int fsdirClose(sqlite3_vtab_cursor *cur){
+  fsdir_cursor *pCur = (fsdir_cursor*)cur;
+
+  fsdirResetCursor(pCur);
+  sqlite3_free(pCur);
+  return SQLITE_OK;
+}
+
+/*
+** Set the error message for the virtual table associated with cursor
+** pCur to the results of vprintf(zFmt, ...).
+*/
+static void fsdirSetErrmsg(fsdir_cursor *pCur, const char *zFmt, ...){
+  va_list ap;
+  va_start(ap, zFmt);
+  pCur->base.pVtab->zErrMsg = sqlite3_vmprintf(zFmt, ap);
+  va_end(ap);
+}
+
+
+/*
+** Advance an fsdir_cursor to its next row of output.
+*/
+static int fsdirNext(sqlite3_vtab_cursor *cur){
+  fsdir_cursor *pCur = (fsdir_cursor*)cur;
+  mode_t m = pCur->sStat.st_mode;
+
+  pCur->iRowid++;
+  if( S_ISDIR(m) ){
+    /* Descend into this directory */
+    int iNew = pCur->iLvl + 1;
+    FsdirLevel *pLvl;
+    if( iNew>=pCur->nLvl ){
+      int nNew = iNew+1;
+      int nByte = nNew*sizeof(FsdirLevel);
+      FsdirLevel *aNew = (FsdirLevel*)sqlite3_realloc(pCur->aLvl, nByte);
+      if( aNew==0 ) return SQLITE_NOMEM;
+      memset(&aNew[pCur->nLvl], 0, sizeof(FsdirLevel)*(nNew-pCur->nLvl));
+      pCur->aLvl = aNew;
+      pCur->nLvl = nNew;
+    }
+    pCur->iLvl = iNew;
+    pLvl = &pCur->aLvl[iNew];
+    
+    pLvl->zDir = pCur->zPath;
+    pCur->zPath = 0;
+    pLvl->pDir = opendir(pLvl->zDir);
+    if( pLvl->pDir==0 ){
+      fsdirSetErrmsg(pCur, "cannot read directory: %s", pCur->zPath);
+      return SQLITE_ERROR;
+    }
+  }
+
+  while( pCur->iLvl>=0 ){
+    FsdirLevel *pLvl = &pCur->aLvl[pCur->iLvl];
+    struct dirent *pEntry = readdir(pLvl->pDir);
+    if( pEntry ){
+      if( pEntry->d_name[0]=='.' ){
+       if( pEntry->d_name[1]=='.' && pEntry->d_name[2]=='\0' ) continue;
+       if( pEntry->d_name[1]=='\0' ) continue;
+      }
+      sqlite3_free(pCur->zPath);
+      pCur->zPath = sqlite3_mprintf("%s/%s", pLvl->zDir, pEntry->d_name);
+      if( pCur->zPath==0 ) return SQLITE_NOMEM;
+      if( fileLinkStat(pCur->zPath, &pCur->sStat) ){
+        fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zPath);
+        return SQLITE_ERROR;
+      }
+      return SQLITE_OK;
+    }
+    closedir(pLvl->pDir);
+    sqlite3_free(pLvl->zDir);
+    pLvl->pDir = 0;
+    pLvl->zDir = 0;
+    pCur->iLvl--;
+  }
+
+  /* EOF */
+  sqlite3_free(pCur->zPath);
+  pCur->zPath = 0;
+  return SQLITE_OK;
+}
+
+/*
+** Return values of columns for the row at which the series_cursor
+** is currently pointing.
+*/
+static int fsdirColumn(
+  sqlite3_vtab_cursor *cur,   /* The cursor */
+  sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
+  int i                       /* Which column to return */
+){
+  fsdir_cursor *pCur = (fsdir_cursor*)cur;
+  switch( i ){
+    case 0: { /* name */
+      sqlite3_result_text(ctx, &pCur->zPath[pCur->nBase], -1, SQLITE_TRANSIENT);
+      break;
+    }
+
+    case 1: /* mode */
+      sqlite3_result_int64(ctx, pCur->sStat.st_mode);
+      break;
+
+    case 2: /* mtime */
+      sqlite3_result_int64(ctx, pCur->sStat.st_mtime);
+      break;
+
+    case 3: { /* data */
+      mode_t m = pCur->sStat.st_mode;
+      if( S_ISDIR(m) ){
+        sqlite3_result_null(ctx);
+#if !defined(_WIN32) && !defined(WIN32)
+      }else if( S_ISLNK(m) ){
+        char aStatic[64];
+        char *aBuf = aStatic;
+        int nBuf = 64;
+        int n;
+
+        while( 1 ){
+          n = readlink(pCur->zPath, aBuf, nBuf);
+          if( n<nBuf ) break;
+          if( aBuf!=aStatic ) sqlite3_free(aBuf);
+          nBuf = nBuf*2;
+          aBuf = sqlite3_malloc(nBuf);
+          if( aBuf==0 ){
+            sqlite3_result_error_nomem(ctx);
+            return SQLITE_NOMEM;
+          }
+        }
+
+        sqlite3_result_text(ctx, aBuf, n, SQLITE_TRANSIENT);
+        if( aBuf!=aStatic ) sqlite3_free(aBuf);
+#endif
+      }else{
+        readFileContents(ctx, pCur->zPath);
+      }
+    }
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Return the rowid for the current row. In this implementation, the
+** first row returned is assigned rowid value 1, and each subsequent
+** row a value 1 more than that of the previous.
+*/
+static int fsdirRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
+  fsdir_cursor *pCur = (fsdir_cursor*)cur;
+  *pRowid = pCur->iRowid;
+  return SQLITE_OK;
+}
+
+/*
+** Return TRUE if the cursor has been moved off of the last
+** row of output.
+*/
+static int fsdirEof(sqlite3_vtab_cursor *cur){
+  fsdir_cursor *pCur = (fsdir_cursor*)cur;
+  return (pCur->zPath==0);
+}
+
+/*
+** xFilter callback.
+*/
+static int fsdirFilter(
+  sqlite3_vtab_cursor *cur, 
+  int idxNum, const char *idxStr,
+  int argc, sqlite3_value **argv
+){
+  const char *zDir = 0;
+  fsdir_cursor *pCur = (fsdir_cursor*)cur;
+  (void)idxStr;
+  fsdirResetCursor(pCur);
+
+  if( idxNum==0 ){
+    fsdirSetErrmsg(pCur, "table function fsdir requires an argument");
+    return SQLITE_ERROR;
+  }
+
+  assert( argc==idxNum && (argc==1 || argc==2) );
+  zDir = (const char*)sqlite3_value_text(argv[0]);
+  if( zDir==0 ){
+    fsdirSetErrmsg(pCur, "table function fsdir requires a non-NULL argument");
+    return SQLITE_ERROR;
+  }
+  if( argc==2 ){
+    pCur->zBase = (const char*)sqlite3_value_text(argv[1]);
+  }
+  if( pCur->zBase ){
+    pCur->nBase = (int)strlen(pCur->zBase)+1;
+    pCur->zPath = sqlite3_mprintf("%s/%s", pCur->zBase, zDir);
+  }else{
+    pCur->zPath = sqlite3_mprintf("%s", zDir);
+  }
+
+  if( pCur->zPath==0 ){
+    return SQLITE_NOMEM;
+  }
+  if( fileLinkStat(pCur->zPath, &pCur->sStat) ){
+    fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zPath);
+    return SQLITE_ERROR;
+  }
+
+  return SQLITE_OK;
+}
+
+/*
+** SQLite will invoke this method one or more times while planning a query
+** that uses the generate_series virtual table.  This routine needs to create
+** a query plan for each invocation and compute an estimated cost for that
+** plan.
+**
+** In this implementation idxNum is used to represent the
+** query plan.  idxStr is unused.
+**
+** The query plan is represented by bits in idxNum:
+**
+**  (1)  start = $value  -- constraint exists
+**  (2)  stop = $value   -- constraint exists
+**  (4)  step = $value   -- constraint exists
+**  (8)  output in descending order
+*/
+static int fsdirBestIndex(
+  sqlite3_vtab *tab,
+  sqlite3_index_info *pIdxInfo
+){
+  int i;                 /* Loop over constraints */
+  int idx4 = -1;
+  int idx5 = -1;
+  const struct sqlite3_index_constraint *pConstraint;
+
+  (void)tab;
+  pConstraint = pIdxInfo->aConstraint;
+  for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){
+    if( pConstraint->usable==0 ) continue;
+    if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;
+    if( pConstraint->iColumn==4 ) idx4 = i;
+    if( pConstraint->iColumn==5 ) idx5 = i;
+  }
+
+  if( idx4<0 ){
+    pIdxInfo->idxNum = 0;
+    pIdxInfo->estimatedCost = (double)(((sqlite3_int64)1) << 50);
+  }else{
+    pIdxInfo->aConstraintUsage[idx4].omit = 1;
+    pIdxInfo->aConstraintUsage[idx4].argvIndex = 1;
+    if( idx5>=0 ){
+      pIdxInfo->aConstraintUsage[idx5].omit = 1;
+      pIdxInfo->aConstraintUsage[idx5].argvIndex = 2;
+      pIdxInfo->idxNum = 2;
+      pIdxInfo->estimatedCost = 10.0;
+    }else{
+      pIdxInfo->idxNum = 1;
+      pIdxInfo->estimatedCost = 100.0;
+    }
+  }
+
+  return SQLITE_OK;
+}
+
+/*
+** Register the "fsdir" virtual table.
+*/
+static int fsdirRegister(sqlite3 *db){
+  static sqlite3_module fsdirModule = {
+    0,                         /* iVersion */
+    0,                         /* xCreate */
+    fsdirConnect,              /* xConnect */
+    fsdirBestIndex,            /* xBestIndex */
+    fsdirDisconnect,           /* xDisconnect */
+    0,                         /* xDestroy */
+    fsdirOpen,                 /* xOpen - open a cursor */
+    fsdirClose,                /* xClose - close a cursor */
+    fsdirFilter,               /* xFilter - configure scan constraints */
+    fsdirNext,                 /* xNext - advance a cursor */
+    fsdirEof,                  /* xEof - check for end of scan */
+    fsdirColumn,               /* xColumn - read data */
+    fsdirRowid,                /* xRowid - read data */
+    0,                         /* xUpdate */
+    0,                         /* xBegin */
+    0,                         /* xSync */
+    0,                         /* xCommit */
+    0,                         /* xRollback */
+    0,                         /* xFindMethod */
+    0,                         /* xRename */
+    0,                         /* xSavepoint */
+    0,                         /* xRelease */
+    0                          /* xRollbackTo */
+  };
+
+  int rc = sqlite3_create_module(db, "fsdir", &fsdirModule, 0);
+  return rc;
+}
+#else         /* SQLITE_OMIT_VIRTUALTABLE */
+# define fsdirRegister(x) SQLITE_OK
+#endif
 
 #ifdef _WIN32
 __declspec(dllexport)
@@ -95,8 +912,15 @@ int sqlite3_fileio_init(
   rc = sqlite3_create_function(db, "readfile", 1, SQLITE_UTF8, 0,
                                readfileFunc, 0, 0);
   if( rc==SQLITE_OK ){
-    rc = sqlite3_create_function(db, "writefile", 2, SQLITE_UTF8, 0,
+    rc = sqlite3_create_function(db, "writefile", -1, SQLITE_UTF8, 0,
                                  writefileFunc, 0, 0);
   }
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_create_function(db, "lsmode", 1, SQLITE_UTF8, 0,
+                                 lsModeFunc, 0, 0);
+  }
+  if( rc==SQLITE_OK ){
+    rc = fsdirRegister(db);
+  }
   return rc;
 }

ext/misc/json1.c

@@ -172,6 +172,7 @@ struct JsonParse {
   u8 nErr;           /* Number of errors seen */
   u16 iDepth;        /* Nesting depth */
   int nJson;         /* Length of the zJson string in bytes */
+  u32 iHold;         /* Replace cache line with the lowest iHold value */
 };
 
 /*
@@ -976,7 +977,8 @@ static int jsonParseFindParents(JsonParse *pParse){
 /*
 ** Magic number used for the JSON parse cache in sqlite3_get_auxdata()
 */
-#define JSON_CACHE_ID  (-429938)
+#define JSON_CACHE_ID  (-429938)  /* First cache entry */
+#define JSON_CACHE_SZ  4          /* Max number of cache entries */
 
 /*
 ** Obtain a complete parse of the JSON found in the first argument
@@ -988,16 +990,42 @@ static int jsonParseFindParents(JsonParse *pParse){
 */
 static JsonParse *jsonParseCached(
   sqlite3_context *pCtx,
-  sqlite3_value **argv
+  sqlite3_value **argv,
+  sqlite3_context *pErrCtx
 ){
   const char *zJson = (const char*)sqlite3_value_text(argv[0]);
   int nJson = sqlite3_value_bytes(argv[0]);
   JsonParse *p;
+  JsonParse *pMatch = 0;
+  int iKey;
+  int iMinKey = 0;
+  u32 iMinHold = 0xffffffff;
+  u32 iMaxHold = 0;
   if( zJson==0 ) return 0;
-  p = (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID);
+  for(iKey=0; iKey<JSON_CACHE_SZ; iKey++){
-  if( p && p->nJson==nJson && memcmp(p->zJson,zJson,nJson)==0 ){
+    p = (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID+iKey);
-    p->nErr = 0;
+    if( p==0 ){
-    return p; /* The cached entry matches, so return it */
+      iMinKey = iKey;
+      break;
+    }
+    if( pMatch==0
+     && p->nJson==nJson
+     && memcmp(p->zJson,zJson,nJson)==0
+    ){
+      p->nErr = 0;
+      pMatch = p;
+    }else if( p->iHold<iMinHold ){
+      iMinHold = p->iHold;
+      iMinKey = iKey;
+    }
+    if( p->iHold>iMaxHold ){
+      iMaxHold = p->iHold;
+    }
+  }
+  if( pMatch ){
+    pMatch->nErr = 0;
+    pMatch->iHold = iMaxHold+1;
+    return pMatch;
   }
   p = sqlite3_malloc( sizeof(*p) + nJson + 1 );
   if( p==0 ){
@@ -1007,13 +1035,15 @@ static JsonParse *jsonParseCached(
   memset(p, 0, sizeof(*p));
   p->zJson = (char*)&p[1];
   memcpy((char*)p->zJson, zJson, nJson+1);
-  if( jsonParse(p, pCtx, p->zJson) ){
+  if( jsonParse(p, pErrCtx, p->zJson) ){
     sqlite3_free(p);
     return 0;
   }
   p->nJson = nJson;
-  sqlite3_set_auxdata(pCtx, JSON_CACHE_ID, p, (void(*)(void*))jsonParseFree);
+  p->iHold = iMaxHold+1;
-  return (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID);
+  sqlite3_set_auxdata(pCtx, JSON_CACHE_ID+iMinKey, p,
+                      (void(*)(void*))jsonParseFree);
+  return (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID+iMinKey);
 }
 
 /*
@@ -1386,7 +1416,7 @@ static void jsonArrayLengthFunc(
   u32 i;
   JsonNode *pNode;
 
-  p = jsonParseCached(ctx, argv);
+  p = jsonParseCached(ctx, argv, ctx);
   if( p==0 ) return;
   assert( p->nNode );
   if( argc==2 ){
@@ -1427,7 +1457,7 @@ static void jsonExtractFunc(
   int i;
 
   if( argc<2 ) return;
-  p = jsonParseCached(ctx, argv);
+  p = jsonParseCached(ctx, argv, ctx);
   if( p==0 ) return;
   jsonInit(&jx, ctx);
   jsonAppendChar(&jx, '[');
@@ -1734,22 +1764,21 @@ static void jsonTypeFunc(
   int argc,
   sqlite3_value **argv
 ){
-  JsonParse x;          /* The parse */
+  JsonParse *p;          /* The parse */
   const char *zPath;
   JsonNode *pNode;
 
-  if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return;
+  p = jsonParseCached(ctx, argv, ctx);
-  assert( x.nNode );
+  if( p==0 ) return;
   if( argc==2 ){
     zPath = (const char*)sqlite3_value_text(argv[1]);
-    pNode = jsonLookup(&x, zPath, 0, ctx);
+    pNode = jsonLookup(p, zPath, 0, ctx);
   }else{
-    pNode = x.aNode;
+    pNode = p->aNode;
   }
   if( pNode ){
     sqlite3_result_text(ctx, jsonType[pNode->eType], -1, SQLITE_STATIC);
   }
-  jsonParseReset(&x);
 }
 
 /*
@@ -1763,15 +1792,10 @@ static void jsonValidFunc(
   int argc,
   sqlite3_value **argv
 ){
-  JsonParse x;          /* The parse */
+  JsonParse *p;          /* The parse */
-  int rc = 0;
-
   UNUSED_PARAM(argc);
-  if( jsonParse(&x, 0, (const char*)sqlite3_value_text(argv[0]))==0 ){
+  p = jsonParseCached(ctx, argv, 0);
-    rc = 1;
+  sqlite3_result_int(ctx, p!=0);
-  }
-  jsonParseReset(&x);
-  sqlite3_result_int(ctx, rc);
 }
 
 
@@ -1802,7 +1826,7 @@ static void jsonArrayStep(
     jsonAppendValue(pStr, argv[0]);
   }
 }
-static void jsonArrayFinal(sqlite3_context *ctx){
+static void jsonArrayCompute(sqlite3_context *ctx, int isFinal){
   JsonString *pStr;
   pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0);
   if( pStr ){
@@ -1811,16 +1835,66 @@ static void jsonArrayFinal(sqlite3_context *ctx){
     if( pStr->bErr ){
       if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx);
       assert( pStr->bStatic );
-    }else{
+    }else if( isFinal ){
-      sqlite3_result_text(ctx, pStr->zBuf, pStr->nUsed,
+      sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed,
                           pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free);
       pStr->bStatic = 1;
+    }else{
+      sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT);
+      pStr->nUsed--;
     }
   }else{
     sqlite3_result_text(ctx, "[]", 2, SQLITE_STATIC);
   }
   sqlite3_result_subtype(ctx, JSON_SUBTYPE);
 }
+static void jsonArrayValue(sqlite3_context *ctx){
+  jsonArrayCompute(ctx, 0);
+}
+static void jsonArrayFinal(sqlite3_context *ctx){
+  jsonArrayCompute(ctx, 1);
+}
+
+#ifndef SQLITE_OMIT_WINDOWFUNC
+/*
+** This method works for both json_group_array() and json_group_object().
+** It works by removing the first element of the group by searching forward
+** to the first comma (",") that is not within a string and deleting all
+** text through that comma.
+*/
+static void jsonGroupInverse(
+  sqlite3_context *ctx,
+  int argc,
+  sqlite3_value **argv
+){
+  int i;
+  int inStr = 0;
+  char *z;
+  JsonString *pStr;
+  UNUSED_PARAM(argc);
+  UNUSED_PARAM(argv);
+  pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0);
+#ifdef NEVER
+  /* pStr is always non-NULL since jsonArrayStep() or jsonObjectStep() will
+  ** always have been called to initalize it */
+  if( NEVER(!pStr) ) return;
+#endif
+  z = pStr->zBuf;
+  for(i=1; z[i]!=',' || inStr; i++){
+    assert( i<pStr->nUsed );
+    if( z[i]=='"' ){
+      inStr = !inStr;
+    }else if( z[i]=='\\' ){
+      i++;
+    }
+  }
+  pStr->nUsed -= i;      
+  memmove(&z[1], &z[i+1], (size_t)pStr->nUsed-1);
+}
+#else
+# define jsonGroupInverse 0
+#endif
+
 
 /*
 ** json_group_obj(NAME,VALUE)
@@ -1852,7 +1926,7 @@ static void jsonObjectStep(
     jsonAppendValue(pStr, argv[1]);
   }
 }
-static void jsonObjectFinal(sqlite3_context *ctx){
+static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){
   JsonString *pStr;
   pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0);
   if( pStr ){
@@ -1860,16 +1934,26 @@ static void jsonObjectFinal(sqlite3_context *ctx){
     if( pStr->bErr ){
       if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx);
       assert( pStr->bStatic );
-    }else{
+    }else if( isFinal ){
-      sqlite3_result_text(ctx, pStr->zBuf, pStr->nUsed,
+      sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed,
                           pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free);
       pStr->bStatic = 1;
+    }else{
+      sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT);
+      pStr->nUsed--;
     }
   }else{
     sqlite3_result_text(ctx, "{}", 2, SQLITE_STATIC);
   }
   sqlite3_result_subtype(ctx, JSON_SUBTYPE);
 }
+static void jsonObjectValue(sqlite3_context *ctx){
+  jsonObjectCompute(ctx, 0);
+}
+static void jsonObjectFinal(sqlite3_context *ctx){
+  jsonObjectCompute(ctx, 1);
+}
+
 
 
 #ifndef SQLITE_OMIT_VIRTUALTABLE
@@ -2118,7 +2202,7 @@ static int jsonEachColumn(
         }
         if( p->eType==JSON_ARRAY ){
           jsonPrintf(30, &x, "[%d]", p->iRowid);
-        }else{
+        }else if( p->eType==JSON_OBJECT ){
           jsonPrintf(pThis->n, &x, ".%.*s", pThis->n-2, pThis->u.zJContent+1);
         }
       }
@@ -2377,9 +2461,12 @@ int sqlite3Json1Init(sqlite3 *db){
      int nArg;
      void (*xStep)(sqlite3_context*,int,sqlite3_value**);
      void (*xFinal)(sqlite3_context*);
+     void (*xValue)(sqlite3_context*);
   } aAgg[] = {
-    { "json_group_array",     1,   jsonArrayStep,   jsonArrayFinal  },
+    { "json_group_array",     1,
-    { "json_group_object",    2,   jsonObjectStep,  jsonObjectFinal },
+      jsonArrayStep,   jsonArrayFinal,  jsonArrayValue  },
+    { "json_group_object",    2,
+      jsonObjectStep,  jsonObjectFinal, jsonObjectValue },
   };
 #ifndef SQLITE_OMIT_VIRTUALTABLE
   static const struct {
@@ -2396,11 +2483,14 @@ int sqlite3Json1Init(sqlite3 *db){
                                  (void*)&aFunc[i].flag,
                                  aFunc[i].xFunc, 0, 0);
   }
+#ifndef SQLITE_OMIT_WINDOWFUNC
   for(i=0; i<sizeof(aAgg)/sizeof(aAgg[0]) && rc==SQLITE_OK; i++){
-    rc = sqlite3_create_function(db, aAgg[i].zName, aAgg[i].nArg,
+    rc = sqlite3_create_window_function(db, aAgg[i].zName, aAgg[i].nArg,
                                  SQLITE_UTF8 | SQLITE_DETERMINISTIC, 0,
-                                 0, aAgg[i].xStep, aAgg[i].xFinal);
+                                 aAgg[i].xStep, aAgg[i].xFinal,
+                                 aAgg[i].xValue, jsonGroupInverse, 0);
   }
+#endif
 #ifndef SQLITE_OMIT_VIRTUALTABLE
   for(i=0; i<sizeof(aMod)/sizeof(aMod[0]) && rc==SQLITE_OK; i++){
     rc = sqlite3_create_module(db, aMod[i].zName, aMod[i].pModule, 0);

ext/misc/memstat.c

@@ -0,0 +1,427 @@
+/*
+** 2018-09-27
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** This file demonstrates an eponymous virtual table that returns information
+** from sqlite3_status64() and sqlite3_db_status().
+**
+** Usage example:
+**
+**     .load ./memstat
+**     .mode quote
+**     .header on
+**     SELECT * FROM memstat;
+*/
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_MEMSTATVTAB)
+#if !defined(SQLITEINT_H)
+#include "sqlite3ext.h"
+#endif
+SQLITE_EXTENSION_INIT1
+#include <assert.h>
+#include <string.h>
+
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+
+/* memstat_vtab is a subclass of sqlite3_vtab which will
+** serve as the underlying representation of a memstat virtual table
+*/
+typedef struct memstat_vtab memstat_vtab;
+struct memstat_vtab {
+  sqlite3_vtab base;  /* Base class - must be first */
+  sqlite3 *db;        /* Database connection for this memstat vtab */
+};
+
+/* memstat_cursor is a subclass of sqlite3_vtab_cursor which will
+** serve as the underlying representation of a cursor that scans
+** over rows of the result
+*/
+typedef struct memstat_cursor memstat_cursor;
+struct memstat_cursor {
+  sqlite3_vtab_cursor base;  /* Base class - must be first */
+  sqlite3 *db;               /* Database connection for this cursor */
+  int iRowid;                /* Current row in aMemstatColumn[] */
+  int iDb;                   /* Which schema we are looking at */
+  int nDb;                   /* Number of schemas */
+  char **azDb;               /* Names of all schemas */
+  sqlite3_int64 aVal[2];     /* Result values */
+};
+
+/*
+** The memstatConnect() method is invoked to create a new
+** memstat_vtab that describes the memstat virtual table.
+**
+** Think of this routine as the constructor for memstat_vtab objects.
+**
+** All this routine needs to do is:
+**
+**    (1) Allocate the memstat_vtab object and initialize all fields.
+**
+**    (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the
+**        result set of queries against memstat will look like.
+*/
+static int memstatConnect(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
+){
+  memstat_vtab *pNew;
+  int rc;
+
+/* Column numbers */
+#define MSV_COLUMN_NAME    0   /* Name of quantity being measured */
+#define MSV_COLUMN_SCHEMA  1   /* schema name */
+#define MSV_COLUMN_VALUE   2   /* Current value */
+#define MSV_COLUMN_HIWTR   3   /* Highwater mark */
+
+  rc = sqlite3_declare_vtab(db,"CREATE TABLE x(name,schema,value,hiwtr)");
+  if( rc==SQLITE_OK ){
+    pNew = sqlite3_malloc( sizeof(*pNew) );
+    *ppVtab = (sqlite3_vtab*)pNew;
+    if( pNew==0 ) return SQLITE_NOMEM;
+    memset(pNew, 0, sizeof(*pNew));
+    pNew->db = db;
+  }
+  return rc;
+}
+
+/*
+** This method is the destructor for memstat_cursor objects.
+*/
+static int memstatDisconnect(sqlite3_vtab *pVtab){
+  sqlite3_free(pVtab);
+  return SQLITE_OK;
+}
+
+/*
+** Constructor for a new memstat_cursor object.
+*/
+static int memstatOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
+  memstat_cursor *pCur;
+  pCur = sqlite3_malloc( sizeof(*pCur) );
+  if( pCur==0 ) return SQLITE_NOMEM;
+  memset(pCur, 0, sizeof(*pCur));
+  pCur->db = ((memstat_vtab*)p)->db;
+  *ppCursor = &pCur->base;
+  return SQLITE_OK;
+}
+
+/*
+** Clear all the schema names from a cursor
+*/
+static void memstatClearSchema(memstat_cursor *pCur){
+  int i;
+  if( pCur->azDb==0 ) return;
+  for(i=0; i<pCur->nDb; i++){
+    sqlite3_free(pCur->azDb[i]);
+  }
+  sqlite3_free(pCur->azDb);
+  pCur->azDb = 0;
+  pCur->nDb = 0;
+}
+
+/*
+** Fill in the azDb[] array for the cursor.
+*/
+static int memstatFindSchemas(memstat_cursor *pCur){
+  sqlite3_stmt *pStmt = 0;
+  int rc;
+  if( pCur->nDb ) return SQLITE_OK;
+  rc = sqlite3_prepare_v2(pCur->db, "PRAGMA database_list", -1, &pStmt, 0);
+  if( rc ){
+    sqlite3_finalize(pStmt);
+    return rc;
+  }
+  while( sqlite3_step(pStmt)==SQLITE_ROW ){
+    char **az, *z;
+    az = sqlite3_realloc(pCur->azDb, sizeof(char*)*(pCur->nDb+1));
+    if( az==0 ){
+      memstatClearSchema(pCur);
+      return SQLITE_NOMEM;
+    }
+    pCur->azDb = az;
+    z = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1));
+    if( z==0 ){
+      memstatClearSchema(pCur);
+      return SQLITE_NOMEM;
+    }
+    pCur->azDb[pCur->nDb] = z;
+    pCur->nDb++;
+  }
+  sqlite3_finalize(pStmt);
+  return SQLITE_OK;
+}
+
+
+/*
+** Destructor for a memstat_cursor.
+*/
+static int memstatClose(sqlite3_vtab_cursor *cur){
+  memstat_cursor *pCur = (memstat_cursor*)cur;
+  memstatClearSchema(pCur);
+  sqlite3_free(cur);
+  return SQLITE_OK;
+}
+
+
+/*
+** Allowed values for aMemstatColumn[].eType
+*/
+#define MSV_GSTAT   0          /* sqlite3_status64() information */
+#define MSV_DB      1          /* sqlite3_db_status() information */
+#define MSV_ZIPVFS  2          /* ZIPVFS file-control with 64-bit return */
+
+/*
+** An array of quantities that can be measured and reported by
+** this virtual table
+*/
+static const struct MemstatColumns {
+  const char *zName;    /* Symbolic name */
+  unsigned char eType;  /* Type of interface */
+  unsigned char mNull;  /* Bitmask of which columns are NULL */
+                        /* 2: dbname,  4: current,  8: hiwtr */
+  int eOp;              /* Opcode */
+} aMemstatColumn[] = {
+ {"MEMORY_USED",            MSV_GSTAT,  2, SQLITE_STATUS_MEMORY_USED          },
+ {"MALLOC_SIZE",            MSV_GSTAT,  6, SQLITE_STATUS_MALLOC_SIZE          },
+ {"MALLOC_COUNT",           MSV_GSTAT,  2, SQLITE_STATUS_MALLOC_COUNT         },
+ {"PAGECACHE_USED",         MSV_GSTAT,  2, SQLITE_STATUS_PAGECACHE_USED       },
+ {"PAGECACHE_OVERFLOW",     MSV_GSTAT,  2, SQLITE_STATUS_PAGECACHE_OVERFLOW   },
+ {"PAGECACHE_SIZE",         MSV_GSTAT,  6, SQLITE_STATUS_PAGECACHE_SIZE       },
+ {"PARSER_STACK",           MSV_GSTAT,  6, SQLITE_STATUS_PARSER_STACK         },
+ {"DB_LOOKASIDE_USED",      MSV_DB,     2, SQLITE_DBSTATUS_LOOKASIDE_USED     },
+ {"DB_LOOKASIDE_HIT",       MSV_DB,     6, SQLITE_DBSTATUS_LOOKASIDE_HIT      },
+ {"DB_LOOKASIDE_MISS_SIZE", MSV_DB,     6, SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE},
+ {"DB_LOOKASIDE_MISS_FULL", MSV_DB,     6, SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL},
+ {"DB_CACHE_USED",          MSV_DB,    10, SQLITE_DBSTATUS_CACHE_USED         },
+#if SQLITE_VERSION_NUMBER >= 3140000
+ {"DB_CACHE_USED_SHARED",   MSV_DB,    10, SQLITE_DBSTATUS_CACHE_USED_SHARED  },
+#endif
+ {"DB_SCHEMA_USED",         MSV_DB,    10, SQLITE_DBSTATUS_SCHEMA_USED        },
+ {"DB_STMT_USED",           MSV_DB,    10, SQLITE_DBSTATUS_STMT_USED          },
+ {"DB_CACHE_HIT",           MSV_DB,    10, SQLITE_DBSTATUS_CACHE_HIT          },
+ {"DB_CACHE_MISS",          MSV_DB,    10, SQLITE_DBSTATUS_CACHE_MISS         },
+ {"DB_CACHE_WRITE",         MSV_DB,    10, SQLITE_DBSTATUS_CACHE_WRITE        },
+#if SQLITE_VERSION_NUMBER >= 3230000
+ {"DB_CACHE_SPILL",         MSV_DB,    10, SQLITE_DBSTATUS_CACHE_SPILL        },
+#endif
+ {"DB_DEFERRED_FKS",        MSV_DB,    10, SQLITE_DBSTATUS_DEFERRED_FKS       },
+#ifdef SQLITE_ENABLE_ZIPVFS
+ {"ZIPVFS_CACHE_USED",      MSV_ZIPVFS, 8, 231454 },
+ {"ZIPVFS_CACHE_HIT",       MSV_ZIPVFS, 8, 231455 },
+ {"ZIPVFS_CACHE_MISS",      MSV_ZIPVFS, 8, 231456 },
+ {"ZIPVFS_CACHE_WRITE",     MSV_ZIPVFS, 8, 231457 },
+ {"ZIPVFS_DIRECT_READ",     MSV_ZIPVFS, 8, 231458 },
+ {"ZIPVFS_DIRECT_BYTES",    MSV_ZIPVFS, 8, 231459 },
+#endif /* SQLITE_ENABLE_ZIPVFS */
+};
+#define MSV_NROW (sizeof(aMemstatColumn)/sizeof(aMemstatColumn[0]))
+
+/*
+** Advance a memstat_cursor to its next row of output.
+*/
+static int memstatNext(sqlite3_vtab_cursor *cur){
+  memstat_cursor *pCur = (memstat_cursor*)cur;
+  int i;
+  assert( pCur->iRowid<=MSV_NROW );
+  while(1){
+    i = (int)pCur->iRowid - 1;
+    if( (aMemstatColumn[i].mNull & 2)!=0 || (++pCur->iDb)>=pCur->nDb ){
+      pCur->iRowid++;
+      if( pCur->iRowid>MSV_NROW ) return SQLITE_OK;  /* End of the table */
+      pCur->iDb = 0;
+      i++;
+    }
+    pCur->aVal[0] = 0;
+    pCur->aVal[1] = 0;    
+    switch( aMemstatColumn[i].eType ){
+      case MSV_GSTAT: {
+        if( sqlite3_libversion_number()>=3010000 ){
+          sqlite3_status64(aMemstatColumn[i].eOp,
+                           &pCur->aVal[0], &pCur->aVal[1],0);
+        }else{
+          int xCur, xHiwtr;
+          sqlite3_status(aMemstatColumn[i].eOp, &xCur, &xHiwtr, 0);
+          pCur->aVal[0] = xCur;
+          pCur->aVal[1] = xHiwtr;
+        }
+        break;
+      }
+      case MSV_DB: {
+        int xCur, xHiwtr;
+        sqlite3_db_status(pCur->db, aMemstatColumn[i].eOp, &xCur, &xHiwtr, 0);
+        pCur->aVal[0] = xCur;
+        pCur->aVal[1] = xHiwtr;
+        break;
+      }
+      case MSV_ZIPVFS: {
+        int rc;
+        rc = sqlite3_file_control(pCur->db, pCur->azDb[pCur->iDb],
+                                  aMemstatColumn[i].eOp, (void*)&pCur->aVal[0]);
+        if( rc!=SQLITE_OK ) continue;
+        break;
+      }
+    }
+    break;
+  }
+  return SQLITE_OK;
+}
+  
+
+/*
+** Return values of columns for the row at which the memstat_cursor
+** is currently pointing.
+*/
+static int memstatColumn(
+  sqlite3_vtab_cursor *cur,   /* The cursor */
+  sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
+  int iCol                    /* Which column to return */
+){
+  memstat_cursor *pCur = (memstat_cursor*)cur;
+  int i;
+  assert( pCur->iRowid>0 && pCur->iRowid<=MSV_NROW );
+  i = (int)pCur->iRowid - 1;
+  if( (aMemstatColumn[i].mNull & (1<<iCol))!=0 ){
+    return SQLITE_OK;
+  }
+  switch( iCol ){
+    case MSV_COLUMN_NAME: {
+      sqlite3_result_text(ctx, aMemstatColumn[i].zName, -1, SQLITE_STATIC);
+      break;
+    }
+    case MSV_COLUMN_SCHEMA: {
+      sqlite3_result_text(ctx, pCur->azDb[pCur->iDb], -1, 0);
+      break;
+    }
+    case MSV_COLUMN_VALUE: {
+      sqlite3_result_int64(ctx, pCur->aVal[0]);
+      break;
+    }
+    case MSV_COLUMN_HIWTR: {
+      sqlite3_result_int64(ctx, pCur->aVal[1]);
+      break;
+    }
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Return the rowid for the current row.  In this implementation, the
+** rowid is the same as the output value.
+*/
+static int memstatRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
+  memstat_cursor *pCur = (memstat_cursor*)cur;
+  *pRowid = pCur->iRowid*1000 + pCur->iDb;
+  return SQLITE_OK;
+}
+
+/*
+** Return TRUE if the cursor has been moved off of the last
+** row of output.
+*/
+static int memstatEof(sqlite3_vtab_cursor *cur){
+  memstat_cursor *pCur = (memstat_cursor*)cur;
+  return pCur->iRowid>MSV_NROW;
+}
+
+/*
+** This method is called to "rewind" the memstat_cursor object back
+** to the first row of output.  This method is always called at least
+** once prior to any call to memstatColumn() or memstatRowid() or 
+** memstatEof().
+*/
+static int memstatFilter(
+  sqlite3_vtab_cursor *pVtabCursor, 
+  int idxNum, const char *idxStr,
+  int argc, sqlite3_value **argv
+){
+  memstat_cursor *pCur = (memstat_cursor *)pVtabCursor;
+  int rc = memstatFindSchemas(pCur);
+  if( rc ) return rc;
+  pCur->iRowid = 0;
+  pCur->iDb = 0;
+  return memstatNext(pVtabCursor);
+}
+
+/*
+** SQLite will invoke this method one or more times while planning a query
+** that uses the memstat virtual table.  This routine needs to create
+** a query plan for each invocation and compute an estimated cost for that
+** plan.
+*/
+static int memstatBestIndex(
+  sqlite3_vtab *tab,
+  sqlite3_index_info *pIdxInfo
+){
+  pIdxInfo->estimatedCost = (double)500;
+  pIdxInfo->estimatedRows = 500;
+  return SQLITE_OK;
+}
+
+/*
+** This following structure defines all the methods for the 
+** memstat virtual table.
+*/
+static sqlite3_module memstatModule = {
+  0,                         /* iVersion */
+  0,                         /* xCreate */
+  memstatConnect,            /* xConnect */
+  memstatBestIndex,          /* xBestIndex */
+  memstatDisconnect,         /* xDisconnect */
+  0,                         /* xDestroy */
+  memstatOpen,               /* xOpen - open a cursor */
+  memstatClose,              /* xClose - close a cursor */
+  memstatFilter,             /* xFilter - configure scan constraints */
+  memstatNext,               /* xNext - advance a cursor */
+  memstatEof,                /* xEof - check for end of scan */
+  memstatColumn,             /* xColumn - read data */
+  memstatRowid,              /* xRowid - read data */
+  0,                         /* xUpdate */
+  0,                         /* xBegin */
+  0,                         /* xSync */
+  0,                         /* xCommit */
+  0,                         /* xRollback */
+  0,                         /* xFindMethod */
+  0,                         /* xRename */
+  0,                         /* xSavepoint */
+  0,                         /* xRelease */
+  0,                         /* xRollbackTo */
+};
+
+#endif /* SQLITE_OMIT_VIRTUALTABLE */
+
+int sqlite3MemstatVtabInit(sqlite3 *db){
+  int rc = SQLITE_OK;
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+  rc = sqlite3_create_module(db, "sqlite_memstat", &memstatModule, 0);
+#endif
+  return rc;
+}
+
+#ifndef SQLITE_CORE
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_memstat_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  int rc = SQLITE_OK;
+  SQLITE_EXTENSION_INIT2(pApi);
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+  rc = sqlite3MemstatVtabInit(db);
+#endif
+  return rc;
+}
+#endif /* SQLITE_CORE */
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_MEMSTATVTAB) */

ext/misc/memvfs.c

@@ -10,23 +10,33 @@
 **
 ******************************************************************************
 **
-** This is an in-memory read-only VFS implementation.  The application
+** This is an in-memory VFS implementation.  The application supplies
-** supplies a block of memory which is the database file, and this VFS
+** a chunk of memory to hold the database file.
-** uses that block of memory.
 **
-** Because there is no place to store journals and no good way to lock
+** Because there is place to store a rollback or wal journal, the database
-** the "file", this VFS is read-only.
+** must use one of journal_mode=MEMORY or journal_mode=NONE.
 **
 ** USAGE:
 **
-**    sqlite3_open_v2("file:/whatever?ptr=0xf05538&sz=14336", &db,
+**    sqlite3_open_v2("file:/whatever?ptr=0xf05538&sz=14336&max=65536", &db,
-**                    SQLITE_OPEN_READONLY | SQLITE_OPEN_URI,
+**                    SQLITE_OPEN_READWRITE | SQLITE_OPEN_URI,
 **                    "memvfs");
 **
-** The ptr= and sz= query parameters are required or the open will fail.
+** These are the query parameters:
-** The ptr= parameter gives the memory address of the buffer holding the
+**
-** read-only database and sz= gives the size of the database.  The parameter
+**    ptr=          The address of the memory buffer that holds the database.
-** values may be in hexadecimal or decimal.  The filename is ignored.
+**
+**    sz=           The current size the database file
+**
+**    maxsz=        The maximum size of the database.  In other words, the
+**                  amount of space allocated for the ptr= buffer.
+**
+**    freeonclose=  If true, then sqlite3_free() is called on the ptr=
+**                  value when the connection closes.
+**
+** The ptr= and sz= query parameters are required.  If maxsz= is omitted,
+** then it defaults to the sz= value.  Parameter values can be in either
+** decimal or hexadecimal.  The filename in the URI is ignored.
 */
 #include <sqlite3ext.h>
 SQLITE_EXTENSION_INIT1
@@ -49,7 +59,9 @@ typedef struct MemFile MemFile;
 struct MemFile {
   sqlite3_file base;              /* IO methods */
   sqlite3_int64 sz;               /* Size of the file */
+  sqlite3_int64 szMax;            /* Space allocated to aData */
   unsigned char *aData;           /* content of the file */
+  int bFreeOnClose;               /* Invoke sqlite3_free() on aData at close */
 };
 
 /*
@@ -144,6 +156,8 @@ static const sqlite3_io_methods mem_io_methods = {
 ** to free.
 */
 static int memClose(sqlite3_file *pFile){
+  MemFile *p = (MemFile *)pFile;
+  if( p->bFreeOnClose ) sqlite3_free(p->aData);
   return SQLITE_OK;
 }
 
@@ -170,21 +184,34 @@ static int memWrite(
   int iAmt,
   sqlite_int64 iOfst
 ){
-  return SQLITE_READONLY;
+  MemFile *p = (MemFile *)pFile;
+  if( iOfst+iAmt>p->sz ){
+    if( iOfst+iAmt>p->szMax ) return SQLITE_FULL;
+    if( iOfst>p->sz ) memset(p->aData+p->sz, 0, iOfst-p->sz);
+    p->sz = iOfst+iAmt;
+  }
+  memcpy(p->aData+iOfst, z, iAmt);
+  return SQLITE_OK;
 }
 
 /*
 ** Truncate an mem-file.
 */
 static int memTruncate(sqlite3_file *pFile, sqlite_int64 size){
-  return SQLITE_READONLY;
+  MemFile *p = (MemFile *)pFile;
+  if( size>p->sz ){
+    if( size>p->szMax ) return SQLITE_FULL;
+    memset(p->aData+p->sz, 0, size-p->sz);
+  }
+  p->sz = size; 
+  return SQLITE_OK;
 }
 
 /*
 ** Sync an mem-file.
 */
 static int memSync(sqlite3_file *pFile, int flags){
-  return SQLITE_READONLY;
+  return SQLITE_OK;
 }
 
 /*
@@ -200,7 +227,7 @@ static int memFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){
 ** Lock an mem-file.
 */
 static int memLock(sqlite3_file *pFile, int eLock){
-  return SQLITE_READONLY;
+  return SQLITE_OK;
 }
 
 /*
@@ -242,7 +269,10 @@ static int memSectorSize(sqlite3_file *pFile){
 ** Return the device characteristic flags supported by an mem-file.
 */
 static int memDeviceCharacteristics(sqlite3_file *pFile){
-  return SQLITE_IOCAP_IMMUTABLE;
+  return SQLITE_IOCAP_ATOMIC | 
+         SQLITE_IOCAP_POWERSAFE_OVERWRITE |
+         SQLITE_IOCAP_SAFE_APPEND |
+         SQLITE_IOCAP_SEQUENTIAL;
 }
 
 /* Create a shared memory file mapping */
@@ -253,12 +283,12 @@ static int memShmMap(
   int bExtend,
   void volatile **pp
 ){
-  return SQLITE_READONLY;
+  return SQLITE_IOERR_SHMMAP;
 }
 
 /* Perform locking on a shared-memory segment */
 static int memShmLock(sqlite3_file *pFile, int offset, int n, int flags){
-  return SQLITE_READONLY;
+  return SQLITE_IOERR_SHMLOCK;
 }
 
 /* Memory barrier operation on shared memory */
@@ -305,6 +335,9 @@ static int memOpen(
   if( p->aData==0 ) return SQLITE_CANTOPEN;
   p->sz = sqlite3_uri_int64(zName,"sz",0);
   if( p->sz<0 ) return SQLITE_CANTOPEN;
+  p->szMax = sqlite3_uri_int64(zName,"max",p->sz);
+  if( p->szMax<p->sz ) return SQLITE_CANTOPEN;
+  p->bFreeOnClose = sqlite3_uri_boolean(zName,"freeonclose",0);
   pFile->pMethods = &mem_io_methods;
   return SQLITE_OK;
 }
@@ -315,7 +348,7 @@ static int memOpen(
 ** returning.
 */
 static int memDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){
-  return SQLITE_READONLY;
+  return SQLITE_IOERR_DELETE;
 }
 
 /*
@@ -328,14 +361,7 @@ static int memAccess(
   int flags, 
   int *pResOut
 ){
-  /* The spec says there are three possible values for flags.  But only
+  *pResOut = 0;
-  ** two of them are actually used */
-  assert( flags==SQLITE_ACCESS_EXISTS || flags==SQLITE_ACCESS_READWRITE );
-  if( flags==SQLITE_ACCESS_READWRITE ){
-    *pResOut = 0;
-  }else{
-    *pResOut = 1;
-  }
   return SQLITE_OK;
 }
 
@@ -416,31 +442,43 @@ static int memCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){
 
 #ifdef MEMVFS_TEST
 /*
-**       memload(FILENAME)
+**       memvfs_from_file(FILENAME, MAXSIZE)
 **
 ** This an SQL function used to help in testing the memvfs VFS.  The
 ** function reads the content of a file into memory and then returns
-** a string that gives the locate and size of the in-memory buffer.
+** a URI that can be handed to ATTACH to attach the memory buffer as
+** a database.  Example:
+**
+**       ATTACH memvfs_from_file('test.db',1048576) AS inmem;
+**
+** The optional MAXSIZE argument gives the size of the memory allocation
+** used to hold the database.  If omitted, it defaults to the size of the
+** file on disk.
 */
 #include <stdio.h>
-static void memvfsMemloadFunc(
+static void memvfsFromFileFunc(
   sqlite3_context *context,
   int argc,
   sqlite3_value **argv
 ){
   unsigned char *p;
   sqlite3_int64 sz;
+  sqlite3_int64 szMax;
   FILE *in;
   const char *zFilename = (const char*)sqlite3_value_text(argv[0]);
-  char zReturn[100];
+  char *zUri;
 
   if( zFilename==0 ) return;
   in = fopen(zFilename, "rb");
   if( in==0 ) return;
   fseek(in, 0, SEEK_END);
-  sz = ftell(in);
+  szMax = sz = ftell(in);
   rewind(in);
-  p = sqlite3_malloc( sz );
+  if( argc>=2 ){
+    szMax = sqlite3_value_int64(argv[1]);
+    if( szMax<sz ) szMax = sz;
+  }
+  p = sqlite3_malloc64( szMax );
   if( p==0 ){
     fclose(in);
     sqlite3_result_error_nomem(context);
@@ -448,18 +486,60 @@ static void memvfsMemloadFunc(
   }
   fread(p, sz, 1, in);
   fclose(in);
-  sqlite3_snprintf(sizeof(zReturn),zReturn,"ptr=%lld&sz=%lld",
+  zUri = sqlite3_mprintf(
-                   (sqlite3_int64)p, sz);
+           "file:/mem?vfs=memvfs&ptr=%lld&sz=%lld&max=%lld&freeonclose=1",
-  sqlite3_result_text(context, zReturn, -1, SQLITE_TRANSIENT);
+                         (sqlite3_int64)p, sz, szMax);
+  sqlite3_result_text(context, zUri, -1, sqlite3_free);
 }
+#endif /* MEMVFS_TEST */
+
+#ifdef MEMVFS_TEST
+/*
+**       memvfs_to_file(SCHEMA, FILENAME)
+**
+** The schema identified by SCHEMA must be a memvfs database.  Write
+** the content of this database into FILENAME.
+*/
+static void memvfsToFileFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  MemFile *p = 0;
+  FILE *out;
+  int rc;
+  sqlite3 *db = sqlite3_context_db_handle(context);
+  sqlite3_vfs *pVfs = 0;
+  const char *zSchema = (const char*)sqlite3_value_text(argv[0]);
+  const char *zFilename = (const char*)sqlite3_value_text(argv[1]);
+
+  if( zFilename==0 ) return;
+  out = fopen(zFilename, "wb");
+  if( out==0 ) return;
+  rc = sqlite3_file_control(db, zSchema, SQLITE_FCNTL_VFS_POINTER, &pVfs);
+  if( rc || pVfs==0 ) return;
+  if( strcmp(pVfs->zName,"memvfs")!=0 ) return;
+  rc = sqlite3_file_control(db, zSchema, SQLITE_FCNTL_FILE_POINTER, &p);
+  if( rc ) return;
+  fwrite(p->aData, 1, (size_t)p->sz, out);
+  fclose(out);
+}
+#endif /* MEMVFS_TEST */
+
+#ifdef MEMVFS_TEST
 /* Called for each new database connection */
 static int memvfsRegister(
   sqlite3 *db,
-  const char **pzErrMsg,
+  char **pzErrMsg,
   const struct sqlite3_api_routines *pThunk
 ){
-  return sqlite3_create_function(db, "memload", 1, SQLITE_UTF8, 0,
+  sqlite3_create_function(db, "memvfs_from_file", 1, SQLITE_UTF8, 0,
-                                 memvfsMemloadFunc, 0, 0);
+                          memvfsFromFileFunc, 0, 0);
+  sqlite3_create_function(db, "memvfs_from_file", 2, SQLITE_UTF8, 0,
+                          memvfsFromFileFunc, 0, 0);
+  sqlite3_create_function(db, "memvfs_to_file", 2, SQLITE_UTF8, 0,
+                          memvfsToFileFunc, 0, 0);
+  return SQLITE_OK;
 }
 #endif /* MEMVFS_TEST */
 
@@ -485,6 +565,9 @@ int sqlite3_memvfs_init(
   if( rc==SQLITE_OK ){
     rc = sqlite3_auto_extension((void(*)(void))memvfsRegister);
   }
+  if( rc==SQLITE_OK ){
+    rc = memvfsRegister(db, pzErrMsg, pApi);
+  }
 #endif
   if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY;
   return rc;

ext/misc/mmapwarm.c

@@ -0,0 +1,108 @@
+/*
+** 2017-09-18
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+*/
+
+#include "sqlite3.h"
+
+
+/*
+** This function is used to touch each page of a mapping of a memory
+** mapped SQLite database. Assuming that the system has sufficient free
+** memory and supports sufficiently large mappings, this causes the OS 
+** to cache the entire database in main memory, making subsequent 
+** database accesses faster.
+**
+** If the second parameter to this function is not NULL, it is the name of
+** the specific database to operate on (i.e. "main" or the name of an
+** attached database).
+**
+** SQLITE_OK is returned if successful, or an SQLite error code otherwise.
+** It is not considered an error if the file is not memory-mapped, or if
+** the mapping does not span the entire file. If an error does occur, a
+** transaction may be left open on the database file.
+**
+** It is illegal to call this function when the database handle has an 
+** open transaction. SQLITE_MISUSE is returned in this case.
+*/
+int sqlite3_mmap_warm(sqlite3 *db, const char *zDb){
+  int rc = SQLITE_OK;
+  char *zSql = 0;
+  int pgsz = 0;
+  int nTotal = 0;
+
+  if( 0==sqlite3_get_autocommit(db) ) return SQLITE_MISUSE;
+
+  /* Open a read-only transaction on the file in question */
+  zSql = sqlite3_mprintf("BEGIN; SELECT * FROM %s%q%ssqlite_master", 
+      (zDb ? "'" : ""), (zDb ? zDb : ""), (zDb ? "'." : "")
+  );
+  if( zSql==0 ) return SQLITE_NOMEM;
+  rc = sqlite3_exec(db, zSql, 0, 0, 0);
+  sqlite3_free(zSql);
+
+  /* Find the SQLite page size of the file */
+  if( rc==SQLITE_OK ){
+    zSql = sqlite3_mprintf("PRAGMA %s%q%spage_size", 
+        (zDb ? "'" : ""), (zDb ? zDb : ""), (zDb ? "'." : "")
+    );
+    if( zSql==0 ){
+      rc = SQLITE_NOMEM;
+    }else{
+      sqlite3_stmt *pPgsz = 0;
+      rc = sqlite3_prepare_v2(db, zSql, -1, &pPgsz, 0);
+      sqlite3_free(zSql);
+      if( rc==SQLITE_OK ){
+        if( sqlite3_step(pPgsz)==SQLITE_ROW ){
+          pgsz = sqlite3_column_int(pPgsz, 0);
+        }
+        rc = sqlite3_finalize(pPgsz);
+      }
+      if( rc==SQLITE_OK && pgsz==0 ){
+        rc = SQLITE_ERROR;
+      }
+    }
+  }
+
+  /* Touch each mmap'd page of the file */
+  if( rc==SQLITE_OK ){
+    int rc2;
+    sqlite3_file *pFd = 0;
+    rc = sqlite3_file_control(db, zDb, SQLITE_FCNTL_FILE_POINTER, &pFd);
+    if( rc==SQLITE_OK && pFd->pMethods->iVersion>=3 ){
+      sqlite3_int64 iPg = 1;
+      sqlite3_io_methods const *p = pFd->pMethods;
+      while( 1 ){
+        unsigned char *pMap;
+        rc = p->xFetch(pFd, pgsz*iPg, pgsz, (void**)&pMap);
+        if( rc!=SQLITE_OK || pMap==0 ) break;
+
+        nTotal += pMap[0];
+        nTotal += pMap[pgsz-1];
+
+        rc = p->xUnfetch(pFd, pgsz*iPg, (void*)pMap);
+        if( rc!=SQLITE_OK ) break;
+        iPg++;
+      }
+      sqlite3_log(SQLITE_OK, 
+          "sqlite3_mmap_warm_cache: Warmed up %d pages of %s", iPg==1?0:iPg,
+          sqlite3_db_filename(db, zDb)
+      );
+    }
+
+    rc2 = sqlite3_exec(db, "END", 0, 0, 0);
+    if( rc==SQLITE_OK ) rc = rc2;
+  }
+
+  return rc;
+}
+

ext/misc/normalize.c

@@ -0,0 +1,707 @@
+/*
+** 2018-01-08
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This file contains code to implement the sqlite3_normalize() function.
+**
+**    char *sqlite3_normalize(const char *zSql);
+**
+** This function takes an SQL string as input and returns a "normalized"
+** version of that string in memory obtained from sqlite3_malloc64().  The
+** caller is responsible for ensuring that the returned memory is freed.
+**
+** If a memory allocation error occurs, this routine returns NULL.
+**
+** The normalization consists of the following transformations:
+**
+**   (1)  Convert every literal (string, blob literal, numeric constant,
+**        or "NULL" constant) into a ?
+**
+**   (2)  Remove all superfluous whitespace, including comments.  Change
+**        all required whitespace to a single space character.
+**
+**   (3)  Lowercase all ASCII characters.
+**
+**   (4)  If an IN or NOT IN operator is followed by a list of 1 or more
+**        values, convert that list into "(?,?,?)".
+**
+** The purpose of normalization is two-fold:
+**
+**   (1)  Sanitize queries by removing potentially private or sensitive
+**        information contained in literals.
+**
+**   (2)  Identify structurally identical queries by comparing their
+**        normalized forms.
+**
+** Command-Line Utility
+** --------------------
+**
+** This file also contains code for a command-line utility that converts
+** SQL queries in text files into their normalized forms.  To build the
+** command-line program, compile this file with -DSQLITE_NORMALIZE_CLI
+** and link it against the SQLite library.
+*/
+#include <sqlite3.h>
+#include <string.h>
+
+/*
+** Implementation note:
+**
+** Much of the tokenizer logic is copied out of the tokenize.c source file
+** of SQLite.  That logic could be simplified for this particular application,
+** but that would impose a risk of introducing subtle errors.  It is best to
+** keep the code as close to the original as possible.
+**
+** The tokenize code is in sync with the SQLite core as of 2018-01-08.
+** Any future changes to the core tokenizer might require corresponding
+** adjustments to the tokenizer logic in this module.
+*/
+
+
+/* Character classes for tokenizing
+**
+** In the sqlite3GetToken() function, a switch() on aiClass[c] is implemented
+** using a lookup table, whereas a switch() directly on c uses a binary search.
+** The lookup table is much faster.  To maximize speed, and to ensure that
+** a lookup table is used, all of the classes need to be small integers and
+** all of them need to be used within the switch.
+*/
+#define CC_X          0    /* The letter 'x', or start of BLOB literal */
+#define CC_KYWD       1    /* Alphabetics or '_'.  Usable in a keyword */
+#define CC_ID         2    /* unicode characters usable in IDs */
+#define CC_DIGIT      3    /* Digits */
+#define CC_DOLLAR     4    /* '$' */
+#define CC_VARALPHA   5    /* '@', '#', ':'.  Alphabetic SQL variables */
+#define CC_VARNUM     6    /* '?'.  Numeric SQL variables */
+#define CC_SPACE      7    /* Space characters */
+#define CC_QUOTE      8    /* '"', '\'', or '`'.  String literals, quoted ids */
+#define CC_QUOTE2     9    /* '['.   [...] style quoted ids */
+#define CC_PIPE      10    /* '|'.   Bitwise OR or concatenate */
+#define CC_MINUS     11    /* '-'.  Minus or SQL-style comment */
+#define CC_LT        12    /* '<'.  Part of < or <= or <> */
+#define CC_GT        13    /* '>'.  Part of > or >= */
+#define CC_EQ        14    /* '='.  Part of = or == */
+#define CC_BANG      15    /* '!'.  Part of != */
+#define CC_SLASH     16    /* '/'.  / or c-style comment */
+#define CC_LP        17    /* '(' */
+#define CC_RP        18    /* ')' */
+#define CC_SEMI      19    /* ';' */
+#define CC_PLUS      20    /* '+' */
+#define CC_STAR      21    /* '*' */
+#define CC_PERCENT   22    /* '%' */
+#define CC_COMMA     23    /* ',' */
+#define CC_AND       24    /* '&' */
+#define CC_TILDA     25    /* '~' */
+#define CC_DOT       26    /* '.' */
+#define CC_ILLEGAL   27    /* Illegal character */
+
+static const unsigned char aiClass[] = {
+/*         x0  x1  x2  x3  x4  x5  x6  x7  x8  x9  xa  xb  xc  xd  xe  xf */
+/* 0x */   27, 27, 27, 27, 27, 27, 27, 27, 27,  7,  7, 27,  7,  7, 27, 27,
+/* 1x */   27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
+/* 2x */    7, 15,  8,  5,  4, 22, 24,  8, 17, 18, 21, 20, 23, 11, 26, 16,
+/* 3x */    3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  5, 19, 12, 14, 13,  6,
+/* 4x */    5,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,
+/* 5x */    1,  1,  1,  1,  1,  1,  1,  1,  0,  1,  1,  9, 27, 27, 27,  1,
+/* 6x */    8,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,
+/* 7x */    1,  1,  1,  1,  1,  1,  1,  1,  0,  1,  1, 27, 10, 27, 25, 27,
+/* 8x */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
+/* 9x */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
+/* Ax */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
+/* Bx */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
+/* Cx */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
+/* Dx */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
+/* Ex */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
+/* Fx */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2
+};
+
+/* An array to map all upper-case characters into their corresponding
+** lower-case character. 
+**
+** SQLite only considers US-ASCII (or EBCDIC) characters.  We do not
+** handle case conversions for the UTF character set since the tables
+** involved are nearly as big or bigger than SQLite itself.
+*/
+static const unsigned char sqlite3UpperToLower[] = {
+      0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 16, 17,
+     18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35,
+     36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53,
+     54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 97, 98, 99,100,101,102,103,
+    104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,
+    122, 91, 92, 93, 94, 95, 96, 97, 98, 99,100,101,102,103,104,105,106,107,
+    108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,
+    126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,
+    144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,
+    162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,
+    180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,
+    198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,
+    216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,
+    234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,
+    252,253,254,255
+};
+
+/*
+** The following 256 byte lookup table is used to support SQLites built-in
+** equivalents to the following standard library functions:
+**
+**   isspace()                        0x01
+**   isalpha()                        0x02
+**   isdigit()                        0x04
+**   isalnum()                        0x06
+**   isxdigit()                       0x08
+**   toupper()                        0x20
+**   SQLite identifier character      0x40
+**   Quote character                  0x80
+**
+** Bit 0x20 is set if the mapped character requires translation to upper
+** case. i.e. if the character is a lower-case ASCII character.
+** If x is a lower-case ASCII character, then its upper-case equivalent
+** is (x - 0x20). Therefore toupper() can be implemented as:
+**
+**   (x & ~(map[x]&0x20))
+**
+** The equivalent of tolower() is implemented using the sqlite3UpperToLower[]
+** array. tolower() is used more often than toupper() by SQLite.
+**
+** Bit 0x40 is set if the character is non-alphanumeric and can be used in an 
+** SQLite identifier.  Identifiers are alphanumerics, "_", "$", and any
+** non-ASCII UTF character. Hence the test for whether or not a character is
+** part of an identifier is 0x46.
+*/
+static const unsigned char sqlite3CtypeMap[256] = {
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,  /* 00..07    ........ */
+  0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00,  /* 08..0f    ........ */
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,  /* 10..17    ........ */
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,  /* 18..1f    ........ */
+  0x01, 0x00, 0x80, 0x00, 0x40, 0x00, 0x00, 0x80,  /* 20..27     !"#$%&' */
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,  /* 28..2f    ()*+,-./ */
+  0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c,  /* 30..37    01234567 */
+  0x0c, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,  /* 38..3f    89:;<=>? */
+
+  0x00, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x02,  /* 40..47    @ABCDEFG */
+  0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,  /* 48..4f    HIJKLMNO */
+  0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,  /* 50..57    PQRSTUVW */
+  0x02, 0x02, 0x02, 0x80, 0x00, 0x00, 0x00, 0x40,  /* 58..5f    XYZ[\]^_ */
+  0x80, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x22,  /* 60..67    `abcdefg */
+  0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22,  /* 68..6f    hijklmno */
+  0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22,  /* 70..77    pqrstuvw */
+  0x22, 0x22, 0x22, 0x00, 0x00, 0x00, 0x00, 0x00,  /* 78..7f    xyz{|}~. */
+
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* 80..87    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* 88..8f    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* 90..97    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* 98..9f    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* a0..a7    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* a8..af    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* b0..b7    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* b8..bf    ........ */
+
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* c0..c7    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* c8..cf    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* d0..d7    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* d8..df    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* e0..e7    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* e8..ef    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* f0..f7    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40   /* f8..ff    ........ */
+};
+#define sqlite3Toupper(x)   ((x)&~(sqlite3CtypeMap[(unsigned char)(x)]&0x20))
+#define sqlite3Isspace(x)   (sqlite3CtypeMap[(unsigned char)(x)]&0x01)
+#define sqlite3Isalnum(x)   (sqlite3CtypeMap[(unsigned char)(x)]&0x06)
+#define sqlite3Isalpha(x)   (sqlite3CtypeMap[(unsigned char)(x)]&0x02)
+#define sqlite3Isdigit(x)   (sqlite3CtypeMap[(unsigned char)(x)]&0x04)
+#define sqlite3Isxdigit(x)  (sqlite3CtypeMap[(unsigned char)(x)]&0x08)
+#define sqlite3Tolower(x)   (sqlite3UpperToLower[(unsigned char)(x)])
+#define sqlite3Isquote(x)   (sqlite3CtypeMap[(unsigned char)(x)]&0x80)
+
+
+/*
+** If X is a character that can be used in an identifier then
+** IdChar(X) will be true.  Otherwise it is false.
+**
+** For ASCII, any character with the high-order bit set is
+** allowed in an identifier.  For 7-bit characters, 
+** sqlite3IsIdChar[X] must be 1.
+**
+** For EBCDIC, the rules are more complex but have the same
+** end result.
+**
+** Ticket #1066.  the SQL standard does not allow '$' in the
+** middle of identifiers.  But many SQL implementations do. 
+** SQLite will allow '$' in identifiers for compatibility.
+** But the feature is undocumented.
+*/
+#define IdChar(C)  ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0)
+
+/*
+** Ignore testcase() macros
+*/
+#define testcase(X)
+
+/*
+** Token values
+*/
+#define TK_SPACE    0
+#define TK_NAME     1
+#define TK_LITERAL  2
+#define TK_PUNCT    3
+#define TK_ERROR    4
+
+#define TK_MINUS    TK_PUNCT
+#define TK_LP       TK_PUNCT
+#define TK_RP       TK_PUNCT
+#define TK_SEMI     TK_PUNCT
+#define TK_PLUS     TK_PUNCT
+#define TK_STAR     TK_PUNCT
+#define TK_SLASH    TK_PUNCT
+#define TK_REM      TK_PUNCT
+#define TK_EQ       TK_PUNCT
+#define TK_LE       TK_PUNCT
+#define TK_NE       TK_PUNCT
+#define TK_LSHIFT   TK_PUNCT
+#define TK_LT       TK_PUNCT
+#define TK_GE       TK_PUNCT
+#define TK_RSHIFT   TK_PUNCT
+#define TK_GT       TK_PUNCT
+#define TK_GE       TK_PUNCT
+#define TK_BITOR    TK_PUNCT
+#define TK_CONCAT   TK_PUNCT
+#define TK_COMMA    TK_PUNCT
+#define TK_BITAND   TK_PUNCT
+#define TK_BITNOT   TK_PUNCT
+#define TK_STRING   TK_LITERAL
+#define TK_ID       TK_NAME
+#define TK_ILLEGAL  TK_ERROR
+#define TK_DOT      TK_PUNCT
+#define TK_INTEGER  TK_LITERAL
+#define TK_FLOAT    TK_LITERAL
+#define TK_VARIABLE TK_LITERAL
+#define TK_BLOB     TK_LITERAL
+
+/*
+** Return the length (in bytes) of the token that begins at z[0]. 
+** Store the token type in *tokenType before returning.
+*/
+static int sqlite3GetToken(const unsigned char *z, int *tokenType){
+  int i, c;
+  switch( aiClass[*z] ){  /* Switch on the character-class of the first byte
+                          ** of the token. See the comment on the CC_ defines
+                          ** above. */
+    case CC_SPACE: {
+      for(i=1; sqlite3Isspace(z[i]); i++){}
+      *tokenType = TK_SPACE;
+      return i;
+    }
+    case CC_MINUS: {
+      if( z[1]=='-' ){
+        for(i=2; (c=z[i])!=0 && c!='\n'; i++){}
+        *tokenType = TK_SPACE;
+        return i;
+      }
+      *tokenType = TK_MINUS;
+      return 1;
+    }
+    case CC_LP: {
+      *tokenType = TK_LP;
+      return 1;
+    }
+    case CC_RP: {
+      *tokenType = TK_RP;
+      return 1;
+    }
+    case CC_SEMI: {
+      *tokenType = TK_SEMI;
+      return 1;
+    }
+    case CC_PLUS: {
+      *tokenType = TK_PLUS;
+      return 1;
+    }
+    case CC_STAR: {
+      *tokenType = TK_STAR;
+      return 1;
+    }
+    case CC_SLASH: {
+      if( z[1]!='*' || z[2]==0 ){
+        *tokenType = TK_SLASH;
+        return 1;
+      }
+      for(i=3, c=z[2]; (c!='*' || z[i]!='/') && (c=z[i])!=0; i++){}
+      if( c ) i++;
+      *tokenType = TK_SPACE;
+      return i;
+    }
+    case CC_PERCENT: {
+      *tokenType = TK_REM;
+      return 1;
+    }
+    case CC_EQ: {
+      *tokenType = TK_EQ;
+      return 1 + (z[1]=='=');
+    }
+    case CC_LT: {
+      if( (c=z[1])=='=' ){
+        *tokenType = TK_LE;
+        return 2;
+      }else if( c=='>' ){
+        *tokenType = TK_NE;
+        return 2;
+      }else if( c=='<' ){
+        *tokenType = TK_LSHIFT;
+        return 2;
+      }else{
+        *tokenType = TK_LT;
+        return 1;
+      }
+    }
+    case CC_GT: {
+      if( (c=z[1])=='=' ){
+        *tokenType = TK_GE;
+        return 2;
+      }else if( c=='>' ){
+        *tokenType = TK_RSHIFT;
+        return 2;
+      }else{
+        *tokenType = TK_GT;
+        return 1;
+      }
+    }
+    case CC_BANG: {
+      if( z[1]!='=' ){
+        *tokenType = TK_ILLEGAL;
+        return 1;
+      }else{
+        *tokenType = TK_NE;
+        return 2;
+      }
+    }
+    case CC_PIPE: {
+      if( z[1]!='|' ){
+        *tokenType = TK_BITOR;
+        return 1;
+      }else{
+        *tokenType = TK_CONCAT;
+        return 2;
+      }
+    }
+    case CC_COMMA: {
+      *tokenType = TK_COMMA;
+      return 1;
+    }
+    case CC_AND: {
+      *tokenType = TK_BITAND;
+      return 1;
+    }
+    case CC_TILDA: {
+      *tokenType = TK_BITNOT;
+      return 1;
+    }
+    case CC_QUOTE: {
+      int delim = z[0];
+      testcase( delim=='`' );
+      testcase( delim=='\'' );
+      testcase( delim=='"' );
+      for(i=1; (c=z[i])!=0; i++){
+        if( c==delim ){
+          if( z[i+1]==delim ){
+            i++;
+          }else{
+            break;
+          }
+        }
+      }
+      if( c=='\'' ){
+        *tokenType = TK_STRING;
+        return i+1;
+      }else if( c!=0 ){
+        *tokenType = TK_ID;
+        return i+1;
+      }else{
+        *tokenType = TK_ILLEGAL;
+        return i;
+      }
+    }
+    case CC_DOT: {
+      if( !sqlite3Isdigit(z[1]) ){
+        *tokenType = TK_DOT;
+        return 1;
+      }
+      /* If the next character is a digit, this is a floating point
+      ** number that begins with ".".  Fall thru into the next case */
+    }
+    case CC_DIGIT: {
+      *tokenType = TK_INTEGER;
+      if( z[0]=='0' && (z[1]=='x' || z[1]=='X') && sqlite3Isxdigit(z[2]) ){
+        for(i=3; sqlite3Isxdigit(z[i]); i++){}
+        return i;
+      }
+      for(i=0; sqlite3Isdigit(z[i]); i++){}
+      if( z[i]=='.' ){
+        i++;
+        while( sqlite3Isdigit(z[i]) ){ i++; }
+        *tokenType = TK_FLOAT;
+      }
+      if( (z[i]=='e' || z[i]=='E') &&
+           ( sqlite3Isdigit(z[i+1]) 
+            || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2]))
+           )
+      ){
+        i += 2;
+        while( sqlite3Isdigit(z[i]) ){ i++; }
+        *tokenType = TK_FLOAT;
+      }
+      while( IdChar(z[i]) ){
+        *tokenType = TK_ILLEGAL;
+        i++;
+      }
+      return i;
+    }
+    case CC_QUOTE2: {
+      for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){}
+      *tokenType = c==']' ? TK_ID : TK_ILLEGAL;
+      return i;
+    }
+    case CC_VARNUM: {
+      *tokenType = TK_VARIABLE;
+      for(i=1; sqlite3Isdigit(z[i]); i++){}
+      return i;
+    }
+    case CC_DOLLAR:
+    case CC_VARALPHA: {
+      int n = 0;
+      testcase( z[0]=='$' );  testcase( z[0]=='@' );
+      testcase( z[0]==':' );  testcase( z[0]=='#' );
+      *tokenType = TK_VARIABLE;
+      for(i=1; (c=z[i])!=0; i++){
+        if( IdChar(c) ){
+          n++;
+        }else if( c=='(' && n>0 ){
+          do{
+            i++;
+          }while( (c=z[i])!=0 && !sqlite3Isspace(c) && c!=')' );
+          if( c==')' ){
+            i++;
+          }else{
+            *tokenType = TK_ILLEGAL;
+          }
+          break;
+        }else if( c==':' && z[i+1]==':' ){
+          i++;
+        }else{
+          break;
+        }
+      }
+      if( n==0 ) *tokenType = TK_ILLEGAL;
+      return i;
+    }
+    case CC_KYWD: {
+      for(i=1; aiClass[z[i]]<=CC_KYWD; i++){}
+      if( IdChar(z[i]) ){
+        /* This token started out using characters that can appear in keywords,
+        ** but z[i] is a character not allowed within keywords, so this must
+        ** be an identifier instead */
+        i++;
+        break;
+      }
+      *tokenType = TK_ID;
+      return i;
+    }
+    case CC_X: {
+      testcase( z[0]=='x' ); testcase( z[0]=='X' );
+      if( z[1]=='\'' ){
+        *tokenType = TK_BLOB;
+        for(i=2; sqlite3Isxdigit(z[i]); i++){}
+        if( z[i]!='\'' || i%2 ){
+          *tokenType = TK_ILLEGAL;
+          while( z[i] && z[i]!='\'' ){ i++; }
+        }
+        if( z[i] ) i++;
+        return i;
+      }
+      /* If it is not a BLOB literal, then it must be an ID, since no
+      ** SQL keywords start with the letter 'x'.  Fall through */
+    }
+    case CC_ID: {
+      i = 1;
+      break;
+    }
+    default: {
+      *tokenType = TK_ILLEGAL;
+      return 1;
+    }
+  }
+  while( IdChar(z[i]) ){ i++; }
+  *tokenType = TK_ID;
+  return i;
+}
+
+char *sqlite3_normalize(const char *zSql){
+  char *z;              /* The output string */
+  sqlite3_int64 nZ;     /* Size of the output string in bytes */
+  sqlite3_int64 nSql;   /* Size of the input string in bytes */
+  int i;                /* Next character to read from zSql[] */
+  int j;                /* Next slot to fill in on z[] */
+  int tokenType;        /* Type of the next token */
+  int n;                /* Size of the next token */
+  int k;                /* Loop counter */
+
+  nSql = strlen(zSql);
+  nZ = nSql;
+  z = sqlite3_malloc64( nZ+2 );
+  if( z==0 ) return 0;
+  for(i=j=0; zSql[i]; i += n){
+    n = sqlite3GetToken((unsigned char*)zSql+i, &tokenType);
+    switch( tokenType ){
+      case TK_SPACE: {
+        break;
+      }
+      case TK_ERROR: {
+        sqlite3_free(z);
+        return 0;
+      }
+      case TK_LITERAL: {
+        z[j++] = '?';
+        break;
+      }
+      case TK_PUNCT:
+      case TK_NAME: {
+        if( n==4 && sqlite3_strnicmp(zSql+i,"NULL",4)==0 ){
+          if( (j>=3 && strncmp(z+j-2,"is",2)==0 && !IdChar(z[j-3]))
+           || (j>=4 && strncmp(z+j-3,"not",3)==0 && !IdChar(z[j-4]))
+          ){
+            /* NULL is a keyword in this case, not a literal value */
+          }else{
+            /* Here the NULL is a literal value */
+            z[j++] = '?';
+            break;
+          }
+        }
+        if( j>0 && IdChar(z[j-1]) && IdChar(zSql[i]) ) z[j++] = ' ';
+        for(k=0; k<n; k++){
+          z[j++] = sqlite3Tolower(zSql[i+k]);
+        }
+        break;
+      }
+    }
+  }
+  while( j>0 && z[j-1]==' ' ){ j--; }
+  if( j>0 && z[j-1]!=';' ){ z[j++] = ';'; }
+  z[j] = 0;
+
+  /* Make a second pass converting "in(...)" where the "..." is not a
+  ** SELECT statement into "in(?,?,?)" */
+  for(i=0; i<j; i=n){
+    char *zIn = strstr(z+i, "in(");
+    int nParen;
+    if( zIn==0 ) break;
+    n = (int)(zIn-z)+3;  /* Index of first char past "in(" */
+    if( n && IdChar(zIn[-1]) ) continue;
+    if( strncmp(zIn, "in(select",9)==0 && !IdChar(zIn[9]) ) continue;
+    if( strncmp(zIn, "in(with",7)==0 && !IdChar(zIn[7]) ) continue;
+    for(nParen=1, k=0; z[n+k]; k++){
+      if( z[n+k]=='(' ) nParen++;
+      if( z[n+k]==')' ){
+        nParen--;
+        if( nParen==0 ) break;
+      }
+    }
+    /* k is the number of bytes in the "..." within "in(...)" */
+    if( k<5 ){
+      z = sqlite3_realloc64(z, j+(5-k)+1);
+      if( z==0 ) return 0;
+      memmove(z+n+5, z+n+k, j-(n+k));
+    }else if( k>5 ){
+      memmove(z+n+5, z+n+k, j-(n+k));
+    }
+    j = j-k+5;
+    z[j] = 0;
+    memcpy(z+n, "?,?,?", 5);
+  }
+  return z;
+}
+
+/*
+** For testing purposes, or to build a stand-alone SQL normalizer program,
+** compile this one source file with the -DSQLITE_NORMALIZE_CLI and link
+** it against any SQLite library.  The resulting command-line program will
+** run sqlite3_normalize() over the text of all files named on the command-
+** line and show the result on standard output.
+*/
+#ifdef SQLITE_NORMALIZE_CLI
+#include <stdio.h>
+#include <stdlib.h>
+
+/*
+** Break zIn up into separate SQL statements and run sqlite3_normalize()
+** on each one.  Print the result of each run.
+*/
+static void normalizeFile(char *zIn){
+  int i;
+  if( zIn==0 ) return;
+  for(i=0; zIn[i]; i++){
+    char cSaved;
+    if( zIn[i]!=';' ) continue;
+    cSaved = zIn[i+1];
+    zIn[i+1] = 0;
+    if( sqlite3_complete(zIn) ){
+      char *zOut = sqlite3_normalize(zIn);
+      if( zOut ){
+        printf("%s\n", zOut);
+        sqlite3_free(zOut);
+      }else{
+        fprintf(stderr, "ERROR: %s\n", zIn);
+      }
+      zIn[i+1] = cSaved;
+      zIn += i+1;
+      i = -1;
+    }else{
+      zIn[i+1] = cSaved;
+    }
+  }
+}
+
+/*
+** The main routine for "sql_normalize".  Read files named on the
+** command-line and run the text of each through sqlite3_normalize().
+*/
+int main(int argc, char **argv){
+  int i;
+  FILE *in;
+  char *zBuf = 0;
+  sqlite3_int64 sz, got;
+
+  for(i=1; i<argc; i++){
+    in = fopen(argv[i], "rb");
+    if( in==0 ){
+      fprintf(stderr, "cannot open \"%s\"\n", argv[i]);
+      continue;
+    }
+    fseek(in, 0, SEEK_END);
+    sz = ftell(in);
+    rewind(in);
+    zBuf = sqlite3_realloc64(zBuf, sz+1);
+    if( zBuf==0 ){
+      fprintf(stderr, "failed to malloc for %lld bytes\n", sz);
+      exit(1);
+    }
+    got = fread(zBuf, 1, sz, in);
+    fclose(in);
+    if( got!=sz ){
+      fprintf(stderr, "only able to read %lld of %lld bytes from \"%s\"\n",
+              got, sz, argv[i]);
+    }else{
+      zBuf[got] = 0;
+      normalizeFile(zBuf);
+    }
+  }
+  sqlite3_free(zBuf);
+}
+#endif /* SQLITE_NORMALIZE_CLI */

ext/misc/rot13.c

@@ -47,9 +47,9 @@ static void rot13func(
   const unsigned char *zIn;
   int nIn;
   unsigned char *zOut;
-  char *zToFree = 0;
+  unsigned char *zToFree = 0;
   int i;
-  char zTemp[100];
+  unsigned char zTemp[100];
   assert( argc==1 );
   if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return;
   zIn = (const unsigned char*)sqlite3_value_text(argv[0]);
@@ -57,7 +57,7 @@ static void rot13func(
   if( nIn<sizeof(zTemp)-1 ){
     zOut = zTemp;
   }else{
-    zOut = zToFree = sqlite3_malloc( nIn+1 );
+    zOut = zToFree = (unsigned char*)sqlite3_malloc64( nIn+1 );
     if( zOut==0 ){
       sqlite3_result_error_nomem(context);
       return;

ext/misc/scrub.c

@@ -131,7 +131,7 @@ static void scrubBackupWrite(ScrubState *p, int pgno, const u8 *pData){
     scrubBackupErr(p, "write failed for page %d", pgno);
     p->rcErr = SQLITE_IOERR;
   }
-  if( pgno>p->iLastPage ) p->iLastPage = pgno;
+  if( (u32)pgno>p->iLastPage ) p->iLastPage = pgno;
 }
 
 /* Prepare a statement against the "db" database. */
@@ -459,7 +459,7 @@ static void scrubBackupBtree(ScrubState *p, int pgno, int iDepth){
     nLocal = K<=X ? K : M;
     if( pc+nLocal > p->szUsable-4 ){ ln=__LINE__; goto btree_corrupt; }
     iChild = scrubBackupInt32(&a[pc+nLocal]);
-    scrubBackupOverflow(p, iChild, P-nLocal);
+    scrubBackupOverflow(p, iChild, (u32)(P-nLocal));
   }
 
   /* Walk the right-most tree */

ext/misc/series.c

@@ -195,8 +195,9 @@ static int seriesColumn(
 }
 
 /*
-** Return the rowid for the current row.  In this implementation, the
+** Return the rowid for the current row. In this implementation, the
-** rowid is the same as the output value.
+** first row returned is assigned rowid value 1, and each subsequent
+** row a value 1 more than that of the previous.
 */
 static int seriesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
   series_cursor *pCur = (series_cursor*)cur;
@@ -269,6 +270,15 @@ static int seriesFilter(
   }else{
     pCur->iStep = 1;
   }
+  for(i=0; i<argc; i++){
+    if( sqlite3_value_type(argv[i])==SQLITE_NULL ){
+      /* If any of the constraints have a NULL value, then return no rows.
+      ** See ticket https://www.sqlite.org/src/info/fac496b61722daf2 */
+      pCur->mnValue = 1;
+      pCur->mxValue = 0;
+      break;
+    }
+  }
   if( idxNum & 8 ){
     pCur->isDesc = 1;
     pCur->iValue = pCur->mxValue;

ext/misc/sha1.c

@@ -10,7 +10,7 @@
 **
 ******************************************************************************
 **
-** This SQLite extension implements a functions that compute SHA1 hashes.
+** This SQLite extension implements functions that compute SHA1 hashes.
 ** Two SQL functions are implemented:
 **
 **     sha1(X)

ext/misc/shathree.c

@@ -10,7 +10,7 @@
 **
 ******************************************************************************
 **
-** This SQLite extension implements a functions that compute SHA1 hashes.
+** This SQLite extension implements functions that compute SHA3 hashes.
 ** Two SQL functions are implemented:
 **
 **     sha3(X,SIZE)
@@ -78,9 +78,9 @@ struct SHA3Context {
 */
 static void KeccakF1600Step(SHA3Context *p){
   int i;
-  u64 B0, B1, B2, B3, B4;
+  u64 b0, b1, b2, b3, b4;
-  u64 C0, C1, C2, C3, C4;
+  u64 c0, c1, c2, c3, c4;
-  u64 D0, D1, D2, D3, D4;
+  u64 d0, d1, d2, d3, d4;
   static const u64 RC[] = {
     0x0000000000000001ULL,  0x0000000000008082ULL,
     0x800000000000808aULL,  0x8000000080008000ULL,
@@ -95,301 +95,301 @@ static void KeccakF1600Step(SHA3Context *p){
     0x8000000080008081ULL,  0x8000000000008080ULL,
     0x0000000080000001ULL,  0x8000000080008008ULL
   };
-# define A00 (p->u.s[0])
+# define a00 (p->u.s[0])
-# define A01 (p->u.s[1])
+# define a01 (p->u.s[1])
-# define A02 (p->u.s[2])
+# define a02 (p->u.s[2])
-# define A03 (p->u.s[3])
+# define a03 (p->u.s[3])
-# define A04 (p->u.s[4])
+# define a04 (p->u.s[4])
-# define A10 (p->u.s[5])
+# define a10 (p->u.s[5])
-# define A11 (p->u.s[6])
+# define a11 (p->u.s[6])
-# define A12 (p->u.s[7])
+# define a12 (p->u.s[7])
-# define A13 (p->u.s[8])
+# define a13 (p->u.s[8])
-# define A14 (p->u.s[9])
+# define a14 (p->u.s[9])
-# define A20 (p->u.s[10])
+# define a20 (p->u.s[10])
-# define A21 (p->u.s[11])
+# define a21 (p->u.s[11])
-# define A22 (p->u.s[12])
+# define a22 (p->u.s[12])
-# define A23 (p->u.s[13])
+# define a23 (p->u.s[13])
-# define A24 (p->u.s[14])
+# define a24 (p->u.s[14])
-# define A30 (p->u.s[15])
+# define a30 (p->u.s[15])
-# define A31 (p->u.s[16])
+# define a31 (p->u.s[16])
-# define A32 (p->u.s[17])
+# define a32 (p->u.s[17])
-# define A33 (p->u.s[18])
+# define a33 (p->u.s[18])
-# define A34 (p->u.s[19])
+# define a34 (p->u.s[19])
-# define A40 (p->u.s[20])
+# define a40 (p->u.s[20])
-# define A41 (p->u.s[21])
+# define a41 (p->u.s[21])
-# define A42 (p->u.s[22])
+# define a42 (p->u.s[22])
-# define A43 (p->u.s[23])
+# define a43 (p->u.s[23])
-# define A44 (p->u.s[24])
+# define a44 (p->u.s[24])
 # define ROL64(a,x) ((a<<x)|(a>>(64-x)))
 
   for(i=0; i<24; i+=4){
-    C0 = A00^A10^A20^A30^A40;
+    c0 = a00^a10^a20^a30^a40;
-    C1 = A01^A11^A21^A31^A41;
+    c1 = a01^a11^a21^a31^a41;
-    C2 = A02^A12^A22^A32^A42;
+    c2 = a02^a12^a22^a32^a42;
-    C3 = A03^A13^A23^A33^A43;
+    c3 = a03^a13^a23^a33^a43;
-    C4 = A04^A14^A24^A34^A44;
+    c4 = a04^a14^a24^a34^a44;
-    D0 = C4^ROL64(C1, 1);
+    d0 = c4^ROL64(c1, 1);
-    D1 = C0^ROL64(C2, 1);
+    d1 = c0^ROL64(c2, 1);
-    D2 = C1^ROL64(C3, 1);
+    d2 = c1^ROL64(c3, 1);
-    D3 = C2^ROL64(C4, 1);
+    d3 = c2^ROL64(c4, 1);
-    D4 = C3^ROL64(C0, 1);
+    d4 = c3^ROL64(c0, 1);
 
-    B0 = (A00^D0);
+    b0 = (a00^d0);
-    B1 = ROL64((A11^D1), 44);
+    b1 = ROL64((a11^d1), 44);
-    B2 = ROL64((A22^D2), 43);
+    b2 = ROL64((a22^d2), 43);
-    B3 = ROL64((A33^D3), 21);
+    b3 = ROL64((a33^d3), 21);
-    B4 = ROL64((A44^D4), 14);
+    b4 = ROL64((a44^d4), 14);
-    A00 =   B0 ^((~B1)&  B2 );
+    a00 =   b0 ^((~b1)&  b2 );
-    A00 ^= RC[i];
+    a00 ^= RC[i];
-    A11 =   B1 ^((~B2)&  B3 );
+    a11 =   b1 ^((~b2)&  b3 );
-    A22 =   B2 ^((~B3)&  B4 );
+    a22 =   b2 ^((~b3)&  b4 );
-    A33 =   B3 ^((~B4)&  B0 );
+    a33 =   b3 ^((~b4)&  b0 );
-    A44 =   B4 ^((~B0)&  B1 );
+    a44 =   b4 ^((~b0)&  b1 );
 
-    B2 = ROL64((A20^D0), 3);
+    b2 = ROL64((a20^d0), 3);
-    B3 = ROL64((A31^D1), 45);
+    b3 = ROL64((a31^d1), 45);
-    B4 = ROL64((A42^D2), 61);
+    b4 = ROL64((a42^d2), 61);
-    B0 = ROL64((A03^D3), 28);
+    b0 = ROL64((a03^d3), 28);
-    B1 = ROL64((A14^D4), 20);
+    b1 = ROL64((a14^d4), 20);
-    A20 =   B0 ^((~B1)&  B2 );
+    a20 =   b0 ^((~b1)&  b2 );
-    A31 =   B1 ^((~B2)&  B3 );
+    a31 =   b1 ^((~b2)&  b3 );
-    A42 =   B2 ^((~B3)&  B4 );
+    a42 =   b2 ^((~b3)&  b4 );
-    A03 =   B3 ^((~B4)&  B0 );
+    a03 =   b3 ^((~b4)&  b0 );
-    A14 =   B4 ^((~B0)&  B1 );
+    a14 =   b4 ^((~b0)&  b1 );
 
-    B4 = ROL64((A40^D0), 18);
+    b4 = ROL64((a40^d0), 18);
-    B0 = ROL64((A01^D1), 1);
+    b0 = ROL64((a01^d1), 1);
-    B1 = ROL64((A12^D2), 6);
+    b1 = ROL64((a12^d2), 6);
-    B2 = ROL64((A23^D3), 25);
+    b2 = ROL64((a23^d3), 25);
-    B3 = ROL64((A34^D4), 8);
+    b3 = ROL64((a34^d4), 8);
-    A40 =   B0 ^((~B1)&  B2 );
+    a40 =   b0 ^((~b1)&  b2 );
-    A01 =   B1 ^((~B2)&  B3 );
+    a01 =   b1 ^((~b2)&  b3 );
-    A12 =   B2 ^((~B3)&  B4 );
+    a12 =   b2 ^((~b3)&  b4 );
-    A23 =   B3 ^((~B4)&  B0 );
+    a23 =   b3 ^((~b4)&  b0 );
-    A34 =   B4 ^((~B0)&  B1 );
+    a34 =   b4 ^((~b0)&  b1 );
 
-    B1 = ROL64((A10^D0), 36);
+    b1 = ROL64((a10^d0), 36);
-    B2 = ROL64((A21^D1), 10);
+    b2 = ROL64((a21^d1), 10);
-    B3 = ROL64((A32^D2), 15);
+    b3 = ROL64((a32^d2), 15);
-    B4 = ROL64((A43^D3), 56);
+    b4 = ROL64((a43^d3), 56);
-    B0 = ROL64((A04^D4), 27);
+    b0 = ROL64((a04^d4), 27);
-    A10 =   B0 ^((~B1)&  B2 );
+    a10 =   b0 ^((~b1)&  b2 );
-    A21 =   B1 ^((~B2)&  B3 );
+    a21 =   b1 ^((~b2)&  b3 );
-    A32 =   B2 ^((~B3)&  B4 );
+    a32 =   b2 ^((~b3)&  b4 );
-    A43 =   B3 ^((~B4)&  B0 );
+    a43 =   b3 ^((~b4)&  b0 );
-    A04 =   B4 ^((~B0)&  B1 );
+    a04 =   b4 ^((~b0)&  b1 );
 
-    B3 = ROL64((A30^D0), 41);
+    b3 = ROL64((a30^d0), 41);
-    B4 = ROL64((A41^D1), 2);
+    b4 = ROL64((a41^d1), 2);
-    B0 = ROL64((A02^D2), 62);
+    b0 = ROL64((a02^d2), 62);
-    B1 = ROL64((A13^D3), 55);
+    b1 = ROL64((a13^d3), 55);
-    B2 = ROL64((A24^D4), 39);
+    b2 = ROL64((a24^d4), 39);
-    A30 =   B0 ^((~B1)&  B2 );
+    a30 =   b0 ^((~b1)&  b2 );
-    A41 =   B1 ^((~B2)&  B3 );
+    a41 =   b1 ^((~b2)&  b3 );
-    A02 =   B2 ^((~B3)&  B4 );
+    a02 =   b2 ^((~b3)&  b4 );
-    A13 =   B3 ^((~B4)&  B0 );
+    a13 =   b3 ^((~b4)&  b0 );
-    A24 =   B4 ^((~B0)&  B1 );
+    a24 =   b4 ^((~b0)&  b1 );
 
-    C0 = A00^A20^A40^A10^A30;
+    c0 = a00^a20^a40^a10^a30;
-    C1 = A11^A31^A01^A21^A41;
+    c1 = a11^a31^a01^a21^a41;
-    C2 = A22^A42^A12^A32^A02;
+    c2 = a22^a42^a12^a32^a02;
-    C3 = A33^A03^A23^A43^A13;
+    c3 = a33^a03^a23^a43^a13;
-    C4 = A44^A14^A34^A04^A24;
+    c4 = a44^a14^a34^a04^a24;
-    D0 = C4^ROL64(C1, 1);
+    d0 = c4^ROL64(c1, 1);
-    D1 = C0^ROL64(C2, 1);
+    d1 = c0^ROL64(c2, 1);
-    D2 = C1^ROL64(C3, 1);
+    d2 = c1^ROL64(c3, 1);
-    D3 = C2^ROL64(C4, 1);
+    d3 = c2^ROL64(c4, 1);
-    D4 = C3^ROL64(C0, 1);
+    d4 = c3^ROL64(c0, 1);
 
-    B0 = (A00^D0);
+    b0 = (a00^d0);
-    B1 = ROL64((A31^D1), 44);
+    b1 = ROL64((a31^d1), 44);
-    B2 = ROL64((A12^D2), 43);
+    b2 = ROL64((a12^d2), 43);
-    B3 = ROL64((A43^D3), 21);
+    b3 = ROL64((a43^d3), 21);
-    B4 = ROL64((A24^D4), 14);
+    b4 = ROL64((a24^d4), 14);
-    A00 =   B0 ^((~B1)&  B2 );
+    a00 =   b0 ^((~b1)&  b2 );
-    A00 ^= RC[i+1];
+    a00 ^= RC[i+1];
-    A31 =   B1 ^((~B2)&  B3 );
+    a31 =   b1 ^((~b2)&  b3 );
-    A12 =   B2 ^((~B3)&  B4 );
+    a12 =   b2 ^((~b3)&  b4 );
-    A43 =   B3 ^((~B4)&  B0 );
+    a43 =   b3 ^((~b4)&  b0 );
-    A24 =   B4 ^((~B0)&  B1 );
+    a24 =   b4 ^((~b0)&  b1 );
 
-    B2 = ROL64((A40^D0), 3);
+    b2 = ROL64((a40^d0), 3);
-    B3 = ROL64((A21^D1), 45);
+    b3 = ROL64((a21^d1), 45);
-    B4 = ROL64((A02^D2), 61);
+    b4 = ROL64((a02^d2), 61);
-    B0 = ROL64((A33^D3), 28);
+    b0 = ROL64((a33^d3), 28);
-    B1 = ROL64((A14^D4), 20);
+    b1 = ROL64((a14^d4), 20);
-    A40 =   B0 ^((~B1)&  B2 );
+    a40 =   b0 ^((~b1)&  b2 );
-    A21 =   B1 ^((~B2)&  B3 );
+    a21 =   b1 ^((~b2)&  b3 );
-    A02 =   B2 ^((~B3)&  B4 );
+    a02 =   b2 ^((~b3)&  b4 );
-    A33 =   B3 ^((~B4)&  B0 );
+    a33 =   b3 ^((~b4)&  b0 );
-    A14 =   B4 ^((~B0)&  B1 );
+    a14 =   b4 ^((~b0)&  b1 );
 
-    B4 = ROL64((A30^D0), 18);
+    b4 = ROL64((a30^d0), 18);
-    B0 = ROL64((A11^D1), 1);
+    b0 = ROL64((a11^d1), 1);
-    B1 = ROL64((A42^D2), 6);
+    b1 = ROL64((a42^d2), 6);
-    B2 = ROL64((A23^D3), 25);
+    b2 = ROL64((a23^d3), 25);
-    B3 = ROL64((A04^D4), 8);
+    b3 = ROL64((a04^d4), 8);
-    A30 =   B0 ^((~B1)&  B2 );
+    a30 =   b0 ^((~b1)&  b2 );
-    A11 =   B1 ^((~B2)&  B3 );
+    a11 =   b1 ^((~b2)&  b3 );
-    A42 =   B2 ^((~B3)&  B4 );
+    a42 =   b2 ^((~b3)&  b4 );
-    A23 =   B3 ^((~B4)&  B0 );
+    a23 =   b3 ^((~b4)&  b0 );
-    A04 =   B4 ^((~B0)&  B1 );
+    a04 =   b4 ^((~b0)&  b1 );
 
-    B1 = ROL64((A20^D0), 36);
+    b1 = ROL64((a20^d0), 36);
-    B2 = ROL64((A01^D1), 10);
+    b2 = ROL64((a01^d1), 10);
-    B3 = ROL64((A32^D2), 15);
+    b3 = ROL64((a32^d2), 15);
-    B4 = ROL64((A13^D3), 56);
+    b4 = ROL64((a13^d3), 56);
-    B0 = ROL64((A44^D4), 27);
+    b0 = ROL64((a44^d4), 27);
-    A20 =   B0 ^((~B1)&  B2 );
+    a20 =   b0 ^((~b1)&  b2 );
-    A01 =   B1 ^((~B2)&  B3 );
+    a01 =   b1 ^((~b2)&  b3 );
-    A32 =   B2 ^((~B3)&  B4 );
+    a32 =   b2 ^((~b3)&  b4 );
-    A13 =   B3 ^((~B4)&  B0 );
+    a13 =   b3 ^((~b4)&  b0 );
-    A44 =   B4 ^((~B0)&  B1 );
+    a44 =   b4 ^((~b0)&  b1 );
 
-    B3 = ROL64((A10^D0), 41);
+    b3 = ROL64((a10^d0), 41);
-    B4 = ROL64((A41^D1), 2);
+    b4 = ROL64((a41^d1), 2);
-    B0 = ROL64((A22^D2), 62);
+    b0 = ROL64((a22^d2), 62);
-    B1 = ROL64((A03^D3), 55);
+    b1 = ROL64((a03^d3), 55);
-    B2 = ROL64((A34^D4), 39);
+    b2 = ROL64((a34^d4), 39);
-    A10 =   B0 ^((~B1)&  B2 );
+    a10 =   b0 ^((~b1)&  b2 );
-    A41 =   B1 ^((~B2)&  B3 );
+    a41 =   b1 ^((~b2)&  b3 );
-    A22 =   B2 ^((~B3)&  B4 );
+    a22 =   b2 ^((~b3)&  b4 );
-    A03 =   B3 ^((~B4)&  B0 );
+    a03 =   b3 ^((~b4)&  b0 );
-    A34 =   B4 ^((~B0)&  B1 );
+    a34 =   b4 ^((~b0)&  b1 );
 
-    C0 = A00^A40^A30^A20^A10;
+    c0 = a00^a40^a30^a20^a10;
-    C1 = A31^A21^A11^A01^A41;
+    c1 = a31^a21^a11^a01^a41;
-    C2 = A12^A02^A42^A32^A22;
+    c2 = a12^a02^a42^a32^a22;
-    C3 = A43^A33^A23^A13^A03;
+    c3 = a43^a33^a23^a13^a03;
-    C4 = A24^A14^A04^A44^A34;
+    c4 = a24^a14^a04^a44^a34;
-    D0 = C4^ROL64(C1, 1);
+    d0 = c4^ROL64(c1, 1);
-    D1 = C0^ROL64(C2, 1);
+    d1 = c0^ROL64(c2, 1);
-    D2 = C1^ROL64(C3, 1);
+    d2 = c1^ROL64(c3, 1);
-    D3 = C2^ROL64(C4, 1);
+    d3 = c2^ROL64(c4, 1);
-    D4 = C3^ROL64(C0, 1);
+    d4 = c3^ROL64(c0, 1);
 
-    B0 = (A00^D0);
+    b0 = (a00^d0);
-    B1 = ROL64((A21^D1), 44);
+    b1 = ROL64((a21^d1), 44);
-    B2 = ROL64((A42^D2), 43);
+    b2 = ROL64((a42^d2), 43);
-    B3 = ROL64((A13^D3), 21);
+    b3 = ROL64((a13^d3), 21);
-    B4 = ROL64((A34^D4), 14);
+    b4 = ROL64((a34^d4), 14);
-    A00 =   B0 ^((~B1)&  B2 );
+    a00 =   b0 ^((~b1)&  b2 );
-    A00 ^= RC[i+2];
+    a00 ^= RC[i+2];
-    A21 =   B1 ^((~B2)&  B3 );
+    a21 =   b1 ^((~b2)&  b3 );
-    A42 =   B2 ^((~B3)&  B4 );
+    a42 =   b2 ^((~b3)&  b4 );
-    A13 =   B3 ^((~B4)&  B0 );
+    a13 =   b3 ^((~b4)&  b0 );
-    A34 =   B4 ^((~B0)&  B1 );
+    a34 =   b4 ^((~b0)&  b1 );
 
-    B2 = ROL64((A30^D0), 3);
+    b2 = ROL64((a30^d0), 3);
-    B3 = ROL64((A01^D1), 45);
+    b3 = ROL64((a01^d1), 45);
-    B4 = ROL64((A22^D2), 61);
+    b4 = ROL64((a22^d2), 61);
-    B0 = ROL64((A43^D3), 28);
+    b0 = ROL64((a43^d3), 28);
-    B1 = ROL64((A14^D4), 20);
+    b1 = ROL64((a14^d4), 20);
-    A30 =   B0 ^((~B1)&  B2 );
+    a30 =   b0 ^((~b1)&  b2 );
-    A01 =   B1 ^((~B2)&  B3 );
+    a01 =   b1 ^((~b2)&  b3 );
-    A22 =   B2 ^((~B3)&  B4 );
+    a22 =   b2 ^((~b3)&  b4 );
-    A43 =   B3 ^((~B4)&  B0 );
+    a43 =   b3 ^((~b4)&  b0 );
-    A14 =   B4 ^((~B0)&  B1 );
+    a14 =   b4 ^((~b0)&  b1 );
 
-    B4 = ROL64((A10^D0), 18);
+    b4 = ROL64((a10^d0), 18);
-    B0 = ROL64((A31^D1), 1);
+    b0 = ROL64((a31^d1), 1);
-    B1 = ROL64((A02^D2), 6);
+    b1 = ROL64((a02^d2), 6);
-    B2 = ROL64((A23^D3), 25);
+    b2 = ROL64((a23^d3), 25);
-    B3 = ROL64((A44^D4), 8);
+    b3 = ROL64((a44^d4), 8);
-    A10 =   B0 ^((~B1)&  B2 );
+    a10 =   b0 ^((~b1)&  b2 );
-    A31 =   B1 ^((~B2)&  B3 );
+    a31 =   b1 ^((~b2)&  b3 );
-    A02 =   B2 ^((~B3)&  B4 );
+    a02 =   b2 ^((~b3)&  b4 );
-    A23 =   B3 ^((~B4)&  B0 );
+    a23 =   b3 ^((~b4)&  b0 );
-    A44 =   B4 ^((~B0)&  B1 );
+    a44 =   b4 ^((~b0)&  b1 );
 
-    B1 = ROL64((A40^D0), 36);
+    b1 = ROL64((a40^d0), 36);
-    B2 = ROL64((A11^D1), 10);
+    b2 = ROL64((a11^d1), 10);
-    B3 = ROL64((A32^D2), 15);
+    b3 = ROL64((a32^d2), 15);
-    B4 = ROL64((A03^D3), 56);
+    b4 = ROL64((a03^d3), 56);
-    B0 = ROL64((A24^D4), 27);
+    b0 = ROL64((a24^d4), 27);
-    A40 =   B0 ^((~B1)&  B2 );
+    a40 =   b0 ^((~b1)&  b2 );
-    A11 =   B1 ^((~B2)&  B3 );
+    a11 =   b1 ^((~b2)&  b3 );
-    A32 =   B2 ^((~B3)&  B4 );
+    a32 =   b2 ^((~b3)&  b4 );
-    A03 =   B3 ^((~B4)&  B0 );
+    a03 =   b3 ^((~b4)&  b0 );
-    A24 =   B4 ^((~B0)&  B1 );
+    a24 =   b4 ^((~b0)&  b1 );
 
-    B3 = ROL64((A20^D0), 41);
+    b3 = ROL64((a20^d0), 41);
-    B4 = ROL64((A41^D1), 2);
+    b4 = ROL64((a41^d1), 2);
-    B0 = ROL64((A12^D2), 62);
+    b0 = ROL64((a12^d2), 62);
-    B1 = ROL64((A33^D3), 55);
+    b1 = ROL64((a33^d3), 55);
-    B2 = ROL64((A04^D4), 39);
+    b2 = ROL64((a04^d4), 39);
-    A20 =   B0 ^((~B1)&  B2 );
+    a20 =   b0 ^((~b1)&  b2 );
-    A41 =   B1 ^((~B2)&  B3 );
+    a41 =   b1 ^((~b2)&  b3 );
-    A12 =   B2 ^((~B3)&  B4 );
+    a12 =   b2 ^((~b3)&  b4 );
-    A33 =   B3 ^((~B4)&  B0 );
+    a33 =   b3 ^((~b4)&  b0 );
-    A04 =   B4 ^((~B0)&  B1 );
+    a04 =   b4 ^((~b0)&  b1 );
 
-    C0 = A00^A30^A10^A40^A20;
+    c0 = a00^a30^a10^a40^a20;
-    C1 = A21^A01^A31^A11^A41;
+    c1 = a21^a01^a31^a11^a41;
-    C2 = A42^A22^A02^A32^A12;
+    c2 = a42^a22^a02^a32^a12;
-    C3 = A13^A43^A23^A03^A33;
+    c3 = a13^a43^a23^a03^a33;
-    C4 = A34^A14^A44^A24^A04;
+    c4 = a34^a14^a44^a24^a04;
-    D0 = C4^ROL64(C1, 1);
+    d0 = c4^ROL64(c1, 1);
-    D1 = C0^ROL64(C2, 1);
+    d1 = c0^ROL64(c2, 1);
-    D2 = C1^ROL64(C3, 1);
+    d2 = c1^ROL64(c3, 1);
-    D3 = C2^ROL64(C4, 1);
+    d3 = c2^ROL64(c4, 1);
-    D4 = C3^ROL64(C0, 1);
+    d4 = c3^ROL64(c0, 1);
 
-    B0 = (A00^D0);
+    b0 = (a00^d0);
-    B1 = ROL64((A01^D1), 44);
+    b1 = ROL64((a01^d1), 44);
-    B2 = ROL64((A02^D2), 43);
+    b2 = ROL64((a02^d2), 43);
-    B3 = ROL64((A03^D3), 21);
+    b3 = ROL64((a03^d3), 21);
-    B4 = ROL64((A04^D4), 14);
+    b4 = ROL64((a04^d4), 14);
-    A00 =   B0 ^((~B1)&  B2 );
+    a00 =   b0 ^((~b1)&  b2 );
-    A00 ^= RC[i+3];
+    a00 ^= RC[i+3];
-    A01 =   B1 ^((~B2)&  B3 );
+    a01 =   b1 ^((~b2)&  b3 );
-    A02 =   B2 ^((~B3)&  B4 );
+    a02 =   b2 ^((~b3)&  b4 );
-    A03 =   B3 ^((~B4)&  B0 );
+    a03 =   b3 ^((~b4)&  b0 );
-    A04 =   B4 ^((~B0)&  B1 );
+    a04 =   b4 ^((~b0)&  b1 );
 
-    B2 = ROL64((A10^D0), 3);
+    b2 = ROL64((a10^d0), 3);
-    B3 = ROL64((A11^D1), 45);
+    b3 = ROL64((a11^d1), 45);
-    B4 = ROL64((A12^D2), 61);
+    b4 = ROL64((a12^d2), 61);
-    B0 = ROL64((A13^D3), 28);
+    b0 = ROL64((a13^d3), 28);
-    B1 = ROL64((A14^D4), 20);
+    b1 = ROL64((a14^d4), 20);
-    A10 =   B0 ^((~B1)&  B2 );
+    a10 =   b0 ^((~b1)&  b2 );
-    A11 =   B1 ^((~B2)&  B3 );
+    a11 =   b1 ^((~b2)&  b3 );
-    A12 =   B2 ^((~B3)&  B4 );
+    a12 =   b2 ^((~b3)&  b4 );
-    A13 =   B3 ^((~B4)&  B0 );
+    a13 =   b3 ^((~b4)&  b0 );
-    A14 =   B4 ^((~B0)&  B1 );
+    a14 =   b4 ^((~b0)&  b1 );
 
-    B4 = ROL64((A20^D0), 18);
+    b4 = ROL64((a20^d0), 18);
-    B0 = ROL64((A21^D1), 1);
+    b0 = ROL64((a21^d1), 1);
-    B1 = ROL64((A22^D2), 6);
+    b1 = ROL64((a22^d2), 6);
-    B2 = ROL64((A23^D3), 25);
+    b2 = ROL64((a23^d3), 25);
-    B3 = ROL64((A24^D4), 8);
+    b3 = ROL64((a24^d4), 8);
-    A20 =   B0 ^((~B1)&  B2 );
+    a20 =   b0 ^((~b1)&  b2 );
-    A21 =   B1 ^((~B2)&  B3 );
+    a21 =   b1 ^((~b2)&  b3 );
-    A22 =   B2 ^((~B3)&  B4 );
+    a22 =   b2 ^((~b3)&  b4 );
-    A23 =   B3 ^((~B4)&  B0 );
+    a23 =   b3 ^((~b4)&  b0 );
-    A24 =   B4 ^((~B0)&  B1 );
+    a24 =   b4 ^((~b0)&  b1 );
 
-    B1 = ROL64((A30^D0), 36);
+    b1 = ROL64((a30^d0), 36);
-    B2 = ROL64((A31^D1), 10);
+    b2 = ROL64((a31^d1), 10);
-    B3 = ROL64((A32^D2), 15);
+    b3 = ROL64((a32^d2), 15);
-    B4 = ROL64((A33^D3), 56);
+    b4 = ROL64((a33^d3), 56);
-    B0 = ROL64((A34^D4), 27);
+    b0 = ROL64((a34^d4), 27);
-    A30 =   B0 ^((~B1)&  B2 );
+    a30 =   b0 ^((~b1)&  b2 );
-    A31 =   B1 ^((~B2)&  B3 );
+    a31 =   b1 ^((~b2)&  b3 );
-    A32 =   B2 ^((~B3)&  B4 );
+    a32 =   b2 ^((~b3)&  b4 );
-    A33 =   B3 ^((~B4)&  B0 );
+    a33 =   b3 ^((~b4)&  b0 );
-    A34 =   B4 ^((~B0)&  B1 );
+    a34 =   b4 ^((~b0)&  b1 );
 
-    B3 = ROL64((A40^D0), 41);
+    b3 = ROL64((a40^d0), 41);
-    B4 = ROL64((A41^D1), 2);
+    b4 = ROL64((a41^d1), 2);
-    B0 = ROL64((A42^D2), 62);
+    b0 = ROL64((a42^d2), 62);
-    B1 = ROL64((A43^D3), 55);
+    b1 = ROL64((a43^d3), 55);
-    B2 = ROL64((A44^D4), 39);
+    b2 = ROL64((a44^d4), 39);
-    A40 =   B0 ^((~B1)&  B2 );
+    a40 =   b0 ^((~b1)&  b2 );
-    A41 =   B1 ^((~B2)&  B3 );
+    a41 =   b1 ^((~b2)&  b3 );
-    A42 =   B2 ^((~B3)&  B4 );
+    a42 =   b2 ^((~b3)&  b4 );
-    A43 =   B3 ^((~B4)&  B0 );
+    a43 =   b3 ^((~b4)&  b0 );
-    A44 =   B4 ^((~B0)&  B1 );
+    a44 =   b4 ^((~b0)&  b1 );
   }
 }
 

ext/misc/sqlar.c

@@ -0,0 +1,121 @@
+/*
+** 2017-12-17
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** Utility functions sqlar_compress() and sqlar_uncompress(). Useful
+** for working with sqlar archives and used by the shell tool's built-in
+** sqlar support.
+*/
+#include "sqlite3ext.h"
+SQLITE_EXTENSION_INIT1
+#include <zlib.h>
+
+/*
+** Implementation of the "sqlar_compress(X)" SQL function.
+**
+** If the type of X is SQLITE_BLOB, and compressing that blob using
+** zlib utility function compress() yields a smaller blob, return the
+** compressed blob. Otherwise, return a copy of X.
+**
+** SQLar uses the "zlib format" for compressed content.  The zlib format
+** contains a two-byte identification header and a four-byte checksum at
+** the end.  This is different from ZIP which uses the raw deflate format.
+**
+** Future enhancements to SQLar might add support for new compression formats.
+** If so, those new formats will be identified by alternative headers in the
+** compressed data.
+*/
+static void sqlarCompressFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  assert( argc==1 );
+  if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){
+    const Bytef *pData = sqlite3_value_blob(argv[0]);
+    uLong nData = sqlite3_value_bytes(argv[0]);
+    uLongf nOut = compressBound(nData);
+    Bytef *pOut;
+
+    pOut = (Bytef*)sqlite3_malloc(nOut);
+    if( pOut==0 ){
+      sqlite3_result_error_nomem(context);
+      return;
+    }else{
+      if( Z_OK!=compress(pOut, &nOut, pData, nData) ){
+        sqlite3_result_error(context, "error in compress()", -1);
+      }else if( nOut<nData ){
+        sqlite3_result_blob(context, pOut, nOut, SQLITE_TRANSIENT);
+      }else{
+        sqlite3_result_value(context, argv[0]);
+      }
+      sqlite3_free(pOut);
+    }
+  }else{
+    sqlite3_result_value(context, argv[0]);
+  }
+}
+
+/*
+** Implementation of the "sqlar_uncompress(X,SZ)" SQL function
+**
+** Parameter SZ is interpreted as an integer. If it is less than or
+** equal to zero, then this function returns a copy of X. Or, if
+** SZ is equal to the size of X when interpreted as a blob, also
+** return a copy of X. Otherwise, decompress blob X using zlib
+** utility function uncompress() and return the results (another
+** blob).
+*/
+static void sqlarUncompressFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  uLong nData;
+  uLongf sz;
+
+  assert( argc==2 );
+  sz = sqlite3_value_int(argv[1]);
+
+  if( sz<=0 || sz==(nData = sqlite3_value_bytes(argv[0])) ){
+    sqlite3_result_value(context, argv[0]);
+  }else{
+    const Bytef *pData= sqlite3_value_blob(argv[0]);
+    Bytef *pOut = sqlite3_malloc(sz);
+    if( Z_OK!=uncompress(pOut, &sz, pData, nData) ){
+      sqlite3_result_error(context, "error in uncompress()", -1);
+    }else{
+      sqlite3_result_blob(context, pOut, sz, SQLITE_TRANSIENT);
+    }
+    sqlite3_free(pOut);
+  }
+}
+
+
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_sqlar_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  int rc = SQLITE_OK;
+  SQLITE_EXTENSION_INIT2(pApi);
+  (void)pzErrMsg;  /* Unused parameter */
+  rc = sqlite3_create_function(db, "sqlar_compress", 1, SQLITE_UTF8, 0,
+                               sqlarCompressFunc, 0, 0);
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_create_function(db, "sqlar_uncompress", 2, SQLITE_UTF8, 0,
+                                 sqlarUncompressFunc, 0, 0);
+  }
+  return rc;
+}

ext/misc/templatevtab.c

@@ -0,0 +1,267 @@
+/*
+** 2018-04-19
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** This file implements a template virtual-table.
+** Developers can make a copy of this file as a baseline for writing
+** new virtual tables and/or table-valued functions.
+**
+** Steps for writing a new virtual table implementation:
+**
+**     (1)  Make a copy of this file.  Perhaps call it "mynewvtab.c"
+**
+**     (2)  Replace this header comment with something appropriate for
+**          the new virtual table
+**
+**     (3)  Change every occurrence of "templatevtab" to some other string
+**          appropriate for the new virtual table.  Ideally, the new string
+**          should be the basename of the source file: "mynewvtab".  Also
+**          globally change "TEMPLATEVTAB" to "MYNEWVTAB".
+**
+**     (4)  Run a test compilation to make sure the unmodified virtual
+**          table works.
+**
+**     (5)  Begin making incremental changes, testing as you go, to evolve
+**          the new virtual table to do what you want it to do.
+**
+** This template is minimal, in the sense that it uses only the required
+** methods on the sqlite3_module object.  As a result, templatevtab is
+** a read-only and eponymous-only table.  Those limitation can be removed
+** by adding new methods.
+**
+** This template implements an eponymous-only virtual table with a rowid and
+** two columns named "a" and "b".  The table as 10 rows with fixed integer
+** values. Usage example:
+**
+**     SELECT rowid, a, b FROM templatevtab;
+*/
+#if !defined(SQLITEINT_H)
+#include "sqlite3ext.h"
+#endif
+SQLITE_EXTENSION_INIT1
+#include <string.h>
+#include <assert.h>
+
+/* templatevtab_vtab is a subclass of sqlite3_vtab which is
+** underlying representation of the virtual table
+*/
+typedef struct templatevtab_vtab templatevtab_vtab;
+struct templatevtab_vtab {
+  sqlite3_vtab base;  /* Base class - must be first */
+  /* Add new fields here, as necessary */
+};
+
+/* templatevtab_cursor is a subclass of sqlite3_vtab_cursor which will
+** serve as the underlying representation of a cursor that scans
+** over rows of the result
+*/
+typedef struct templatevtab_cursor templatevtab_cursor;
+struct templatevtab_cursor {
+  sqlite3_vtab_cursor base;  /* Base class - must be first */
+  /* Insert new fields here.  For this templatevtab we only keep track
+  ** of the rowid */
+  sqlite3_int64 iRowid;      /* The rowid */
+};
+
+/*
+** The templatevtabConnect() method is invoked to create a new
+** template virtual table.
+**
+** Think of this routine as the constructor for templatevtab_vtab objects.
+**
+** All this routine needs to do is:
+**
+**    (1) Allocate the templatevtab_vtab object and initialize all fields.
+**
+**    (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the
+**        result set of queries against the virtual table will look like.
+*/
+static int templatevtabConnect(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
+){
+  templatevtab_vtab *pNew;
+  int rc;
+
+  rc = sqlite3_declare_vtab(db,
+           "CREATE TABLE x(a,b)"
+       );
+  /* For convenience, define symbolic names for the index to each column. */
+#define TEMPLATEVTAB_A  0
+#define TEMPLATEVTAB_B  1
+  if( rc==SQLITE_OK ){
+    pNew = sqlite3_malloc( sizeof(*pNew) );
+    *ppVtab = (sqlite3_vtab*)pNew;
+    if( pNew==0 ) return SQLITE_NOMEM;
+    memset(pNew, 0, sizeof(*pNew));
+  }
+  return rc;
+}
+
+/*
+** This method is the destructor for templatevtab_vtab objects.
+*/
+static int templatevtabDisconnect(sqlite3_vtab *pVtab){
+  templatevtab_vtab *p = (templatevtab_vtab*)pVtab;
+  sqlite3_free(p);
+  return SQLITE_OK;
+}
+
+/*
+** Constructor for a new templatevtab_cursor object.
+*/
+static int templatevtabOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
+  templatevtab_cursor *pCur;
+  pCur = sqlite3_malloc( sizeof(*pCur) );
+  if( pCur==0 ) return SQLITE_NOMEM;
+  memset(pCur, 0, sizeof(*pCur));
+  *ppCursor = &pCur->base;
+  return SQLITE_OK;
+}
+
+/*
+** Destructor for a templatevtab_cursor.
+*/
+static int templatevtabClose(sqlite3_vtab_cursor *cur){
+  templatevtab_cursor *pCur = (templatevtab_cursor*)cur;
+  sqlite3_free(pCur);
+  return SQLITE_OK;
+}
+
+
+/*
+** Advance a templatevtab_cursor to its next row of output.
+*/
+static int templatevtabNext(sqlite3_vtab_cursor *cur){
+  templatevtab_cursor *pCur = (templatevtab_cursor*)cur;
+  pCur->iRowid++;
+  return SQLITE_OK;
+}
+
+/*
+** Return values of columns for the row at which the templatevtab_cursor
+** is currently pointing.
+*/
+static int templatevtabColumn(
+  sqlite3_vtab_cursor *cur,   /* The cursor */
+  sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
+  int i                       /* Which column to return */
+){
+  templatevtab_cursor *pCur = (templatevtab_cursor*)cur;
+  switch( i ){
+    case TEMPLATEVTAB_A:
+      sqlite3_result_int(ctx, 1000 + pCur->iRowid);
+      break;
+    default:
+      assert( i==TEMPLATEVTAB_B );
+      sqlite3_result_int(ctx, 2000 + pCur->iRowid);
+      break;
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Return the rowid for the current row.  In this implementation, the
+** rowid is the same as the output value.
+*/
+static int templatevtabRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
+  templatevtab_cursor *pCur = (templatevtab_cursor*)cur;
+  *pRowid = pCur->iRowid;
+  return SQLITE_OK;
+}
+
+/*
+** Return TRUE if the cursor has been moved off of the last
+** row of output.
+*/
+static int templatevtabEof(sqlite3_vtab_cursor *cur){
+  templatevtab_cursor *pCur = (templatevtab_cursor*)cur;
+  return pCur->iRowid>=10;
+}
+
+/*
+** This method is called to "rewind" the templatevtab_cursor object back
+** to the first row of output.  This method is always called at least
+** once prior to any call to templatevtabColumn() or templatevtabRowid() or 
+** templatevtabEof().
+*/
+static int templatevtabFilter(
+  sqlite3_vtab_cursor *pVtabCursor, 
+  int idxNum, const char *idxStr,
+  int argc, sqlite3_value **argv
+){
+  templatevtab_cursor *pCur = (templatevtab_cursor *)pVtabCursor;
+  pCur->iRowid = 1;
+  return SQLITE_OK;
+}
+
+/*
+** SQLite will invoke this method one or more times while planning a query
+** that uses the virtual table.  This routine needs to create
+** a query plan for each invocation and compute an estimated cost for that
+** plan.
+*/
+static int templatevtabBestIndex(
+  sqlite3_vtab *tab,
+  sqlite3_index_info *pIdxInfo
+){
+  pIdxInfo->estimatedCost = (double)10;
+  pIdxInfo->estimatedRows = 10;
+  return SQLITE_OK;
+}
+
+/*
+** This following structure defines all the methods for the 
+** virtual table.
+*/
+static sqlite3_module templatevtabModule = {
+  /* iVersion    */ 0,
+  /* xCreate     */ 0,
+  /* xConnect    */ templatevtabConnect,
+  /* xBestIndex  */ templatevtabBestIndex,
+  /* xDisconnect */ templatevtabDisconnect,
+  /* xDestroy    */ 0,
+  /* xOpen       */ templatevtabOpen,
+  /* xClose      */ templatevtabClose,
+  /* xFilter     */ templatevtabFilter,
+  /* xNext       */ templatevtabNext,
+  /* xEof        */ templatevtabEof,
+  /* xColumn     */ templatevtabColumn,
+  /* xRowid      */ templatevtabRowid,
+  /* xUpdate     */ 0,
+  /* xBegin      */ 0,
+  /* xSync       */ 0,
+  /* xCommit     */ 0,
+  /* xRollback   */ 0,
+  /* xFindMethod */ 0,
+  /* xRename     */ 0,
+  /* xSavepoint  */ 0,
+  /* xRelease    */ 0,
+  /* xRollbackTo */ 0
+};
+
+
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_templatevtab_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  int rc = SQLITE_OK;
+  SQLITE_EXTENSION_INIT2(pApi);
+  rc = sqlite3_create_module(db, "templatevtab", &templatevtabModule, 0);
+  return rc;
+}

ext/misc/unionvtab.c

@@ -56,6 +56,8 @@
 **
 ** SWARMVTAB
 **
+**  LEGACY SYNTAX:
+**
 **   A "swarmvtab" virtual table is created similarly to a unionvtab table:
 **
 **     CREATE VIRTUAL TABLE <name>
@@ -66,13 +68,78 @@
 **   the database file containing the source table.  The <callback> option
 **   is optional.  If included, it is the name of an application-defined
 **   SQL function that is invoked with the URI of the file, if the file
-**   does not already exist on disk.
+**   does not already exist on disk when required by swarmvtab.
+**
+**  NEW SYNTAX:
+**
+**   Using the new syntax, a swarmvtab table is created with:
+**
+**      CREATE VIRTUAL TABLE <name> USING swarmvtab(
+**        <sql-statement> [, <options>]
+**      );
+**
+**   where valid <options> are:
+**
+**      missing=<udf-function-name>
+**      openclose=<udf-function-name>
+**      maxopen=<integer>
+**      <sql-parameter>=<text-value>
+**
+**   The <sql-statement> must return the same 4 columns as for a swarmvtab
+**   table in legacy mode. However, it may also return a 5th column - the
+**   "context" column. The text value returned in this column is not used
+**   at all by the swarmvtab implementation, except that it is passed as
+**   an additional argument to the two UDF functions that may be invoked
+**   (see below).
+**
+**   The "missing" option, if present, specifies the name of an SQL UDF
+**   function to be invoked if a database file is not already present on
+**   disk when required by swarmvtab. If the <sql-statement> did not provide
+**   a context column, it is invoked as:
+**
+**     SELECT <missing-udf>(<database filename/uri>);
+**
+**   Or, if there was a context column:
+**
+**     SELECT <missing-udf>(<database filename/uri>, <context>);
+**
+**   The "openclose" option may also specify a UDF function. This function
+**   is invoked right before swarmvtab opens a database, and right after
+**   it closes one. The first argument - or first two arguments, if
+**   <sql-statement> supplied the context column - is the same as for
+**   the "missing" UDF. Following this, the UDF is passed integer value
+**   0 before a db is opened, and 1 right after it is closed. If both
+**   a missing and openclose UDF is supplied, the application should expect
+**   the following sequence of calls (for a single database):
+**
+**      SELECT <openclose-udf>(<db filename>, <context>, 0);
+**      if( db not already on disk ){
+**          SELECT <missing-udf>(<db filename>, <context>);
+**      }
+**      ... swarmvtab uses database ...
+**      SELECT <openclose-udf>(<db filename>, <context>, 1);
+**
+**   The "maxopen" option is used to configure the maximum number of
+**   database files swarmvtab will hold open simultaneously (default 9).
+**
+**   If an option name begins with a ":" character, then it is assumed
+**   to be an SQL parameter. In this case, the specified text value is
+**   bound to the same variable of the <sql-statement> before it is 
+**   executed. It is an error of the named SQL parameter does not exist.
+**   For example:
+**
+**     CREATE VIRTUAL TABLE swarm USING swarmvtab(
+**       'SELECT :path || localfile, tbl, min, max FROM swarmdir',
+**       :path='/home/user/databases/'
+**       missing='missing_func'
+**     );
 */
 
 #include "sqlite3ext.h"
 SQLITE_EXTENSION_INIT1
 #include <assert.h>
 #include <string.h>
+#include <stdlib.h>
 
 #ifndef SQLITE_OMIT_VIRTUALTABLE
 
@@ -128,6 +195,7 @@ struct UnionSrc {
 
   /* Fields used by swarmvtab only */
   char *zFile;                    /* Database file containing table zTab */
+  char *zContext;                 /* Context string, if any */
   int nUser;                      /* Current number of users */
   sqlite3 *db;                    /* Database handle */
   UnionSrc *pNextClosable;        /* Next in list of closable sources */
@@ -145,8 +213,11 @@ struct UnionTab {
   UnionSrc *aSrc;                 /* Array of source tables, sorted by rowid */
 
   /* Used by swarmvtab only */
+  int bHasContext;                /* Has context strings */
   char *zSourceStr;               /* Expected unionSourceToStr() value */
-  char *zNotFoundCallback;        /* UDF to invoke if file not found on open */
+  sqlite3_stmt *pNotFound;        /* UDF to invoke if file not found on open */
+  sqlite3_stmt *pOpenClose;       /* UDF to invoke on open and close */
+
   UnionSrc *pClosable;            /* First in list of closable sources */
   int nOpen;                      /* Current number of open sources */
   int nMaxOpen;                   /* Maximum number of open sources */
@@ -351,6 +422,39 @@ static void unionFinalize(int *pRc, sqlite3_stmt *pStmt, char **pzErr){
   }
 }
 
+/*
+** If an "openclose" UDF was supplied when this virtual table was created,
+** invoke it now. The first argument passed is the name of the database
+** file for source pSrc. The second is integer value bClose.
+**
+** If successful, return SQLITE_OK. Otherwise an SQLite error code. In this
+** case if argument pzErr is not NULL, also set (*pzErr) to an English
+** language error message. The caller is responsible for eventually freeing 
+** any error message using sqlite3_free().
+*/
+static int unionInvokeOpenClose(
+  UnionTab *pTab, 
+  UnionSrc *pSrc, 
+  int bClose,
+  char **pzErr
+){
+  int rc = SQLITE_OK;
+  if( pTab->pOpenClose ){
+    sqlite3_bind_text(pTab->pOpenClose, 1, pSrc->zFile, -1, SQLITE_STATIC);
+    if( pTab->bHasContext ){
+      sqlite3_bind_text(pTab->pOpenClose, 2, pSrc->zContext, -1, SQLITE_STATIC);
+    }
+    sqlite3_bind_int(pTab->pOpenClose, 2+pTab->bHasContext, bClose);
+    sqlite3_step(pTab->pOpenClose);
+    if( SQLITE_OK!=(rc = sqlite3_reset(pTab->pOpenClose)) ){
+      if( pzErr ){
+        *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db));
+      }
+    }
+  }
+  return rc;
+}
+
 /*
 ** This function is a no-op for unionvtab. For swarmvtab, it attempts to
 ** close open database files until at most nMax are open. An SQLite error
@@ -358,13 +462,16 @@ static void unionFinalize(int *pRc, sqlite3_stmt *pStmt, char **pzErr){
 */
 static void unionCloseSources(UnionTab *pTab, int nMax){
   while( pTab->pClosable && pTab->nOpen>nMax ){
+    UnionSrc *p;
     UnionSrc **pp;
     for(pp=&pTab->pClosable; (*pp)->pNextClosable; pp=&(*pp)->pNextClosable);
-    assert( (*pp)->db );
+    p = *pp;
-    sqlite3_close((*pp)->db);
+    assert( p->db );
-    (*pp)->db = 0;
+    sqlite3_close(p->db);
+    p->db = 0;
     *pp = 0;
     pTab->nOpen--;
+    unionInvokeOpenClose(pTab, p, 1, 0);
   }
 }
 
@@ -377,13 +484,19 @@ static int unionDisconnect(sqlite3_vtab *pVtab){
     int i;
     for(i=0; i<pTab->nSrc; i++){
       UnionSrc *pSrc = &pTab->aSrc[i];
+      int bHaveSrcDb = (pSrc->db!=0);
+      sqlite3_close(pSrc->db);
+      if( bHaveSrcDb ){
+        unionInvokeOpenClose(pTab, pSrc, 1, 0);
+      }
       sqlite3_free(pSrc->zDb);
       sqlite3_free(pSrc->zTab);
       sqlite3_free(pSrc->zFile);
-      sqlite3_close(pSrc->db);
+      sqlite3_free(pSrc->zContext);
     }
+    sqlite3_finalize(pTab->pNotFound);
+    sqlite3_finalize(pTab->pOpenClose);
     sqlite3_free(pTab->zSourceStr);
-    sqlite3_free(pTab->zNotFoundCallback);
     sqlite3_free(pTab->aSrc);
     sqlite3_free(pTab);
   }
@@ -496,29 +609,31 @@ static int unionSourceCheck(UnionTab *pTab, char **pzErr){
   return rc;
 }
 
-
 /*
 ** Try to open the swarmvtab database.  If initially unable, invoke the
 ** not-found callback UDF and then try again.
 */
 static int unionOpenDatabaseInner(UnionTab *pTab, UnionSrc *pSrc, char **pzErr){
-  int rc = SQLITE_OK;
+  static const int openFlags = SQLITE_OPEN_READONLY | SQLITE_OPEN_URI;
-  static const int openFlags = 
+  int rc;
-       SQLITE_OPEN_READONLY | SQLITE_OPEN_URI;
+
+  rc = unionInvokeOpenClose(pTab, pSrc, 0, pzErr);
+  if( rc!=SQLITE_OK ) return rc;
+
   rc = sqlite3_open_v2(pSrc->zFile, &pSrc->db, openFlags, 0);
   if( rc==SQLITE_OK ) return rc;
-  if( pTab->zNotFoundCallback ){
+  if( pTab->pNotFound ){
-    char *zSql = sqlite3_mprintf("SELECT \"%w\"(%Q);",
-                    pTab->zNotFoundCallback, pSrc->zFile);
     sqlite3_close(pSrc->db);
     pSrc->db = 0;
-    if( zSql==0 ){
+    sqlite3_bind_text(pTab->pNotFound, 1, pSrc->zFile, -1, SQLITE_STATIC);
-      *pzErr = sqlite3_mprintf("out of memory");
+    if( pTab->bHasContext ){
-      return SQLITE_NOMEM;
+      sqlite3_bind_text(pTab->pNotFound, 2, pSrc->zContext, -1, SQLITE_STATIC);
+    }
+    sqlite3_step(pTab->pNotFound);
+    if( SQLITE_OK!=(rc = sqlite3_reset(pTab->pNotFound)) ){
+      *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db));
+      return rc;
     }
-    rc = sqlite3_exec(pTab->db, zSql, 0, 0, pzErr);
-    sqlite3_free(zSql);
-    if( rc ) return rc;
     rc = sqlite3_open_v2(pSrc->zFile, &pSrc->db, openFlags, 0);
   }
   if( rc!=SQLITE_OK ){
@@ -572,6 +687,7 @@ static int unionOpenDatabase(UnionTab *pTab, int iSrc, char **pzErr){
     }else{
       sqlite3_close(pSrc->db);
       pSrc->db = 0;
+      unionInvokeOpenClose(pTab, pSrc, 1, 0);
     }
   }
 
@@ -627,6 +743,132 @@ static int unionFinalizeCsrStmt(UnionCsr *pCsr){
   return rc;
 }
 
+/* 
+** Return true if the argument is a space, tab, CR or LF character.
+*/
+static int union_isspace(char c){
+  return (c==' ' || c=='\n' || c=='\r' || c=='\t');
+}
+
+/* 
+** Return true if the argument is an alphanumeric character in the 
+** ASCII range.
+*/
+static int union_isidchar(char c){
+  return ((c>='a' && c<='z') || (c>='A' && c<'Z') || (c>='0' && c<='9'));
+}
+
+/*
+** This function is called to handle all arguments following the first 
+** (the SQL statement) passed to a swarmvtab (not unionvtab) CREATE 
+** VIRTUAL TABLE statement. It may bind parameters to the SQL statement 
+** or configure members of the UnionTab object passed as the second
+** argument.
+**
+** Refer to header comments at the top of this file for a description
+** of the arguments parsed.
+**
+** This function is a no-op if *pRc is other than SQLITE_OK when it is
+** called. Otherwise, if an error occurs, *pRc is set to an SQLite error
+** code. In this case *pzErr may be set to point to a buffer containing
+** an English language error message. It is the responsibility of the 
+** caller to eventually free the buffer using sqlite3_free().
+*/
+static void unionConfigureVtab(
+  int *pRc,                       /* IN/OUT: Error code */
+  UnionTab *pTab,                 /* Table to configure */
+  sqlite3_stmt *pStmt,            /* SQL statement to find sources */
+  int nArg,                       /* Number of entries in azArg[] array */
+  const char * const *azArg,      /* Array of arguments to consider */
+  char **pzErr                    /* OUT: Error message */
+){
+  int rc = *pRc;
+  int i;
+  if( rc==SQLITE_OK ){
+    pTab->bHasContext = (sqlite3_column_count(pStmt)>4);
+  }
+  for(i=0; rc==SQLITE_OK && i<nArg; i++){
+    char *zArg = unionStrdup(&rc, azArg[i]);
+    if( zArg ){
+      int nOpt = 0;               /* Size of option name in bytes */
+      char *zOpt;                 /* Pointer to option name */
+      char *zVal;                 /* Pointer to value */
+
+      unionDequote(zArg);
+      zOpt = zArg;
+      while( union_isspace(*zOpt) ) zOpt++;
+      zVal = zOpt;
+      if( *zVal==':' ) zVal++;
+      while( union_isidchar(*zVal) ) zVal++;
+      nOpt = (int)(zVal-zOpt);
+
+      while( union_isspace(*zVal) ) zVal++;
+      if( *zVal=='=' ){
+        zOpt[nOpt] = '\0';
+        zVal++;
+        while( union_isspace(*zVal) ) zVal++;
+        zVal = unionStrdup(&rc, zVal);
+        if( zVal ){
+          unionDequote(zVal);
+          if( zOpt[0]==':' ){
+            /* A value to bind to the SQL statement */
+            int iParam = sqlite3_bind_parameter_index(pStmt, zOpt);
+            if( iParam==0 ){
+              *pzErr = sqlite3_mprintf(
+                  "swarmvtab: no such SQL parameter: %s", zOpt
+              );
+              rc = SQLITE_ERROR;
+            }else{
+              rc = sqlite3_bind_text(pStmt, iParam, zVal, -1, SQLITE_TRANSIENT);
+            }
+          }else if( nOpt==7 && 0==sqlite3_strnicmp(zOpt, "maxopen", 7) ){
+            pTab->nMaxOpen = atoi(zVal);
+            if( pTab->nMaxOpen<=0 ){
+              *pzErr = sqlite3_mprintf("swarmvtab: illegal maxopen value");
+              rc = SQLITE_ERROR;
+            }
+          }else if( nOpt==7 && 0==sqlite3_strnicmp(zOpt, "missing", 7) ){
+            if( pTab->pNotFound ){
+              *pzErr = sqlite3_mprintf(
+                  "swarmvtab: duplicate \"missing\" option");
+              rc = SQLITE_ERROR;
+            }else{
+              pTab->pNotFound = unionPreparePrintf(&rc, pzErr, pTab->db,
+                  "SELECT \"%w\"(?%s)", zVal, pTab->bHasContext ? ",?" : ""
+              );
+            }
+          }else if( nOpt==9 && 0==sqlite3_strnicmp(zOpt, "openclose", 9) ){
+            if( pTab->pOpenClose ){
+              *pzErr = sqlite3_mprintf(
+                  "swarmvtab: duplicate \"openclose\" option");
+              rc = SQLITE_ERROR;
+            }else{
+              pTab->pOpenClose = unionPreparePrintf(&rc, pzErr, pTab->db,
+                  "SELECT \"%w\"(?,?%s)", zVal, pTab->bHasContext ? ",?" : ""
+              );
+            }
+          }else{
+            *pzErr = sqlite3_mprintf("swarmvtab: unrecognized option: %s",zOpt);
+            rc = SQLITE_ERROR;
+          }
+          sqlite3_free(zVal);
+        }
+      }else{
+        if( i==0 && nArg==1 ){
+          pTab->pNotFound = unionPreparePrintf(&rc, pzErr, pTab->db,
+              "SELECT \"%w\"(?)", zArg
+          );
+        }else{
+          *pzErr = sqlite3_mprintf( "swarmvtab: parse error: %s", azArg[i]);
+          rc = SQLITE_ERROR;
+        }
+      }
+      sqlite3_free(zArg);
+    }
+  }
+  *pRc = rc;
+}
+
 /* 
 ** xConnect/xCreate method.
 **
@@ -654,7 +896,7 @@ static int unionConnect(
     /* unionvtab tables may only be created in the temp schema */
     *pzErr = sqlite3_mprintf("%s tables must be created in TEMP schema", zVtab);
     rc = SQLITE_ERROR;
-  }else if( argc!=4 && argc!=5 ){
+  }else if( argc<4 || (argc>4 && bSwarm==0) ){
     *pzErr = sqlite3_mprintf("wrong number of arguments for %s", zVtab);
     rc = SQLITE_ERROR;
   }else{
@@ -673,6 +915,17 @@ static int unionConnect(
 
     /* Allocate the UnionTab structure */
     pTab = unionMalloc(&rc, sizeof(UnionTab));
+    if( pTab ){
+      assert( rc==SQLITE_OK );
+      pTab->db = db;
+      pTab->bSwarm = bSwarm;
+      pTab->nMaxOpen = SWARMVTAB_MAX_OPEN;
+    }
+
+    /* Parse other CVT arguments, if any */
+    if( bSwarm ){
+      unionConfigureVtab(&rc, pTab, pStmt, argc-4, &argv[4], pzErr);
+    }
 
     /* Iterate through the rows returned by the SQL statement specified
     ** as an argument to the CREATE VIRTUAL TABLE statement. */
@@ -715,17 +968,15 @@ static int unionConnect(
         }else{
           pSrc->zDb = unionStrdup(&rc, zDb);
         }
+        if( pTab->bHasContext ){
+          const char *zContext = (const char*)sqlite3_column_text(pStmt, 4);
+          pSrc->zContext = unionStrdup(&rc, zContext);
+        }
       }
     }
     unionFinalize(&rc, pStmt, pzErr);
     pStmt = 0;
 
-    /* Capture the not-found callback UDF name */
-    if( rc==SQLITE_OK && argc>=5 ){
-      pTab->zNotFoundCallback = unionStrdup(&rc, argv[4]);
-      unionDequote(pTab->zNotFoundCallback);
-    }
-
     /* It is an error if the SELECT statement returned zero rows. If only
     ** because there is no way to determine the schema of the virtual 
     ** table in this case.  */
@@ -738,9 +989,6 @@ static int unionConnect(
     ** compatible schemas. For swarmvtab, attach the first database and
     ** check that the first table is a rowid table only.  */
     if( rc==SQLITE_OK ){
-      pTab->db = db;
-      pTab->bSwarm = bSwarm;
-      pTab->nMaxOpen = SWARMVTAB_MAX_OPEN;
       if( bSwarm ){
         rc = unionOpenDatabase(pTab, 0, pzErr);
       }else{

ext/misc/vtablog.c

@@ -0,0 +1,509 @@
+/*
+** 2017-08-10
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** This file implements a virtual table that prints diagnostic information
+** on stdout when its key interfaces are called.  This is intended for
+** interactive analysis and debugging of virtual table interfaces.
+**
+** Usage example:
+**
+**     .load ./vtablog
+**     CREATE VIRTUAL TABLE temp.log USING vtablog(
+**        schema='CREATE TABLE x(a,b,c)',
+**        rows=25
+**     );
+**     SELECT * FROM log;
+*/
+#include "sqlite3ext.h"
+SQLITE_EXTENSION_INIT1
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <string.h>
+#include <ctype.h>
+
+
+/* vtablog_vtab is a subclass of sqlite3_vtab which will
+** serve as the underlying representation of a vtablog virtual table
+*/
+typedef struct vtablog_vtab vtablog_vtab;
+struct vtablog_vtab {
+  sqlite3_vtab base;  /* Base class - must be first */
+  int nRow;           /* Number of rows in the table */
+  int iInst;          /* Instance number for this vtablog table */
+  int nCursor;        /* Number of cursors created */
+};
+
+/* vtablog_cursor is a subclass of sqlite3_vtab_cursor which will
+** serve as the underlying representation of a cursor that scans
+** over rows of the result
+*/
+typedef struct vtablog_cursor vtablog_cursor;
+struct vtablog_cursor {
+  sqlite3_vtab_cursor base;  /* Base class - must be first */
+  int iCursor;               /* Cursor number */
+  sqlite3_int64 iRowid;      /* The rowid */
+};
+
+/* Skip leading whitespace.  Return a pointer to the first non-whitespace
+** character, or to the zero terminator if the string has only whitespace */
+static const char *vtablog_skip_whitespace(const char *z){
+  while( isspace((unsigned char)z[0]) ) z++;
+  return z;
+}
+
+/* Remove trailing whitespace from the end of string z[] */
+static void vtablog_trim_whitespace(char *z){
+  size_t n = strlen(z);
+  while( n>0 && isspace((unsigned char)z[n]) ) n--;
+  z[n] = 0;
+}
+
+/* Dequote the string */
+static void vtablog_dequote(char *z){
+  int j;
+  char cQuote = z[0];
+  size_t i, n;
+
+  if( cQuote!='\'' && cQuote!='"' ) return;
+  n = strlen(z);
+  if( n<2 || z[n-1]!=z[0] ) return;
+  for(i=1, j=0; i<n-1; i++){
+    if( z[i]==cQuote && z[i+1]==cQuote ) i++;
+    z[j++] = z[i];
+  }
+  z[j] = 0;
+}
+
+/* Check to see if the string is of the form:  "TAG = VALUE" with optional
+** whitespace before and around tokens.  If it is, return a pointer to the
+** first character of VALUE.  If it is not, return NULL.
+*/
+static const char *vtablog_parameter(const char *zTag, int nTag, const char *z){
+  z = vtablog_skip_whitespace(z);
+  if( strncmp(zTag, z, nTag)!=0 ) return 0;
+  z = vtablog_skip_whitespace(z+nTag);
+  if( z[0]!='=' ) return 0;
+  return vtablog_skip_whitespace(z+1);
+}
+
+/* Decode a parameter that requires a dequoted string.
+**
+** Return non-zero on an error.
+*/
+static int vtablog_string_parameter(
+  char **pzErr,            /* Leave the error message here, if there is one */
+  const char *zParam,      /* Parameter we are checking for */
+  const char *zArg,        /* Raw text of the virtual table argment */
+  char **pzVal             /* Write the dequoted string value here */
+){
+  const char *zValue;
+  zValue = vtablog_parameter(zParam,(int)strlen(zParam),zArg);
+  if( zValue==0 ) return 0;
+  if( *pzVal ){
+    *pzErr = sqlite3_mprintf("more than one '%s' parameter", zParam);
+    return 1;
+  }
+  *pzVal = sqlite3_mprintf("%s", zValue);
+  if( *pzVal==0 ){
+    *pzErr = sqlite3_mprintf("out of memory");
+    return 1;
+  }
+  vtablog_trim_whitespace(*pzVal);
+  vtablog_dequote(*pzVal);
+  return 0;
+}
+
+#if 0 /* not used - yet */
+/* Return 0 if the argument is false and 1 if it is true.  Return -1 if
+** we cannot really tell.
+*/
+static int vtablog_boolean(const char *z){
+  if( sqlite3_stricmp("yes",z)==0
+   || sqlite3_stricmp("on",z)==0
+   || sqlite3_stricmp("true",z)==0
+   || (z[0]=='1' && z[1]==0)
+  ){
+    return 1;
+  }
+  if( sqlite3_stricmp("no",z)==0
+   || sqlite3_stricmp("off",z)==0
+   || sqlite3_stricmp("false",z)==0
+   || (z[0]=='0' && z[1]==0)
+  ){
+    return 0;
+  }
+  return -1;
+}
+#endif
+
+/*
+** The vtablogConnect() method is invoked to create a new
+** vtablog_vtab that describes the vtablog virtual table.
+**
+** Think of this routine as the constructor for vtablog_vtab objects.
+**
+** All this routine needs to do is:
+**
+**    (1) Allocate the vtablog_vtab object and initialize all fields.
+**
+**    (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the
+**        result set of queries against vtablog will look like.
+*/
+static int vtablogConnectCreate(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr,
+  int isCreate
+){
+  static int nInst = 0;
+  vtablog_vtab *pNew;
+  int i;
+  int rc;
+  int iInst = ++nInst;
+  char *zSchema = 0;
+  char *zNRow = 0;
+
+  printf("vtablog%s(tab=%d):\n", isCreate ? "Create" : "Connect", iInst);
+  printf("  argc=%d\n", argc);
+  for(i=0; i<argc; i++){
+    printf("  argv[%d] = ", i);
+    if( argv[i] ){
+      printf("[%s]\n", argv[i]);
+    }else{
+      printf("NULL\n");
+    }
+  }
+
+  for(i=3; i<argc; i++){
+    const char *z = argv[i];
+    if( vtablog_string_parameter(pzErr, "schema", z, &zSchema) ){
+      return SQLITE_ERROR;
+    }
+    if( vtablog_string_parameter(pzErr, "rows", z, &zNRow) ){
+      return SQLITE_ERROR;
+    }
+  }
+
+  if( zSchema==0 ){
+    *pzErr = sqlite3_mprintf("no schema defined");
+    return SQLITE_ERROR;
+  }
+  rc = sqlite3_declare_vtab(db, zSchema);
+  if( rc==SQLITE_OK ){
+    pNew = sqlite3_malloc( sizeof(*pNew) );
+    *ppVtab = (sqlite3_vtab*)pNew;
+    if( pNew==0 ) return SQLITE_NOMEM;
+    memset(pNew, 0, sizeof(*pNew));
+    pNew->nRow = 10;
+    if( zNRow ) pNew->nRow = atoi(zNRow);
+    pNew->iInst = iInst;
+  }
+  return rc;
+}
+static int vtablogCreate(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
+){
+  return vtablogConnectCreate(db,pAux,argc,argv,ppVtab,pzErr,1);
+}
+static int vtablogConnect(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
+){
+  return vtablogConnectCreate(db,pAux,argc,argv,ppVtab,pzErr,0);
+}
+
+
+/*
+** This method is the destructor for vtablog_cursor objects.
+*/
+static int vtablogDisconnect(sqlite3_vtab *pVtab){
+  vtablog_vtab *pTab = (vtablog_vtab*)pVtab;
+  printf("vtablogDisconnect(%d)\n", pTab->iInst);
+  sqlite3_free(pVtab);
+  return SQLITE_OK;
+}
+
+/*
+** This method is the destructor for vtablog_cursor objects.
+*/
+static int vtablogDestroy(sqlite3_vtab *pVtab){
+  vtablog_vtab *pTab = (vtablog_vtab*)pVtab;
+  printf("vtablogDestroy(%d)\n", pTab->iInst);
+  sqlite3_free(pVtab);
+  return SQLITE_OK;
+}
+
+/*
+** Constructor for a new vtablog_cursor object.
+*/
+static int vtablogOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
+  vtablog_vtab *pTab = (vtablog_vtab*)p;
+  vtablog_cursor *pCur;
+  printf("vtablogOpen(tab=%d, cursor=%d)\n", pTab->iInst, ++pTab->nCursor);
+  pCur = sqlite3_malloc( sizeof(*pCur) );
+  if( pCur==0 ) return SQLITE_NOMEM;
+  memset(pCur, 0, sizeof(*pCur));
+  pCur->iCursor = pTab->nCursor;
+  *ppCursor = &pCur->base;
+  return SQLITE_OK;
+}
+
+/*
+** Destructor for a vtablog_cursor.
+*/
+static int vtablogClose(sqlite3_vtab_cursor *cur){
+  vtablog_cursor *pCur = (vtablog_cursor*)cur;
+  vtablog_vtab *pTab = (vtablog_vtab*)cur->pVtab;
+  printf("vtablogClose(tab=%d, cursor=%d)\n", pTab->iInst, pCur->iCursor);
+  sqlite3_free(cur);
+  return SQLITE_OK;
+}
+
+
+/*
+** Advance a vtablog_cursor to its next row of output.
+*/
+static int vtablogNext(sqlite3_vtab_cursor *cur){
+  vtablog_cursor *pCur = (vtablog_cursor*)cur;
+  vtablog_vtab *pTab = (vtablog_vtab*)cur->pVtab;
+  printf("vtablogNext(tab=%d, cursor=%d)  rowid %d -> %d\n", 
+         pTab->iInst, pCur->iCursor, (int)pCur->iRowid, (int)pCur->iRowid+1);
+  pCur->iRowid++;
+  return SQLITE_OK;
+}
+
+/*
+** Return values of columns for the row at which the vtablog_cursor
+** is currently pointing.
+*/
+static int vtablogColumn(
+  sqlite3_vtab_cursor *cur,   /* The cursor */
+  sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
+  int i                       /* Which column to return */
+){
+  vtablog_cursor *pCur = (vtablog_cursor*)cur;
+  vtablog_vtab *pTab = (vtablog_vtab*)cur->pVtab;
+  char zVal[50];
+
+  if( i<26 ){
+    sqlite3_snprintf(sizeof(zVal),zVal,"%c%d", 
+                     "abcdefghijklmnopqrstuvwyz"[i], pCur->iRowid);
+  }else{
+    sqlite3_snprintf(sizeof(zVal),zVal,"{%d}%d", i, pCur->iRowid);
+  }
+  printf("vtablogColumn(tab=%d, cursor=%d, i=%d): [%s]\n",
+         pTab->iInst, pCur->iCursor, i, zVal);
+  sqlite3_result_text(ctx, zVal, -1, SQLITE_TRANSIENT);
+  return SQLITE_OK;
+}
+
+/*
+** Return the rowid for the current row.  In this implementation, the
+** rowid is the same as the output value.
+*/
+static int vtablogRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
+  vtablog_cursor *pCur = (vtablog_cursor*)cur;
+  vtablog_vtab *pTab = (vtablog_vtab*)cur->pVtab;
+  printf("vtablogRowid(tab=%d, cursor=%d): %d\n",
+         pTab->iInst, pCur->iCursor, (int)pCur->iRowid);
+  *pRowid = pCur->iRowid;
+  return SQLITE_OK;
+}
+
+/*
+** Return TRUE if the cursor has been moved off of the last
+** row of output.
+*/
+static int vtablogEof(sqlite3_vtab_cursor *cur){
+  vtablog_cursor *pCur = (vtablog_cursor*)cur;
+  vtablog_vtab *pTab = (vtablog_vtab*)cur->pVtab;
+  int rc = pCur->iRowid >= pTab->nRow;
+  printf("vtablogEof(tab=%d, cursor=%d): %d\n",
+         pTab->iInst, pCur->iCursor, rc);
+  return rc;
+}
+
+/*
+** Output an sqlite3_value object's value as an SQL literal.
+*/
+static void vtablogQuote(sqlite3_value *p){
+  char z[50];
+  switch( sqlite3_value_type(p) ){
+    case SQLITE_NULL: {
+      printf("NULL");
+      break;
+    }
+    case SQLITE_INTEGER: {
+      sqlite3_snprintf(50,z,"%lld", sqlite3_value_int64(p));
+      printf("%s", z);
+      break;
+    }
+    case SQLITE_FLOAT: {
+      sqlite3_snprintf(50,z,"%!.20g", sqlite3_value_double(p));
+      printf("%s", z);
+      break;
+    }
+    case SQLITE_BLOB: {
+      int n = sqlite3_value_bytes(p);
+      const unsigned char *z = (const unsigned char*)sqlite3_value_blob(p);
+      int i;
+      printf("x'");
+      for(i=0; i<n; i++) printf("%02x", z[i]);
+      printf("'");
+      break;
+    }
+    case SQLITE_TEXT: {
+      const char *z = (const char*)sqlite3_value_text(p);
+      int i;
+      char c;
+      for(i=0; (c = z[i])!=0 && c!='\''; i++){}
+      if( c==0 ){
+        printf("'%s'",z);
+      }else{
+        printf("'");
+        while( *z ){
+          for(i=0; (c = z[i])!=0 && c!='\''; i++){}
+          if( c=='\'' ) i++;
+          if( i ){
+            printf("%.*s", i, z);
+            z += i;
+          }
+          if( c=='\'' ){
+            printf("'");
+            continue;
+          }
+          if( c==0 ){
+            break;
+          }
+          z++;
+        }
+        printf("'");
+      }
+      break;
+    }
+  }
+}
+
+
+/*
+** This method is called to "rewind" the vtablog_cursor object back
+** to the first row of output.  This method is always called at least
+** once prior to any call to vtablogColumn() or vtablogRowid() or 
+** vtablogEof().
+*/
+static int vtablogFilter(
+  sqlite3_vtab_cursor *cur,
+  int idxNum, const char *idxStr,
+  int argc, sqlite3_value **argv
+){
+  vtablog_cursor *pCur = (vtablog_cursor *)cur;
+  vtablog_vtab *pTab = (vtablog_vtab*)cur->pVtab;
+  printf("vtablogFilter(tab=%d, cursor=%d):\n", pTab->iInst, pCur->iCursor);
+  pCur->iRowid = 0;
+  return SQLITE_OK;
+}
+
+/*
+** SQLite will invoke this method one or more times while planning a query
+** that uses the vtablog virtual table.  This routine needs to create
+** a query plan for each invocation and compute an estimated cost for that
+** plan.
+*/
+static int vtablogBestIndex(
+  sqlite3_vtab *tab,
+  sqlite3_index_info *pIdxInfo
+){
+  vtablog_vtab *pTab = (vtablog_vtab*)tab;
+  printf("vtablogBestIndex(tab=%d):\n", pTab->iInst);
+  pIdxInfo->estimatedCost = (double)500;
+  pIdxInfo->estimatedRows = 500;
+  return SQLITE_OK;
+}
+
+/*
+** SQLite invokes this method to INSERT, UPDATE, or DELETE content from
+** the table. 
+**
+** This implementation does not actually make any changes to the table
+** content.  It merely logs the fact that the method was invoked
+*/
+static int vtablogUpdate(
+  sqlite3_vtab *tab,
+  int argc,
+  sqlite3_value **argv,
+  sqlite_int64 *pRowid
+){
+  vtablog_vtab *pTab = (vtablog_vtab*)tab;
+  int i;
+  printf("vtablogUpdate(tab=%d):\n", pTab->iInst);
+  printf("  argc=%d\n", argc);
+  for(i=0; i<argc; i++){
+    printf("  argv[%d]=", i);
+    vtablogQuote(argv[i]);
+    printf("\n");
+  }
+  return SQLITE_OK;
+}
+
+/*
+** This following structure defines all the methods for the 
+** vtablog virtual table.
+*/
+static sqlite3_module vtablogModule = {
+  0,                         /* iVersion */
+  vtablogCreate,             /* xCreate */
+  vtablogConnect,            /* xConnect */
+  vtablogBestIndex,          /* xBestIndex */
+  vtablogDisconnect,         /* xDisconnect */
+  vtablogDestroy,            /* xDestroy */
+  vtablogOpen,               /* xOpen - open a cursor */
+  vtablogClose,              /* xClose - close a cursor */
+  vtablogFilter,             /* xFilter - configure scan constraints */
+  vtablogNext,               /* xNext - advance a cursor */
+  vtablogEof,                /* xEof - check for end of scan */
+  vtablogColumn,             /* xColumn - read data */
+  vtablogRowid,              /* xRowid - read data */
+  vtablogUpdate,             /* xUpdate */
+  0,                         /* xBegin */
+  0,                         /* xSync */
+  0,                         /* xCommit */
+  0,                         /* xRollback */
+  0,                         /* xFindMethod */
+  0,                         /* xRename */
+  0,                         /* xSavepoint */
+  0,                         /* xRelease */
+  0,                         /* xRollbackTo */
+};
+
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_vtablog_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  int rc;
+  SQLITE_EXTENSION_INIT2(pApi);
+  rc = sqlite3_create_module(db, "vtablog", &vtablogModule, 0);
+  return rc;
+}

ext/misc/zorder.c

@@ -0,0 +1,102 @@
+/*
+** 2018-02-09
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** SQL functions for z-order (Morton code) transformations.
+**
+**      zorder(X0,X0,..,xN)      Generate an N+1 dimension Morton code
+**
+**      unzorder(Z,N,I)          Extract the I-th dimension from N-dimensional
+**                               Morton code Z.
+*/
+#include "sqlite3ext.h"
+SQLITE_EXTENSION_INIT1
+#include <assert.h>
+#include <string.h>
+
+/*
+** Functions:     zorder(X0,X1,....)
+**
+** Convert integers X0, X1, ... into morton code.
+**
+** The output is a signed 64-bit integer.  If any argument is too large,
+** an error is thrown.
+*/
+static void zorderFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  sqlite3_int64 z, x[63];
+  int i, j;
+  z = 0;
+  for(i=0; i<argc; i++){
+    x[i] = sqlite3_value_int64(argv[i]);
+  }
+  if( argc>0 ){
+    for(i=0; i<63; i++){
+      j = i%argc;
+      z |= (x[j]&1)<<i;
+      x[j] >>= 1;
+    }
+  }
+  sqlite3_result_int64(context, z);
+  for(i=0; i<argc; i++){
+    if( x[i] ){
+      sqlite3_result_error(context, "parameter too large", -1);
+    }
+  }
+}
+
+
+/*
+** Functions:     unzorder(Z,N,I)
+**
+** Assuming that Z is an N-dimensional Morton code, extract the I-th
+** dimension.
+*/
+static void unzorderFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  sqlite3_int64 z, n, i, x;
+  int j, k;
+  z = sqlite3_value_int64(argv[0]);
+  n = sqlite3_value_int64(argv[1]);
+  i = sqlite3_value_int64(argv[2]);
+  x = 0;
+  for(k=0, j=i; j<63; j+=n, k++){
+    x |= ((z>>j)&1)<<k;
+  }
+  sqlite3_result_int64(context, x);
+}
+
+
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_zorder_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  int rc = SQLITE_OK;
+  SQLITE_EXTENSION_INIT2(pApi);
+  (void)pzErrMsg;  /* Unused parameter */
+  rc = sqlite3_create_function(db, "zorder", -1, SQLITE_UTF8, 0,
+                               zorderFunc, 0, 0);
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_create_function(db, "unzorder", 3, SQLITE_UTF8, 0,
+                               unzorderFunc, 0, 0);
+  }
+  return rc;
+}

ext/rbu/rbu.c

@@ -29,7 +29,9 @@ void usage(const char *zArgv0){
 "Where options are:\n"
 "\n"
 "    -step NSTEP\n"
+"    -statstep NSTATSTEP\n"
 "    -vacuum\n"
+"    -presql SQL\n"
 "\n"
 "  If the -vacuum switch is not present, argument RBU-DB must be an RBU\n"
 "  database containing an update suitable for target database TARGET-DB.\n"
@@ -76,11 +78,13 @@ int main(int argc, char **argv){
   const char *zTarget;            /* Target database to apply RBU to */
   const char *zRbu;               /* Database containing RBU */
   char zBuf[200];                 /* Buffer for printf() */
-  char *zErrmsg;                  /* Error message, if any */
+  char *zErrmsg = 0;              /* Error message, if any */
   sqlite3rbu *pRbu;               /* RBU handle */
   int nStep = 0;                  /* Maximum number of step() calls */
+  int nStatStep = 0;              /* Report stats after this many step calls */
   int bVacuum = 0;
-  int rc;
+  const char *zPreSql = 0;
+  int rc = SQLITE_OK;
   sqlite3_int64 nProgress = 0;
   int nArgc = argc-2;
 
@@ -90,9 +94,18 @@ int main(int argc, char **argv){
     int nArg = strlen(zArg);
     if( nArg>1 && nArg<=8 && 0==memcmp(zArg, "-vacuum", nArg) ){
       bVacuum = 1;
+    }else if( nArg>1 && nArg<=7 
+           && 0==memcmp(zArg, "-presql", nArg) && i<nArg-1 ){
+      i++;
+      zPreSql = argv[i];
     }else if( nArg>1 && nArg<=5 && 0==memcmp(zArg, "-step", nArg) && i<nArg-1 ){
       i++;
       nStep = atoi(argv[i]);
+    }else if( nArg>1 && nArg<=9 
+           && 0==memcmp(zArg, "-statstep", nArg) && i<nArg-1 
+    ){
+      i++;
+      nStatStep = atoi(argv[i]);
     }else{
       usage(argv[0]);
     }
@@ -112,13 +125,40 @@ int main(int argc, char **argv){
   }
   report_rbu_vfs(pRbu);
 
+  if( zPreSql && pRbu ){
+    sqlite3 *dbMain = sqlite3rbu_db(pRbu, 0);
+    rc = sqlite3_exec(dbMain, zPreSql, 0, 0, 0);
+    if( rc==SQLITE_OK ){
+      sqlite3 *dbRbu = sqlite3rbu_db(pRbu, 1);
+      rc = sqlite3_exec(dbRbu, zPreSql, 0, 0, 0);
+    }
+  }
+
   /* If nStep is less than or equal to zero, call
   ** sqlite3rbu_step() until either the RBU has been completely applied
   ** or an error occurs. Or, if nStep is greater than zero, call
   ** sqlite3rbu_step() a maximum of nStep times.  */
-  for(i=0; (nStep<=0 || i<nStep) && sqlite3rbu_step(pRbu)==SQLITE_OK; i++);
+  if( rc==SQLITE_OK ){
-  nProgress = sqlite3rbu_progress(pRbu);
+    for(i=0; (nStep<=0 || i<nStep) && sqlite3rbu_step(pRbu)==SQLITE_OK; i++){
-  rc = sqlite3rbu_close(pRbu, &zErrmsg);
+      if( nStatStep>0 && (i % nStatStep)==0 ){
+        sqlite3_int64 nUsed;
+        sqlite3_int64 nHighwater;
+        sqlite3_status64(SQLITE_STATUS_MEMORY_USED, &nUsed, &nHighwater, 0);
+        fprintf(stdout, "memory used=%lld highwater=%lld", nUsed, nHighwater);
+        if( bVacuum==0 ){
+          int one;
+          int two;
+          sqlite3rbu_bp_progress(pRbu, &one, &two);
+          fprintf(stdout, "  progress=%d/%d\n", one, two);
+        }else{
+          fprintf(stdout, "\n");
+        }
+        fflush(stdout);
+      }
+    }
+    nProgress = sqlite3rbu_progress(pRbu);
+    rc = sqlite3rbu_close(pRbu, &zErrmsg);
+  }
 
   /* Let the user know what happened. */
   switch( rc ){

ext/rbu/rbu1.test

@@ -139,6 +139,7 @@ foreach {tn3 create_vfs destroy_vfs} {
   foreach {tn2 cmd} {
       1 run_rbu 
       2 step_rbu 3 step_rbu_uri 4 step_rbu_state
+      5 step_rbu_legacy
   } {
     foreach {tn schema} {
       1 {

ext/rbu/rbu_common.tcl

@@ -70,7 +70,24 @@ proc step_rbu {target rbu} {
   set rc
 }
 
+proc step_rbu_legacy {target rbu} {
+  while 1 {
+    sqlite3rbu rbu $target $rbu
+    set state [rbu state]
+    check_prestep_state $target $state
+    set rc [rbu step]
+    check_poststep_state $rc $target $state
+    rbu close
+    if {$rc != "SQLITE_OK"} break
+    sqlite3 tmpdb $rbu
+    tmpdb eval { DELETE FROM rbu_state WHERE k==10 }
+    tmpdb close
+  }
+  set rc
+}
+
 proc do_rbu_vacuum_test {tn step} {
+  forcedelete state.db
   uplevel [list do_test $tn.1 {
     if {$step==0} { sqlite3rbu_vacuum rbu test.db state.db }
     while 1 {

ext/rbu/rbucollate.test

@@ -0,0 +1,63 @@
+# 2018 March 22
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+source [file join [file dirname [info script]] rbu_common.tcl]
+set ::testprefix rbucollate
+
+ifcapable !icu_collations {
+  finish_test
+  return
+}
+
+db close
+sqlite3_shutdown
+sqlite3_config_uri 1
+reset_db
+ 
+# Create a simple RBU database. That expects to write to a table:
+#
+#   CREATE TABLE t1(a PRIMARY KEY, b, c);
+#
+proc create_rbu1 {filename} {
+  forcedelete $filename
+  sqlite3 rbu1 $filename  
+  rbu1 eval {
+    CREATE TABLE data_t1(a, b, c, rbu_control);
+    INSERT INTO data_t1 VALUES('a', 'one', 1, 0);
+    INSERT INTO data_t1 VALUES('b', 'two', 2, 0);
+    INSERT INTO data_t1 VALUES('c', 'three', 3, 0);
+  }
+  rbu1 close
+  return $filename
+}
+
+do_execsql_test 1.0 {
+  SELECT icu_load_collation('en_US', 'my-collate');
+  CREATE TABLE t1(a COLLATE "my-collate" PRIMARY KEY, b, c);
+} {{}}
+
+do_test 1.2 {
+  create_rbu1 testrbu.db
+  sqlite3rbu rbu test.db testrbu.db
+  rbu dbMain_eval { SELECT icu_load_collation('en_US', 'my-collate') }
+  rbu dbRbu_eval { SELECT icu_load_collation('en_US', 'my-collate') }
+  while 1 {
+    set rc [rbu step]
+    if {$rc!="SQLITE_OK"} break
+  }
+  rbu close
+  db eval { SELECT * FROM t1 }
+} {a one 1 b two 2 c three 3}
+
+#forcedelete testrbu.db
+finish_test
+