Sync this branch with the latest trunk.

FossilOrigin-Name: 7a44fa5a350a3f19b8e9f5196d22535788885f8c0e849572202bf64a055ddc2d

.fossil-settings/empty-dirs

@@ -0,0 +1 @@
+compat

.fossil-settings/ignore-glob

@@ -0,0 +1 @@
+compat/*

Makefile.in

@@ -22,7 +22,7 @@ TOP = @abs_srcdir@
 #
 BCC = @BUILD_CC@ @BUILD_CFLAGS@
 
-# TCC is the C Compile and options for use in building executables that 
+# TCC is the C Compile and options for use in building executables that
 # will run on the target platform.  (BCC and TCC are usually the
 # same unless your are cross-compiling.)  Separate CC and CFLAGS macros
 # are provide so that these aspects of the build process can be changed
@@ -32,10 +32,11 @@ CC = @CC@
 CFLAGS = @CPPFLAGS@ @CFLAGS@
 TCC = ${CC} ${CFLAGS} -I. -I${TOP}/src -I${TOP}/ext/rtree -I${TOP}/ext/icu
 TCC += -I${TOP}/ext/fts3 -I${TOP}/ext/async -I${TOP}/ext/session
+TCC += -I${TOP}/ext/userauth
 
 # Define this for the autoconf-based build, so that the code knows it can
 # include the generated config.h
-# 
+#
 TCC += -D_HAVE_SQLITE_CONFIG_H -DBUILD_sqlite
 
 # Define -DNDEBUG to compile without debugging (i.e., for production usage)
@@ -66,7 +67,7 @@ LIBREADLINE = @TARGET_READLINE_LIBS@
 TCC += -DSQLITE_THREADSAFE=@SQLITE_THREADSAFE@
 
 # Any target libraries which libsqlite must be linked against
-# 
+#
 TLIBS = @LIBS@ $(LIBS)
 
 # Flags controlling use of the in memory btree implementation
@@ -78,8 +79,8 @@ TLIBS = @LIBS@ $(LIBS)
 TEMP_STORE = -DSQLITE_TEMP_STORE=@TEMP_STORE@
 
 # Enable/disable loadable extensions, and other optional features
-# based on configuration. (-DSQLITE_OMIT*, -DSQLITE_ENABLE*).  
+# based on configuration. (-DSQLITE_OMIT*, -DSQLITE_ENABLE*).
-# The same set of OMIT and ENABLE flags should be passed to the 
+# The same set of OMIT and ENABLE flags should be passed to the
 # LEMON parser generator and the mkkeywordhash tool as well.
 OPT_FEATURE_FLAGS = @OPT_FEATURE_FLAGS@
 
@@ -89,6 +90,9 @@ TCC += $(OPT_FEATURE_FLAGS)
 # ie.  make "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1".
 TCC += $(OPTS)
 
+# Add in compile-time options for some libraries used by extensions
+TCC += @HAVE_ZLIB@
+
 # Version numbers and release number for the SQLite being compiled.
 #
 VERSION = @VERSION@
@@ -123,8 +127,8 @@ SHLIB_SUFFIX = @TCL_SHLIB_SUFFIX@
 # If gcov support was enabled by the configure script, add the appropriate
 # flags here.  It's not always as easy as just having the user add the right
 # CFLAGS / LDFLAGS, because libtool wants to use CFLAGS when linking, which
-# causes build errors with -fprofile-arcs -ftest-coverage with some GCCs.  
+# causes build errors with -fprofile-arcs -ftest-coverage with some GCCs.
-# Supposedly GCC does the right thing if you use --coverage, but in 
+# Supposedly GCC does the right thing if you use --coverage, but in
 # practice it still fails.  See:
 #
 # http://www.mail-archive.com/debian-gcc@lists.debian.org/msg26197.html
@@ -166,7 +170,8 @@ USE_AMALGAMATION = @USE_AMALGAMATION@
 #
 LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \
          backup.lo bitvec.lo btmutex.lo btree.lo build.lo \
-         callback.lo complete.lo ctime.lo date.lo dbstat.lo delete.lo \
+         callback.lo complete.lo ctime.lo \
+         date.lo dbpage.lo dbstat.lo delete.lo \
          expr.lo fault.lo fkey.lo \
          fts3.lo fts3_aux.lo fts3_expr.lo fts3_hash.lo fts3_icu.lo \
          fts3_porter.lo fts3_snippet.lo fts3_tokenizer.lo fts3_tokenizer1.lo \
@@ -176,17 +181,17 @@ LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \
          func.lo global.lo hash.lo \
          icu.lo insert.lo json1.lo legacy.lo loadext.lo \
          main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \
-         memjournal.lo \
+         memdb.lo memjournal.lo \
          mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \
          notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \
          pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
          random.lo resolve.lo rowset.lo rtree.lo \
          sqlite3session.lo select.lo sqlite3rbu.lo status.lo stmt.lo \
          table.lo threads.lo tokenize.lo treeview.lo trigger.lo \
-         update.lo util.lo vacuum.lo \
+         update.lo userauth.lo upsert.lo util.lo vacuum.lo \
          vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
          vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \
-         utf.lo vtab.lo
+         window.lo utf.lo vtab.lo
 
 # Object files for the amalgamation.
 #
@@ -215,6 +220,7 @@ SRC = \
   $(TOP)/src/complete.c \
   $(TOP)/src/ctime.c \
   $(TOP)/src/date.c \
+  $(TOP)/src/dbpage.c \
   $(TOP)/src/dbstat.c \
   $(TOP)/src/delete.c \
   $(TOP)/src/expr.c \
@@ -235,6 +241,7 @@ SRC = \
   $(TOP)/src/mem2.c \
   $(TOP)/src/mem3.c \
   $(TOP)/src/mem5.c \
+  $(TOP)/src/memdb.c \
   $(TOP)/src/memjournal.c \
   $(TOP)/src/msvc.h \
   $(TOP)/src/mutex.c \
@@ -265,7 +272,7 @@ SRC = \
   $(TOP)/src/rowset.c \
   $(TOP)/src/select.c \
   $(TOP)/src/status.c \
-  $(TOP)/src/shell.c \
+  $(TOP)/src/shell.c.in \
   $(TOP)/src/sqlite.h.in \
   $(TOP)/src/sqlite3ext.h \
   $(TOP)/src/sqliteInt.h \
@@ -278,6 +285,7 @@ SRC = \
   $(TOP)/src/trigger.c \
   $(TOP)/src/utf.c \
   $(TOP)/src/update.c \
+  $(TOP)/src/upsert.c \
   $(TOP)/src/util.c \
   $(TOP)/src/vacuum.c \
   $(TOP)/src/vdbe.c \
@@ -297,7 +305,8 @@ SRC = \
   $(TOP)/src/where.c \
   $(TOP)/src/wherecode.c \
   $(TOP)/src/whereexpr.c \
-  $(TOP)/src/whereInt.h
+  $(TOP)/src/whereInt.h \
+  $(TOP)/src/window.c
 
 # Source code for extensions
 #
@@ -342,10 +351,14 @@ SRC += \
   $(TOP)/ext/icu/icu.c
 SRC += \
   $(TOP)/ext/rtree/rtree.h \
-  $(TOP)/ext/rtree/rtree.c
+  $(TOP)/ext/rtree/rtree.c \
+  $(TOP)/ext/rtree/geopoly.c
 SRC += \
   $(TOP)/ext/session/sqlite3session.c \
   $(TOP)/ext/session/sqlite3session.h
+SRC += \
+  $(TOP)/ext/userauth/userauth.c \
+  $(TOP)/ext/userauth/sqlite3userauth.h
 SRC += \
   $(TOP)/ext/rbu/sqlite3rbu.h \
   $(TOP)/ext/rbu/sqlite3rbu.c
@@ -362,6 +375,7 @@ SRC += \
   parse.c \
   parse.h \
   config.h \
+  shell.c \
   sqlite3.h
 
 # Source code to the test files.
@@ -393,6 +407,7 @@ TESTSRC = \
   $(TOP)/src/test_intarray.c \
   $(TOP)/src/test_journal.c \
   $(TOP)/src/test_malloc.c \
+  $(TOP)/src/test_md5.c \
   $(TOP)/src/test_multiplex.c \
   $(TOP)/src/test_mutex.c \
   $(TOP)/src/test_onefile.c \
@@ -404,24 +419,29 @@ TESTSRC = \
   $(TOP)/src/test_server.c \
   $(TOP)/src/test_superlock.c \
   $(TOP)/src/test_syscall.c \
+  $(TOP)/src/test_tclsh.c \
   $(TOP)/src/test_tclvar.c \
   $(TOP)/src/test_thread.c \
   $(TOP)/src/test_vfs.c \
   $(TOP)/src/test_windirent.c \
+  $(TOP)/src/test_window.c \
   $(TOP)/src/test_wsd.c       \
   $(TOP)/ext/fts3/fts3_term.c \
   $(TOP)/ext/fts3/fts3_test.c  \
   $(TOP)/ext/session/test_session.c \
-  $(TOP)/ext/rbu/test_rbu.c 
+  $(TOP)/ext/rbu/test_rbu.c
 
 # Statically linked extensions
 #
 TESTSRC += \
+  $(TOP)/ext/expert/sqlite3expert.c \
+  $(TOP)/ext/expert/test_expert.c \
   $(TOP)/ext/misc/amatch.c \
   $(TOP)/ext/misc/carray.c \
   $(TOP)/ext/misc/closure.c \
   $(TOP)/ext/misc/csv.c \
   $(TOP)/ext/misc/eval.c \
+  $(TOP)/ext/misc/explain.c \
   $(TOP)/ext/misc/fileio.c \
   $(TOP)/ext/misc/fuzzer.c \
   $(TOP)/ext/fts5/fts5_tcl.c \
@@ -430,6 +450,7 @@ TESTSRC += \
   $(TOP)/ext/misc/ieee754.c \
   $(TOP)/ext/misc/mmapwarm.c \
   $(TOP)/ext/misc/nextchar.c \
+  $(TOP)/ext/misc/normalize.c \
   $(TOP)/ext/misc/percentile.c \
   $(TOP)/ext/misc/regexp.c \
   $(TOP)/ext/misc/remember.c \
@@ -437,7 +458,9 @@ TESTSRC += \
   $(TOP)/ext/misc/spellfix.c \
   $(TOP)/ext/misc/totype.c \
   $(TOP)/ext/misc/unionvtab.c \
-  $(TOP)/ext/misc/wholenumber.c
+  $(TOP)/ext/misc/wholenumber.c \
+  $(TOP)/ext/misc/zipfile.c \
+  $(TOP)/ext/userauth/userauth.c
 
 # Source code to the library files needed by the test fixture
 #
@@ -449,9 +472,11 @@ TESTSRC2 = \
   $(TOP)/src/build.c \
   $(TOP)/src/ctime.c \
   $(TOP)/src/date.c \
+  $(TOP)/src/dbpage.c \
   $(TOP)/src/dbstat.c \
   $(TOP)/src/expr.c \
   $(TOP)/src/func.c \
+  $(TOP)/src/global.c \
   $(TOP)/src/insert.c \
   $(TOP)/src/wal.c \
   $(TOP)/src/main.c \
@@ -478,6 +503,7 @@ TESTSRC2 = \
   $(TOP)/src/where.c \
   $(TOP)/src/wherecode.c \
   $(TOP)/src/whereexpr.c \
+  $(TOP)/src/window.c \
   parse.c \
   $(TOP)/ext/fts3/fts3.c \
   $(TOP)/ext/fts3/fts3_aux.c \
@@ -487,7 +513,7 @@ TESTSRC2 = \
   $(TOP)/ext/fts3/fts3_write.c \
   $(TOP)/ext/async/sqlite3async.c \
   $(TOP)/ext/session/sqlite3session.c \
-  $(TOP)/ext/misc/stmt.c 
+  $(TOP)/ext/misc/stmt.c
 
 # Header files used by all library source files.
 #
@@ -534,11 +560,14 @@ EXTHDR += \
   $(TOP)/ext/fts3/fts3_hash.h \
   $(TOP)/ext/fts3/fts3_tokenizer.h
 EXTHDR += \
-  $(TOP)/ext/rtree/rtree.h
+  $(TOP)/ext/rtree/rtree.h \
+  $(TOP)/ext/rtree/geopoly.c
 EXTHDR += \
   $(TOP)/ext/icu/sqliteicu.h
 EXTHDR += \
   $(TOP)/ext/rtree/sqlite3rtree.h
+EXTHDR += \
+  $(TOP)/ext/userauth/sqlite3userauth.h
 
 # executables needed for testing
 #
@@ -547,7 +576,8 @@ TESTPROGS = \
   sqlite3$(TEXE) \
   sqlite3_analyzer$(TEXE) \
   sqldiff$(TEXE) \
-  dbhash$(TEXE)
+  dbhash$(TEXE) \
+  sqltclsh$(TEXE)
 
 # Databases containing fuzzer test cases
 #
@@ -556,7 +586,9 @@ FUZZDATA = \
   $(TOP)/test/fuzzdata2.db \
   $(TOP)/test/fuzzdata3.db \
   $(TOP)/test/fuzzdata4.db \
-  $(TOP)/test/fuzzdata5.db
+  $(TOP)/test/fuzzdata5.db \
+  $(TOP)/test/fuzzdata6.db \
+  $(TOP)/test/fuzzdata7.db
 
 # Standard options to testfixture
 #
@@ -565,15 +597,22 @@ TESTOPTS = --verbose=file --output=test-out.txt
 # Extra compiler options for various shell tools
 #
 SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4
-# SHELL_OPT += -DSQLITE_ENABLE_FTS5
+#SHELL_OPT += -DSQLITE_ENABLE_FTS5
+SHELL_OPT += -DSQLITE_ENABLE_RTREE
 SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS
 SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
 SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB
+SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB
+SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
+SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC
+SHELL_OPT += -DSQLITE_ENABLE_DESERIALIZE
+SHELL_OPT += -DSQLITE_INTROSPECTION_PRAGMAS
 FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
 FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ
 FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
+FUZZCHECK_OPT += -DSQLITE_PRINTF_PRECISION_LIMIT=1000
 FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c
-DBFUZZ_OPT = 
+DBFUZZ_OPT =
 
 # This is the default Makefile target.  The objects listed here
 # are what get build when you type just "make" with no arguments.
@@ -597,9 +636,9 @@ libtclsqlite3.la:	tclsqlite.lo libsqlite3.la
 		-version-info "8:6:8" \
 		-avoid-version
 
-sqlite3$(TEXE):	$(TOP)/src/shell.c sqlite3.c
+sqlite3$(TEXE):	shell.c sqlite3.c
 	$(LTLINK) $(READLINE_FLAGS) $(SHELL_OPT) -o $@ \
-		$(TOP)/src/shell.c sqlite3.c \
+		shell.c sqlite3.c \
 		$(LIBREADLINE) $(TLIBS) -rpath "$(libdir)"
 
 sqldiff$(TEXE):	$(TOP)/tool/sqldiff.c sqlite3.lo sqlite3.h
@@ -629,9 +668,28 @@ ossshell$(TEXE):	$(TOP)/test/ossfuzz.c $(TOP)/test/ossshell.c sqlite3.c sqlite3.
 	$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/ossshell.c \
              $(TOP)/test/ossfuzz.c sqlite3.c $(TLIBS)
 
+sessionfuzz$(TEXE):	$(TOP)/test/sessionfuzz.c sqlite3.c sqlite3.h
+	$(CC) $(CFLAGS) -I. -o $@ $(TOP)/test/sessionfuzz.c $(TLIBS)
+
 dbfuzz$(TEXE):	$(TOP)/test/dbfuzz.c sqlite3.c sqlite3.h
 	$(LTLINK) -o $@ $(DBFUZZ_OPT) $(TOP)/test/dbfuzz.c sqlite3.c $(TLIBS)
 
+DBFUZZ2_OPTS = \
+  -DSQLITE_THREADSAFE=0 \
+  -DSQLITE_OMIT_LOAD_EXTENSION \
+  -DSQLITE_ENABLE_DESERIALIZE \
+  -DSQLITE_DEBUG \
+  -DSQLITE_ENABLE_DBSTAT_VTAB \
+  -DSQLITE_ENABLE_RTREE \
+  -DSQLITE_ENABLE_FTS4 \
+  -DSQLITE_EANBLE_FTS5
+
+dbfuzz2:	$(TOP)/test/dbfuzz2.c sqlite3.c sqlite3.h
+	clang-6.0 -I. -g -O0 -fsanitize=fuzzer,undefined,address -o dbfuzz2 \
+		$(DBFUZZ2_OPTS) $(TOP)/test/dbfuzz2.c sqlite3.c
+	mkdir -p dbfuzz2-dir
+	cp $(TOP)/test/dbfuzz2-seed* dbfuzz2-dir
+
 mptester$(TEXE):	sqlite3.lo $(TOP)/mptest/mptest.c
 	$(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.lo \
 		$(TLIBS) -rpath "$(libdir)"
@@ -668,7 +726,7 @@ mptest:	mptester$(TEXE)
 
 sqlite3.c:	.target_source $(TOP)/tool/mksqlite3c.tcl
 	$(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl
-	cp tsrc/shell.c tsrc/sqlite3ext.h .
+	cp tsrc/sqlite3ext.h .
 	cp $(TOP)/ext/session/sqlite3session.h .
 
 sqlite3ext.h:	.target_source
@@ -752,6 +810,9 @@ ctime.lo:	$(TOP)/src/ctime.c $(HDR)
 date.lo:	$(TOP)/src/date.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/date.c
 
+dbpage.lo:	$(TOP)/src/dbpage.c $(HDR)
+	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/dbpage.c
+
 dbstat.lo:	$(TOP)/src/dbstat.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/dbstat.c
 
@@ -806,6 +867,9 @@ mem3.lo:	$(TOP)/src/mem3.c $(HDR)
 mem5.lo:	$(TOP)/src/mem5.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/mem5.c
 
+memdb.lo:	$(TOP)/src/memdb.c $(HDR)
+	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/memdb.c
+
 memjournal.lo:	$(TOP)/src/memjournal.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/memjournal.c
 
@@ -884,6 +948,9 @@ trigger.lo:	$(TOP)/src/trigger.c $(HDR)
 update.lo:	$(TOP)/src/update.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/update.c
 
+upsert.lo:	$(TOP)/src/upsert.c $(HDR)
+	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/upsert.c
+
 utf.lo:	$(TOP)/src/utf.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/utf.c
 
@@ -932,11 +999,14 @@ wherecode.lo:	$(TOP)/src/wherecode.c $(HDR)
 whereexpr.lo:	$(TOP)/src/whereexpr.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/whereexpr.c
 
+window.lo:	$(TOP)/src/window.c $(HDR)
+	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/window.c
+
 tclsqlite.lo:	$(TOP)/src/tclsqlite.c $(HDR)
 	$(LTCOMPILE) -DUSE_TCL_STUBS=1 -c $(TOP)/src/tclsqlite.c
 
 tclsqlite-shell.lo:	$(TOP)/src/tclsqlite.c $(HDR)
-	$(LTCOMPILE) -DTCLSH=1 -o $@ -c $(TOP)/src/tclsqlite.c
+	$(LTCOMPILE) -DTCLSH -o $@ -c $(TOP)/src/tclsqlite.c
 
 tclsqlite-stubs.lo:	$(TOP)/src/tclsqlite.c $(HDR)
 	$(LTCOMPILE) -DUSE_TCL_STUBS=1 -o $@ -c $(TOP)/src/tclsqlite.c
@@ -971,6 +1041,23 @@ keywordhash.h:	$(TOP)/tool/mkkeywordhash.c
 	$(BCC) -o mkkeywordhash$(BEXE) $(OPT_FEATURE_FLAGS) $(OPTS) $(TOP)/tool/mkkeywordhash.c
 	./mkkeywordhash$(BEXE) >keywordhash.h
 
+# Source files that go into making shell.c
+SHELL_SRC = \
+	$(TOP)/src/shell.c.in \
+        $(TOP)/ext/misc/appendvfs.c \
+	$(TOP)/ext/misc/shathree.c \
+	$(TOP)/ext/misc/fileio.c \
+	$(TOP)/ext/misc/completion.c \
+	$(TOP)/ext/misc/sqlar.c \
+	$(TOP)/ext/expert/sqlite3expert.c \
+	$(TOP)/ext/expert/sqlite3expert.h \
+	$(TOP)/ext/misc/zipfile.c \
+        $(TOP)/src/test_windirent.c
+
+shell.c:	$(SHELL_SRC) $(TOP)/tool/mkshellc.tcl
+	$(TCLSH_CMD) $(TOP)/tool/mkshellc.tcl >shell.c
+
+
 
 
 # Rules to build the extension objects.
@@ -1038,7 +1125,10 @@ fts3_write.lo:	$(TOP)/ext/fts3/fts3_write.c $(HDR) $(EXTHDR)
 rtree.lo:	$(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR)
 	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c
 
-sqlite3session.lo:	$(TOP)/ext/session/sqlite3session.c $(HDR) $(EXTHDR)
+sqlite3session.lo:	$(TOP)/ext/userauth/userauth.c $(HDR) $(EXTHDR)
+	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/userauth/userauth.c
+
+userauth.lo:	$(TOP)/ext/session/sqlite3session.c $(HDR) $(EXTHDR)
 	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/session/sqlite3session.c
 
 json1.lo:	$(TOP)/ext/misc/json1.c
@@ -1066,7 +1156,7 @@ FTS5_SRC = \
    $(TOP)/ext/fts5/fts5_varint.c \
    $(TOP)/ext/fts5/fts5_vocab.c  \
 
-fts5parse.c:	$(TOP)/ext/fts5/fts5parse.y lemon 
+fts5parse.c:	$(TOP)/ext/fts5/fts5parse.y lemon
 	cp $(TOP)/ext/fts5/fts5parse.y .
 	rm -f fts5parse.h
 	./lemon$(BEXE) $(OPTS) fts5parse.y
@@ -1091,12 +1181,14 @@ sqlite3rbu.lo:	$(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR)
 # necessary because the test fixture requires non-API symbols which are
 # hidden when the library is built via the amalgamation).
 #
-TESTFIXTURE_FLAGS  = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
+TESTFIXTURE_FLAGS  = -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
-TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE 
+TESTFIXTURE_FLAGS += -DTCLSH_INIT_PROC=sqlite3TestInit
+TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE
 TESTFIXTURE_FLAGS += -DBUILD_sqlite
 TESTFIXTURE_FLAGS += -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
 TESTFIXTURE_FLAGS += -DSQLITE_DEFAULT_PAGE_SIZE=1024
 TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_STMTVTAB
+TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_DBPAGE_VTAB
 
 TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.la
 TESTFIXTURE_SRC1 = sqlite3.c
@@ -1107,6 +1199,10 @@ testfixture$(TEXE):	$(TESTFIXTURE_SRC)
 	$(LTLINK) -DSQLITE_NO_SYNC=1 $(TEMP_STORE) $(TESTFIXTURE_FLAGS) \
 		-o $@ $(TESTFIXTURE_SRC) $(LIBTCL) $(TLIBS)
 
+coretestprogs:	$(TESTPROGS)
+
+testprogs:	coretestprogs srcck1$(BEXE) fuzzcheck$(TEXE) sessionfuzz$(TEXE)
+
 # A very detailed test running most or all test cases
 fulltest:	$(TESTPROGS) fuzztest
 	./testfixture$(TEXE) $(TOP)/test/all.test $(TESTOPTS)
@@ -1120,14 +1216,17 @@ fulltestonly:	$(TESTPROGS) fuzztest
 	./testfixture$(TEXE) $(TOP)/test/full.test
 
 # Fuzz testing
-fuzztest:	fuzzcheck$(TEXE) $(FUZZDATA)
+fuzztest:	fuzzcheck$(TEXE) $(FUZZDATA) sessionfuzz$(TEXE) $(TOP)/test/sessionfuzz-data1.db
 	./fuzzcheck$(TEXE) $(FUZZDATA)
+	./sessionfuzz$(TEXE) run $(TOP)/test/sessionfuzz-data1.db
 
-fastfuzztest:	fuzzcheck$(TEXE) $(FUZZDATA)
+fastfuzztest:	fuzzcheck$(TEXE) $(FUZZDATA) sessionfuzz$(TEXE) $(TOP)/test/sessionfuzz-data1.db
 	./fuzzcheck$(TEXE) --limit-mem 100M $(FUZZDATA)
+	./sessionfuzz$(TEXE) run $(TOP)/test/sessionfuzz-data1.db
 
-valgrindfuzz:	fuzzcheck$(TEXT) $(FUZZDATA)
+valgrindfuzz:	fuzzcheck$(TEXT) $(FUZZDATA) sessionfuzz$(TEXE) $(TOP)/test/sessionfuzz-data1.db
 	valgrind ./fuzzcheck$(TEXE) --cell-size-check --limit-mem 10M --timeout 600 $(FUZZDATA)
+	valgrind ./sessionfuzz$(TEXE) run $(TOP)/test/sessionfuzz-data1.db
 
 # The veryquick.test TCL tests.
 #
@@ -1157,18 +1256,37 @@ valgrindtest:	$(TESTPROGS) valgrindfuzz
 smoketest:	$(TESTPROGS) fuzzcheck$(TEXE)
 	./testfixture$(TEXE) $(TOP)/test/main.test $(TESTOPTS)
 
-sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl
+sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in
-	echo "#define TCLSH 2" > $@
+	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in >sqlite3_analyzer.c
-	echo "#define SQLITE_ENABLE_DBSTAT_VTAB 1" >> $@
-	cat sqlite3.c $(TOP)/src/tclsqlite.c >> $@
-	echo "static const char *tclsh_main_loop(void){" >> $@
-	echo "static const char *zMainloop = " >> $@
-	$(TCLSH_CMD) $(TOP)/tool/tostr.tcl $(TOP)/tool/spaceanal.tcl >> $@
-	echo "; return zMainloop; }" >> $@
 
 sqlite3_analyzer$(TEXE): sqlite3_analyzer.c
 	$(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS)
 
+sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in
+	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in >sqltclsh.c
+
+sqltclsh$(TEXE): sqltclsh.c
+	$(LTLINK) sqltclsh.c -o $@ $(LIBTCL) $(TLIBS)
+
+sqlite3_expert$(TEXE): $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c
+	$(LTLINK)	$(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c -o sqlite3_expert $(TLIBS)
+
+CHECKER_DEPS =\
+  $(TOP)/tool/mkccode.tcl \
+  sqlite3.c \
+  $(TOP)/src/tclsqlite.c \
+  $(TOP)/ext/repair/sqlite3_checker.tcl \
+  $(TOP)/ext/repair/checkindex.c \
+  $(TOP)/ext/repair/checkfreelist.c \
+  $(TOP)/ext/misc/btreeinfo.c \
+  $(TOP)/ext/repair/sqlite3_checker.c.in
+
+sqlite3_checker.c:	$(CHECKER_DEPS)
+	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/ext/repair/sqlite3_checker.c.in >$@
+
+sqlite3_checker$(TEXE):	sqlite3_checker.c
+	$(LTLINK) sqlite3_checker.c -o $@ $(LIBTCL) $(TLIBS)
+
 dbdump$(TEXE): $(TOP)/ext/misc/dbdump.c sqlite3.lo
 	$(LTLINK) -DDBDUMP_STANDALONE -o $@ \
            $(TOP)/ext/misc/dbdump.c sqlite3.lo $(TLIBS)
@@ -1185,12 +1303,21 @@ showjournal$(TEXE):	$(TOP)/tool/showjournal.c sqlite3.lo
 showwal$(TEXE):	$(TOP)/tool/showwal.c sqlite3.lo
 	$(LTLINK) -o $@ $(TOP)/tool/showwal.c sqlite3.lo $(TLIBS)
 
+showshm$(TEXE):	$(TOP)/tool/showshm.c
+	$(LTLINK) -o $@ $(TOP)/tool/showshm.c
+
 changeset$(TEXE):	$(TOP)/ext/session/changeset.c sqlite3.lo
 	$(LTLINK) -o $@ $(TOP)/ext/session/changeset.c sqlite3.lo $(TLIBS)
 
+changesetfuzz$(TEXE):	$(TOP)/ext/session/changesetfuzz.c sqlite3.lo
+	$(LTLINK) -o $@ $(TOP)/ext/session/changesetfuzz.c sqlite3.lo $(TLIBS)
+
 rollback-test$(TEXE):	$(TOP)/tool/rollback-test.c sqlite3.lo
 	$(LTLINK) -o $@ $(TOP)/tool/rollback-test.c sqlite3.lo $(TLIBS)
 
+atrc$(TEXX): $(TOP)/test/atrc.c sqlite3.lo
+	$(LTLINK) -o $@ $(TOP)/test/atrc.c sqlite3.lo $(TLIBS)
+
 LogEst$(TEXE):	$(TOP)/tool/logest.c sqlite3.h
 	$(LTLINK) -I. -o $@ $(TOP)/tool/logest.c
 
@@ -1205,7 +1332,7 @@ KV_OPT += -DSQLITE_DIRECT_OVERFLOW_READ
 kvtest$(TEXE):	$(TOP)/test/kvtest.c sqlite3.c
 	$(LTLINK) $(KV_OPT) -o $@ $(TOP)/test/kvtest.c sqlite3.c $(TLIBS)
 
-rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.lo 
+rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.lo
 	$(LTLINK) -I. -o $@ $(TOP)/ext/rbu/rbu.c sqlite3.lo $(TLIBS)
 
 loadfts$(EXE): $(TOP)/tool/loadfts.c libsqlite3.la
@@ -1233,7 +1360,7 @@ snapshot-tarball: sqlite3.c
 # The next two rules are used to support the "threadtest" target. Building
 # threadtest runs a few thread-safety tests that are implemented in C. This
 # target is invoked by the releasetest.tcl script.
-# 
+#
 THREADTEST3_SRC = $(TOP)/test/threadtest3.c    \
                   $(TOP)/test/tt3_checkpoint.c \
                   $(TOP)/test/tt3_index.c      \
@@ -1247,7 +1374,7 @@ threadtest3$(TEXE): sqlite3.lo $(THREADTEST3_SRC)
 threadtest: threadtest3$(TEXE)
 	./threadtest3$(TEXE)
 
-releasetest:	
+releasetest:
 	$(TCLSH_CMD) $(TOP)/test/releasetest.tcl
 
 # Standard install and cleanup targets
@@ -1255,7 +1382,7 @@ releasetest:
 lib_install:	libsqlite3.la
 	$(INSTALL) -d $(DESTDIR)$(libdir)
 	$(LTINSTALL) libsqlite3.la $(DESTDIR)$(libdir)
-	
+
 install:	sqlite3$(TEXE) lib_install sqlite3.h sqlite3.pc ${HAVE_TCL:1=tcl_install}
 	$(INSTALL) -d $(DESTDIR)$(bindir)
 	$(LTINSTALL) sqlite3$(TEXE) $(DESTDIR)$(bindir)
@@ -1273,7 +1400,7 @@ tcl_install:	lib_install libtclsqlite3.la pkgIndex.tcl
 	rm -f $(DESTDIR)$(TCLLIBDIR)/libtclsqlite3.la $(DESTDIR)$(TCLLIBDIR)/libtclsqlite3.a
 	$(INSTALL) -m 0644 pkgIndex.tcl $(DESTDIR)$(TCLLIBDIR)
 
-clean:	
+clean:
 	rm -f *.lo *.la *.o sqlite3$(TEXE) libsqlite3.la
 	rm -f sqlite3.h opcodes.*
 	rm -rf .libs .deps

Makefile.msc

@@ -92,6 +92,29 @@ SPLIT_AMALGAMATION = 0
 !ENDIF
 
 # <<mark>>
+# Set this non-0 to have this makefile assume the Tcl shell executable
+# (tclsh*.exe) is available in the PATH.  By default, this is disabled
+# for compatibility with older build environments.  This setting only
+# applies if TCLSH_CMD is not set manually.
+#
+!IFNDEF USE_TCLSH_IN_PATH
+USE_TCLSH_IN_PATH = 0
+!ENDIF
+
+# Set this non-0 to use zlib, possibly compiling it from source code.
+#
+!IFNDEF USE_ZLIB
+USE_ZLIB = 0
+!ENDIF
+
+# Set this non-0 to build zlib from source code.  This is enabled by
+# default and in that case it will be assumed that the ZLIBDIR macro
+# points to the top-level source code directory for zlib.
+#
+!IFNDEF BUILD_ZLIB
+BUILD_ZLIB = 1
+!ENDIF
+
 # Set this non-0 to use the International Components for Unicode (ICU).
 #
 !IFNDEF USE_ICU
@@ -316,6 +339,13 @@ SQLITE_TCL_DEP =
 !IF $(MINIMAL_AMALGAMATION)==0
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_GEOPOLY=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_JSON1=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBSTAT_VTAB=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_INTROSPECTION_PRAGMAS=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DESERIALIZE=1
 !ENDIF
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1
 !ENDIF
@@ -597,6 +627,10 @@ SHELL_COMPILE_OPTS = $(SHELL_CCONV_OPTS)
 !IFNDEF SHELL_CORE_SRC
 !IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
 SHELL_CORE_SRC =
+# <<mark>>
+!ELSEIF $(USE_AMALGAMATION)==0
+SHELL_CORE_SRC =
+# <</mark>>
 !ELSE
 SHELL_CORE_SRC = $(SQLITE3C)
 !ENDIF
@@ -607,16 +641,33 @@ SHELL_CORE_SRC = $(SQLITE3C)
 !IFNDEF SHELL_CORE_DEP
 !IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
 SHELL_CORE_DEP = $(SQLITE3DLL)
+# <<mark>>
+!ELSEIF $(USE_AMALGAMATION)==0
+SHELL_CORE_DEP = libsqlite3.lib
+# <</mark>>
 !ELSE
 SHELL_CORE_DEP =
 !ENDIF
 !ENDIF
 
+# <<mark>>
+# If zlib support is enabled, add the dependencies for it.
+#
+!IF $(USE_ZLIB)!=0 && $(BUILD_ZLIB)!=0
+SHELL_CORE_DEP = zlib $(SHELL_CORE_DEP)
+TESTFIXTURE_DEP = zlib $(TESTFIXTURE_DEP)
+!ENDIF
+# <</mark>>
+
 # This is the core library that the shell executable should link with.
 #
 !IFNDEF SHELL_CORE_LIB
 !IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
 SHELL_CORE_LIB = $(SQLITE3LIB)
+# <<mark>>
+!ELSEIF $(USE_AMALGAMATION)==0
+SHELL_CORE_LIB = libsqlite3.lib
+# <</mark>>
 !ELSE
 SHELL_CORE_LIB =
 !ENDIF
@@ -802,12 +853,16 @@ RCC = $(RCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1
 # prior to running nmake in order to match the actual installed location and
 # version on this machine.
 #
+!IFNDEF TCLDIR
+TCLDIR = $(TOP)\compat\tcl
+!ENDIF
+
 !IFNDEF TCLINCDIR
-TCLINCDIR = c:\tcl\include
+TCLINCDIR = $(TCLDIR)\include
 !ENDIF
 
 !IFNDEF TCLLIBDIR
-TCLLIBDIR = c:\tcl\lib
+TCLLIBDIR = $(TCLDIR)\lib
 !ENDIF
 
 !IFNDEF LIBTCL
@@ -819,7 +874,32 @@ LIBTCLSTUB = tclstub86.lib
 !ENDIF
 
 !IFNDEF LIBTCLPATH
-LIBTCLPATH = c:\tcl\bin
+LIBTCLPATH = $(TCLDIR)\bin
+!ENDIF
+
+# The locations of the zlib header and library files.  These variables
+# (ZLIBINCDIR, ZLIBLIBDIR, and ZLIBLIB) may be overridden via the environment
+# prior to running nmake in order to match the actual installed (or source
+# code) location on this machine.
+#
+!IFNDEF ZLIBDIR
+ZLIBDIR = $(TOP)\compat\zlib
+!ENDIF
+
+!IFNDEF ZLIBINCDIR
+ZLIBINCDIR = $(ZLIBDIR)
+!ENDIF
+
+!IFNDEF ZLIBLIBDIR
+ZLIBLIBDIR = $(ZLIBDIR)
+!ENDIF
+
+!IFNDEF ZLIBLIB
+!IF $(DYNAMIC_SHELL)!=0
+ZLIBLIB = zdll.lib
+!ELSE
+ZLIBLIB = zlib.lib
+!ENDIF
 !ENDIF
 
 # The locations of the ICU header and library files.  These variables
@@ -827,12 +907,16 @@ LIBTCLPATH = c:\tcl\bin
 # prior to running nmake in order to match the actual installed location on
 # this machine.
 #
+!IFNDEF ICUDIR
+ICUDIR = $(TOP)\compat\icu
+!ENDIF
+
 !IFNDEF ICUINCDIR
-ICUINCDIR = c:\icu\include
+ICUINCDIR = $(ICUDIR)\include
 !ENDIF
 
 !IFNDEF ICULIBDIR
-ICULIBDIR = c:\icu\lib
+ICULIBDIR = $(ICUDIR)\lib
 !ENDIF
 
 !IFNDEF LIBICU
@@ -845,7 +929,11 @@ LIBICU = icuuc.lib icuin.lib
 # specific Tcl shell to use.
 #
 !IFNDEF TCLSH_CMD
+!IF $(USE_TCLSH_IN_PATH)!=0 || !EXIST("$(TCLDIR)\bin\tclsh.exe")
 TCLSH_CMD = tclsh
+!ELSE
+TCLSH_CMD = $(TCLDIR)\bin\tclsh.exe
+!ENDIF
 !ENDIF
 # <</mark>>
 
@@ -951,6 +1039,15 @@ BCC = $(BCC) -Zi
 !ENDIF
 
 # <<mark>>
+# If zlib support is enabled, add the compiler options for it.
+#
+!IF $(USE_ZLIB)!=0
+TCC = $(TCC) -DSQLITE_HAVE_ZLIB=1
+RCC = $(RCC) -DSQLITE_HAVE_ZLIB=1
+TCC = $(TCC) -I$(ZLIBINCDIR)
+RCC = $(RCC) -I$(ZLIBINCDIR)
+!ENDIF
+
 # If ICU support is enabled, add the compiler options for it.
 #
 !IF $(USE_ICU)!=0
@@ -974,7 +1071,7 @@ LTLINK = $(TCC) -Fe$@
 # If requested, link to the RPCRT4 library.
 #
 !IF $(USE_RPCRT4_LIB)!=0
-LTLINK = $(LTLINK) rpcrt4.lib
+LTLIBS = $(LTLIBS) rpcrt4.lib
 !ENDIF
 
 # If a platform was set, force the linker to target that.
@@ -1071,8 +1168,15 @@ LDFLAGS = $(LDOPTS)
 # Start with the Tcl related linker options.
 #
 !IF $(NO_TCL)==0
-LTLIBPATHS = /LIBPATH:$(TCLLIBDIR)
+TCLLIBPATHS = $(TCLLIBPATHS) /LIBPATH:$(TCLLIBDIR)
-LTLIBS = $(LIBTCL)
+TCLLIBS = $(TCLLIBS) $(LIBTCL)
+!ENDIF
+
+# If zlib support is enabled, add the linker options for it.
+#
+!IF $(USE_ZLIB)!=0
+LTLIBPATHS = $(LTLIBPATHS) /LIBPATH:$(ZLIBLIBDIR)
+LTLIBS = $(LTLIBS) $(ZLIBLIB)
 !ENDIF
 
 # If ICU support is enabled, add the linker options for it.
@@ -1091,26 +1195,27 @@ LTLIBS = $(LTLIBS) $(LIBICU)
 #
 LIBOBJS0 = vdbe.lo parse.lo alter.lo analyze.lo attach.lo auth.lo \
          backup.lo bitvec.lo btmutex.lo btree.lo build.lo \
-         callback.lo complete.lo ctime.lo date.lo dbstat.lo delete.lo \
+         callback.lo complete.lo ctime.lo \
+         date.lo dbpage.lo dbstat.lo delete.lo \
          expr.lo fault.lo fkey.lo \
          fts3.lo fts3_aux.lo fts3_expr.lo fts3_hash.lo fts3_icu.lo \
          fts3_porter.lo fts3_snippet.lo fts3_tokenizer.lo fts3_tokenizer1.lo \
          fts3_tokenize_vtab.lo fts3_unicode.lo fts3_unicode2.lo fts3_write.lo \
          fts5.lo \
          func.lo global.lo hash.lo \
-         icu.lo insert.lo legacy.lo loadext.lo \
+         icu.lo insert.lo json1.lo legacy.lo loadext.lo \
          main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \
-         memjournal.lo \
+         memdb.lo memjournal.lo \
          mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \
          notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \
          pager.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
          random.lo resolve.lo rowset.lo rtree.lo \
-         sqlite3session.lo select.lo sqlite3rbu.lo status.lo \
+         sqlite3session.lo select.lo sqlite3rbu.lo status.lo stmt.lo \
          table.lo threads.lo tokenize.lo treeview.lo trigger.lo \
-         update.lo util.lo vacuum.lo \
+         update.lo upsert.lo util.lo vacuum.lo \
          vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
          vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \
-         utf.lo vtab.lo
+         window.lo utf.lo vtab.lo
 # <</mark>>
 
 # Object files for the amalgamation.
@@ -1154,6 +1259,7 @@ SRC00 = \
   $(TOP)\src\complete.c \
   $(TOP)\src\ctime.c \
   $(TOP)\src\date.c \
+  $(TOP)\src\dbpage.c \
   $(TOP)\src\dbstat.c \
   $(TOP)\src\delete.c \
   $(TOP)\src\expr.c \
@@ -1172,6 +1278,7 @@ SRC00 = \
   $(TOP)\src\mem2.c \
   $(TOP)\src\mem3.c \
   $(TOP)\src\mem5.c \
+  $(TOP)\src\memdb.c \
   $(TOP)\src\memjournal.c \
   $(TOP)\src\mutex.c \
   $(TOP)\src\mutex_noop.c \
@@ -1204,6 +1311,7 @@ SRC01 = \
   $(TOP)\src\trigger.c \
   $(TOP)\src\utf.c \
   $(TOP)\src\update.c \
+  $(TOP)\src\upsert.c \
   $(TOP)\src\util.c \
   $(TOP)\src\vacuum.c \
   $(TOP)\src\vdbe.c \
@@ -1218,12 +1326,8 @@ SRC01 = \
   $(TOP)\src\walker.c \
   $(TOP)\src\where.c \
   $(TOP)\src\wherecode.c \
-  $(TOP)\src\whereexpr.c
+  $(TOP)\src\whereexpr.c \
-
+  $(TOP)\src\window.c
-# Shell source code files.
-#
-SRC02 = \
-  $(TOP)\src\shell.c
 
 # Core miscellaneous files.
 #
@@ -1316,6 +1420,7 @@ SRC09 = \
   $(TOP)\ext\fts3\fts3_tokenizer.h \
   $(TOP)\ext\icu\sqliteicu.h \
   $(TOP)\ext\rtree\rtree.h \
+  $(TOP)\ext\rtree\geopoly.c \
   $(TOP)\ext\rbu\sqlite3rbu.h \
   $(TOP)\ext\session\sqlite3session.h
 
@@ -1331,6 +1436,7 @@ SRC11 = \
   keywordhash.h \
   opcodes.h \
   parse.h \
+  shell.c \
   $(SQLITE3H)
 
 # Generated Tcl header files
@@ -1345,7 +1451,7 @@ SRC12 =
 
 # All source code files.
 #
-SRC = $(SRC00) $(SRC01) $(SRC02) $(SRC03) $(SRC04) $(SRC05) $(SRC06) $(SRC07) $(SRC08) $(SRC09) $(SRC10) $(SRC11)
+SRC = $(SRC00) $(SRC01) $(SRC03) $(SRC04) $(SRC05) $(SRC06) $(SRC07) $(SRC08) $(SRC09) $(SRC10) $(SRC11)
 
 # Source code to the test files.
 #
@@ -1376,6 +1482,7 @@ TESTSRC = \
   $(TOP)\src\test_intarray.c \
   $(TOP)\src\test_journal.c \
   $(TOP)\src\test_malloc.c \
+  $(TOP)\src\test_md5.c \
   $(TOP)\src\test_multiplex.c \
   $(TOP)\src\test_mutex.c \
   $(TOP)\src\test_onefile.c \
@@ -1387,10 +1494,12 @@ TESTSRC = \
   $(TOP)\src\test_server.c \
   $(TOP)\src\test_superlock.c \
   $(TOP)\src\test_syscall.c \
+  $(TOP)\src\test_tclsh.c \
   $(TOP)\src\test_tclvar.c \
   $(TOP)\src\test_thread.c \
   $(TOP)\src\test_vfs.c \
   $(TOP)\src\test_windirent.c \
+  $(TOP)\src\test_window.c \
   $(TOP)\src\test_wsd.c \
   $(TOP)\ext\fts3\fts3_term.c \
   $(TOP)\ext\fts3\fts3_test.c \
@@ -1400,11 +1509,14 @@ TESTSRC = \
 # Statically linked extensions.
 #
 TESTEXT = \
+  $(TOP)\ext\expert\sqlite3expert.c \
+  $(TOP)\ext\expert\test_expert.c \
   $(TOP)\ext\misc\amatch.c \
   $(TOP)\ext\misc\carray.c \
   $(TOP)\ext\misc\closure.c \
   $(TOP)\ext\misc\csv.c \
   $(TOP)\ext\misc\eval.c \
+  $(TOP)\ext\misc\explain.c \
   $(TOP)\ext\misc\fileio.c \
   $(TOP)\ext\misc\fuzzer.c \
   $(TOP)\ext\fts5\fts5_tcl.c \
@@ -1413,6 +1525,7 @@ TESTEXT = \
   $(TOP)\ext\misc\ieee754.c \
   $(TOP)\ext\misc\mmapwarm.c \
   $(TOP)\ext\misc\nextchar.c \
+  $(TOP)\ext\misc\normalize.c \
   $(TOP)\ext\misc\percentile.c \
   $(TOP)\ext\misc\regexp.c \
   $(TOP)\ext\misc\remember.c \
@@ -1422,6 +1535,12 @@ TESTEXT = \
   $(TOP)\ext\misc\unionvtab.c \
   $(TOP)\ext\misc\wholenumber.c
 
+# If use of zlib is enabled, add the "zipfile.c" source file.
+#
+!IF $(USE_ZLIB)!=0
+TESTEXT = $(TESTEXT) $(TOP)\ext\misc\zipfile.c
+!ENDIF
+
 # Source code to the library files needed by the test fixture
 # (non-amalgamation)
 #
@@ -1477,7 +1596,8 @@ EXTHDR = $(EXTHDR) \
   $(TOP)\ext\fts3\fts3_hash.h \
   $(TOP)\ext\fts3\fts3_tokenizer.h
 EXTHDR = $(EXTHDR) \
-  $(TOP)\ext\rtree\rtree.h
+  $(TOP)\ext\rtree\rtree.h \
+  $(TOP)\ext\rtree\geopoly.c
 EXTHDR = $(EXTHDR) \
   $(TOP)\ext\icu\sqliteicu.h
 EXTHDR = $(EXTHDR) \
@@ -1491,8 +1611,10 @@ TESTPROGS = \
   testfixture.exe \
   $(SQLITE3EXE) \
   sqlite3_analyzer.exe \
+  sqlite3_checker.exe \
   sqldiff.exe \
-  dbhash.exe
+  dbhash.exe \
+  sqltclsh.exe
 
 # Databases containing fuzzer test cases
 #
@@ -1501,27 +1623,31 @@ FUZZDATA = \
   $(TOP)\test\fuzzdata2.db \
   $(TOP)\test\fuzzdata3.db \
   $(TOP)\test\fuzzdata4.db \
-  $(TOP)\test\fuzzdata5.db
+  $(TOP)\test\fuzzdata5.db \
+  $(TOP)\test\fuzzdata6.db \
+  $(TOP)\test\fuzzdata7.db
 # <</mark>>
 
 # Additional compiler options for the shell.  These are only effective
 # when the shell is not being dynamically linked.
 #
 !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
-SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DESERIALIZE=1
 !ENDIF
 
 # <<mark>>
 # Extra compiler options for various test tools.
 #
-MPTESTER_COMPILE_OPTS = -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS5
+MPTESTER_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS5
 FUZZERSHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1
-FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ -DSQLITE_MAX_MEMORY=50000000
+FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ -DSQLITE_MAX_MEMORY=50000000 -DSQLITE_PRINTF_PRECISION_LIMIT=1000
 FUZZCHECK_SRC = $(TOP)\test\fuzzcheck.c $(TOP)\test\ossfuzz.c
 OSSSHELL_SRC = $(TOP)\test\ossshell.c $(TOP)\test\ossfuzz.c
 DBFUZZ_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION
 KV_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_DIRECT_OVERFLOW_READ
-DBSELFTEST_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5
 ST_COMPILE_OPTS = -DSQLITE_THREADSAFE=0
 
 # Standard options to testfixture.
@@ -1540,7 +1666,15 @@ ALL_TCL_TARGETS =
 # This is the default Makefile target.  The objects listed here
 # are what get build when you type just "make" with no arguments.
 #
-all:	dll libsqlite3.lib shell $(ALL_TCL_TARGETS)
+core:	dll libsqlite3.lib shell
+
+# Targets that require the Tcl library.
+#
+tcl:	$(ALL_TCL_TARGETS)
+
+# This Makefile target builds all of the standard binaries.
+#
+all:	core tcl
 
 # Dynamic link library section.
 #
@@ -1565,12 +1699,12 @@ $(SQLITE3DLL):	$(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP)
 sqlite3.def:	libsqlite3.lib
 	echo EXPORTS > sqlite3.def
 	dumpbin /all libsqlite3.lib \
-		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3(?:session|changeset|changegroup)?_[^@]*)(?:@\d+)?$$" \1 \
+		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3(?:session|changeset|changegroup|rebaser)?_[^@]*)(?:@\d+)?$$" \1 \
 		| sort >> sqlite3.def
 # <</block2>>
 
-$(SQLITE3EXE):	$(TOP)\src\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
+$(SQLITE3EXE):	shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
-	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\src\shell.c $(SHELL_CORE_SRC) \
+	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) shell.c $(SHELL_CORE_SRC) \
 		/link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS)
 
 # <<mark>>
@@ -1581,13 +1715,13 @@ dbhash.exe:	$(TOP)\tool\dbhash.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) $(TOP)\tool\dbhash.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
 
 scrub.exe:	$(TOP)\ext\misc\scrub.c $(SQLITE3C) $(SQLITE3H)
-	$(LTLINK) $(NO_WARN) $(TOP)\ext\misc\scrub.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
+	$(LTLINK) $(NO_WARN) -DSCRUB_STANDALONE=1 $(TOP)\ext\misc\scrub.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
 
 srcck1.exe:	$(TOP)\tool\srcck1.c
 	$(BCC) $(NO_WARN) -Fe$@ $(TOP)\tool\srcck1.c
 
-sourcetest:	srcck1.exe sqlite3.c
+sourcetest:	srcck1.exe $(SQLITE3C)
-	srcck1.exe sqlite3.c
+	srcck1.exe $(SQLITE3C)
 
 fuzzershell.exe:	$(TOP)\tool\fuzzershell.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) $(FUZZERSHELL_COMPILE_OPTS) $(TOP)\tool\fuzzershell.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
@@ -1601,6 +1735,9 @@ fuzzcheck.exe:	$(FUZZCHECK_SRC) $(SQLITE3C) $(SQLITE3H)
 ossshell.exe:	$(OSSSHELL_SRC) $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) $(FUZZCHECK_COMPILE_OPTS) $(OSSSHELL_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
 
+sessionfuzz.exe:	zlib $(TOP)\test\sessionfuzz.c $(SQLITE3C) $(SQLITE3H)
+	$(LTLINK) $(NO_WARN) -I$(ZLIBINCDIR) $(TOP)\test\sessionfuzz.c /link $(LDFLAGS) $(LTLINKOPTS) /LIBPATH:$(ZLIBLIBDIR) $(ZLIBLIB)
+
 mptester.exe:	$(TOP)\mptest\mptest.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) $(MPTESTER_COMPILE_OPTS) $(TOP)\mptest\mptest.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
 
@@ -1629,7 +1766,6 @@ mptest:	mptester.exe
 	-mkdir tsrc
 	for %i in ($(SRC00)) do copy /Y %i tsrc
 	for %i in ($(SRC01)) do copy /Y %i tsrc
-	for %i in ($(SRC02)) do copy /Y %i tsrc
 	for %i in ($(SRC03)) do copy /Y %i tsrc
 	for %i in ($(SRC04)) do copy /Y %i tsrc
 	for %i in ($(SRC05)) do copy /Y %i tsrc
@@ -1649,7 +1785,6 @@ mptest:	mptester.exe
 
 sqlite3.c:	.target_source sqlite3ext.h $(MKSQLITE3C_TOOL)
 	$(TCLSH_CMD) $(MKSQLITE3C_TOOL) $(MKSQLITE3C_ARGS)
-	copy tsrc\shell.c .
 	copy $(TOP)\ext\session\sqlite3session.h .
 
 sqlite3-all.c:	sqlite3.c $(TOP)\tool\split-sqlite3c.tcl
@@ -1747,7 +1882,10 @@ ctime.lo:	$(TOP)\src\ctime.c $(HDR)
 date.lo:	$(TOP)\src\date.c $(HDR)
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\date.c
 
-dbstat.lo:	$(TOP)\src\date.c $(HDR)
+dbpage.lo:	$(TOP)\src\dbpage.c $(HDR)
+	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\dbpage.c
+
+dbstat.lo:	$(TOP)\src\dbstat.c $(HDR)
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\dbstat.c
 
 delete.lo:	$(TOP)\src\delete.c $(HDR)
@@ -1801,6 +1939,9 @@ mem3.lo:	$(TOP)\src\mem3.c $(HDR)
 mem5.lo:	$(TOP)\src\mem5.c $(HDR)
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\mem5.c
 
+memdb.lo:	$(TOP)\src\memdb.c $(HDR)
+	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\memdb.c
+
 memjournal.lo:	$(TOP)\src\memjournal.c $(HDR)
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\memjournal.c
 
@@ -1879,6 +2020,9 @@ trigger.lo:	$(TOP)\src\trigger.c $(HDR)
 update.lo:	$(TOP)\src\update.c $(HDR)
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\update.c
 
+upsert.lo:	$(TOP)\src\upsert.c $(HDR)
+	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\upsert.c
+
 utf.lo:	$(TOP)\src\utf.c $(HDR)
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\utf.c
 
@@ -1927,14 +2071,17 @@ wherecode.lo:	$(TOP)\src\wherecode.c $(HDR)
 whereexpr.lo:	$(TOP)\src\whereexpr.c $(HDR)
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\whereexpr.c
 
+window.lo:	$(TOP)\src\window.c $(HDR)
+	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\window.c
+
 tclsqlite.lo:	$(TOP)\src\tclsqlite.c $(HDR) $(SQLITE_TCL_DEP)
 	$(LTCOMPILE) $(NO_WARN) -DUSE_TCL_STUBS=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c
 
 tclsqlite-shell.lo:	$(TOP)\src\tclsqlite.c $(HDR) $(SQLITE_TCL_DEP)
-	$(LTCOMPILE) $(NO_WARN) -DTCLSH=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c
+	$(LTCOMPILE) $(NO_WARN) -DTCLSH -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c
 
 tclsqlite3.exe:	tclsqlite-shell.lo $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS)
-	$(LTLINK) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /OUT:$@ tclsqlite-shell.lo $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
+	$(LTLINK) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) /OUT:$@ tclsqlite-shell.lo $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS)
 
 # Rules to build opcodes.c and opcodes.h
 #
@@ -1974,7 +2121,29 @@ mkkeywordhash.exe:	$(TOP)\tool\mkkeywordhash.c
 keywordhash.h:	$(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe
 	.\mkkeywordhash.exe > keywordhash.h
 
+# Source files that go into making shell.c
+SHELL_SRC = \
+	$(TOP)\src\shell.c.in \
+	$(TOP)\ext\misc\appendvfs.c \
+	$(TOP)\ext\misc\shathree.c \
+	$(TOP)\ext\misc\fileio.c \
+	$(TOP)\ext\misc\completion.c \
+	$(TOP)\ext\expert\sqlite3expert.c \
+	$(TOP)\ext\expert\sqlite3expert.h \
+	$(TOP)\src\test_windirent.c
 
+# If use of zlib is enabled, add the "zipfile.c" source file.
+#
+!IF $(USE_ZLIB)!=0
+SHELL_SRC = $(SHELL_SRC) $(TOP)\ext\misc\sqlar.c
+SHELL_SRC = $(SHELL_SRC) $(TOP)\ext\misc\zipfile.c
+!ENDIF
+
+shell.c:	$(SHELL_SRC) $(TOP)\tool\mkshellc.tcl
+	$(TCLSH_CMD) $(TOP)\tool\mkshellc.tcl > shell.c
+
+zlib:
+	pushd $(ZLIBDIR) && $(MAKE) /f win32\Makefile.msc clean $(ZLIBLIB) && popd
 
 # Rules to build the extension objects.
 #
@@ -2038,6 +2207,12 @@ fts3_unicode2.lo:	$(TOP)\ext\fts3\fts3_unicode2.c $(HDR) $(EXTHDR)
 fts3_write.lo:	$(TOP)\ext\fts3\fts3_write.c $(HDR) $(EXTHDR)
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_write.c
 
+json1.lo:	$(TOP)\ext\misc\json1.c $(HDR) $(EXTHDR)
+	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\misc\json1.c
+
+stmt.lo:	$(TOP)\ext\misc\stmt.c $(HDR) $(EXTHDR)
+	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\misc\stmt.c
+
 rtree.lo:	$(TOP)\ext\rtree\rtree.c $(HDR) $(EXTHDR)
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\rtree\rtree.c
 
@@ -2063,6 +2238,24 @@ FTS5_SRC = \
    $(TOP)\ext\fts5\fts5_varint.c \
    $(TOP)\ext\fts5\fts5_vocab.c
 
+LSM1_SRC = \
+   $(TOP)\ext\lsm1\lsm.h \
+   $(TOP)\ext\lsm1\lsmInt.h \
+   $(TOP)\ext\lsm1\lsm_ckpt.c \
+   $(TOP)\ext\lsm1\lsm_file.c \
+   $(TOP)\ext\lsm1\lsm_log.c \
+   $(TOP)\ext\lsm1\lsm_main.c \
+   $(TOP)\ext\lsm1\lsm_mem.c \
+   $(TOP)\ext\lsm1\lsm_mutex.c \
+   $(TOP)\ext\lsm1\lsm_shared.c \
+   $(TOP)\ext\lsm1\lsm_sorted.c \
+   $(TOP)\ext\lsm1\lsm_str.c \
+   $(TOP)\ext\lsm1\lsm_tree.c \
+   $(TOP)\ext\lsm1\lsm_unix.c \
+   $(TOP)\ext\lsm1\lsm_varint.c \
+   $(TOP)\ext\lsm1\lsm_vtab.c \
+   $(TOP)\ext\lsm1\lsm_win32.c
+
 fts5parse.c:	$(TOP)\ext\fts5\fts5parse.y lemon.exe
 	copy $(TOP)\ext\fts5\fts5parse.y .
 	del /Q fts5parse.h 2>NUL
@@ -2074,6 +2267,10 @@ fts5.c:	$(FTS5_SRC)
 	$(TCLSH_CMD) $(TOP)\ext\fts5\tool\mkfts5c.tcl
 	copy $(TOP)\ext\fts5\fts5.h .
 
+lsm1.c:	$(LSM1_SRC)
+	$(TCLSH_CMD) $(TOP)\ext\lsm1\tool\mklsm1c.tcl
+	copy $(TOP)\ext\lsm1\lsm.h .
+
 fts5.lo:	fts5.c $(HDR) $(EXTHDR)
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c fts5.c
 
@@ -2093,12 +2290,14 @@ sqlite3rbu.lo:	$(TOP)\ext\rbu\sqlite3rbu.c $(HDR) $(EXTHDR)
 # necessary because the test fixture requires non-API symbols which are
 # hidden when the library is built via the amalgamation).
 #
-TESTFIXTURE_FLAGS = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
+TESTFIXTURE_FLAGS = -DTCLSH_INIT_PROC=sqlite3TestInit -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERVER=1 -DSQLITE_PRIVATE=""
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_CORE $(NO_WARN)
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_DEFAULT_PAGE_SIZE=1024
-TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB
+TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1
+TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1
+TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_JSON1=1
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) $(TEST_CCONV_OPTS)
 
 TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2)
@@ -2129,16 +2328,20 @@ sqlite_tcl.h:
 		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact "Tcl_HashEntry *(*createProc)" "Tcl_HashEntry *(SQLITE_TCLAPI *createProc)" >> $(SQLITETCLH)
 !ENDIF
 
-testfixture.exe:	$(TESTFIXTURE_SRC) $(SQLITE3H) $(LIBRESOBJS) $(HDR) $(SQLITE_TCL_DEP)
+testfixture.exe:	$(TESTFIXTURE_SRC) $(TESTFIXTURE_DEP) $(SQLITE3H) $(LIBRESOBJS) $(HDR) $(SQLITE_TCL_DEP)
 	$(LTLINK) -DSQLITE_NO_SYNC=1 $(TESTFIXTURE_FLAGS) \
 		-DBUILD_sqlite -I$(TCLINCDIR) \
 		$(TESTFIXTURE_SRC) \
-		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
+		/link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS)
 
 extensiontest:	testfixture.exe testloadext.dll
 	@set PATH=$(LIBTCLPATH);$(PATH)
 	.\testfixture.exe $(TOP)\test\loadext.test $(TESTOPTS)
 
+coretestprogs:	$(TESTPROGS)
+
+testprogs:	coretestprogs srcck1.exe fuzzcheck.exe sessionfuzz.exe
+
 fulltest:	$(TESTPROGS) fuzztest
 	@set PATH=$(LIBTCLPATH);$(PATH)
 	.\testfixture.exe $(TOP)\test\all.test $(TESTOPTS)
@@ -2178,24 +2381,45 @@ smoketest:	$(TESTPROGS)
 	@set PATH=$(LIBTCLPATH);$(PATH)
 	.\testfixture.exe $(TOP)\test\main.test $(TESTOPTS)
 
-sqlite3_analyzer.c:	$(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl $(SQLITE_TCL_DEP)
+sqlite3_analyzer.c:	$(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in $(SQLITE_TCL_DEP)
-	echo #define TCLSH 2 > $@
+	$(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in > $@
-	echo #define SQLITE_ENABLE_DBSTAT_VTAB 1 >> $@
-	copy $@ + $(SQLITE3C) + $(TOP)\src\tclsqlite.c $@
-	echo static const char *tclsh_main_loop(void){ >> $@
-	echo static const char *zMainloop = >> $@
-	$(TCLSH_CMD) $(TOP)\tool\tostr.tcl $(TOP)\tool\spaceanal.tcl >> $@
-	echo ; return zMainloop; } >> $@
 
 sqlite3_analyzer.exe:	sqlite3_analyzer.c $(LIBRESOBJS)
 	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_analyzer.c \
-		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
+		/link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS)
 
-dbdump.exe:	$(TOP)\ext\misc\dbdump.c $(SQLITE3C) $(SQLITE3H)
+sqltclsh.c: sqlite3.c $(TOP)\src\tclsqlite.c $(TOP)\tool\sqltclsh.tcl $(TOP)\ext\misc\appendvfs.c $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqltclsh.c.in
+	$(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqltclsh.c.in >sqltclsh.c
+
+sqltclsh.exe: sqltclsh.c  $(SHELL_CORE_DEP) $(LIBRESOBJS)
+	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqltclsh.c \
+		/link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS)
+
+sqlite3_expert.exe: $(SQLITE3C) $(TOP)\ext\expert\sqlite3expert.h $(TOP)\ext\expert\sqlite3expert.c $(TOP)\ext\expert\expert.c
+	$(LTLINK) $(NO_WARN)	$(TOP)\ext\expert\sqlite3expert.c $(TOP)\ext\expert\expert.c $(SQLITE3C) $(TLIBS)
+
+CHECKER_DEPS =\
+  $(TOP)/tool/mkccode.tcl \
+  sqlite3.c \
+  $(TOP)/src/tclsqlite.c \
+  $(TOP)/ext/repair/sqlite3_checker.tcl \
+  $(TOP)/ext/repair/checkindex.c \
+  $(TOP)/ext/repair/checkfreelist.c \
+  $(TOP)/ext/misc/btreeinfo.c \
+  $(TOP)/ext/repair/sqlite3_checker.c.in
+
+sqlite3_checker.c:	$(CHECKER_DEPS)
+	$(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\ext\repair\sqlite3_checker.c.in > $@
+
+sqlite3_checker.exe:	sqlite3_checker.c $(LIBRESOBJS)
+	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_checker.c \
+		/link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS)
+
+dbdump.exe:	$(TOP)\ext\misc\dbdump.c $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS)
 	$(LTLINK) $(NO_WARN) -DDBDUMP_STANDALONE $(TOP)\ext\misc\dbdump.c $(SQLITE3C) \
 		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS)
 
-testloadext.lo:	$(TOP)\src\test_loadext.c
+testloadext.lo:	$(TOP)\src\test_loadext.c $(SQLITE3H)
 	$(LTCOMPILE) $(NO_WARN) -c $(TOP)\src\test_loadext.c
 
 testloadext.dll:	testloadext.lo
@@ -2217,11 +2441,19 @@ showwal.exe:	$(TOP)\tool\showwal.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
 		$(TOP)\tool\showwal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
 
+showshm.exe:	$(TOP)\tool\showshm.c
+	$(LTLINK) $(NO_WARN)	$(TOP)\tool\showshm.c /link $(LDFLAGS) $(LTLINKOPTS)
+
 changeset.exe:	$(TOP)\ext\session\changeset.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
 		-DSQLITE_ENABLE_SESSION=1 -DSQLITE_ENABLE_PREUPDATE_HOOK=1 \
 		$(TOP)\ext\session\changeset.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
 
+changesetfuzz.exe:	$(TOP)\ext\session\changesetfuzz.c $(SQLITE3C) $(SQLITE3H)
+	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
+		-DSQLITE_ENABLE_SESSION=1 -DSQLITE_ENABLE_PREUPDATE_HOOK=1 \
+		$(TOP)\ext\session\changesetfuzz.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
+
 fts3view.exe:	$(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
 		$(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
@@ -2230,6 +2462,10 @@ rollback-test.exe:	$(TOP)\tool\rollback-test.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
 		$(TOP)\tool\rollback-test.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
 
+atrc.exe:	$(TOP)\test\atrc.c $(SQLITE3C) $(SQLITE3H)
+	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
+		$(TOP)\test\atrc.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
+
 LogEst.exe:	$(TOP)\tool\logest.c $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) $(TOP)\tool\LogEst.c /link $(LDFLAGS) $(LTLINKOPTS)
 
@@ -2245,9 +2481,6 @@ kvtest.exe:	$(TOP)\test\kvtest.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) $(KV_COMPILE_OPTS) \
 		$(TOP)\test\kvtest.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
 
-dbselftest.exe:	$(TOP)\test\dbselftest.c $(SQLITE3C) $(SQLITE3H)
-	$(LTLINK) $(NO_WARN) $(DBSELFTEST_COMPILE_OPTS) $(TOP)\test\dbselftest.c $(SQLITE3C)
-
 rbu.exe:	$(TOP)\ext\rbu\rbu.c $(TOP)\ext\rbu\sqlite3rbu.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) -DSQLITE_ENABLE_RBU \
 		$(TOP)\ext\rbu\rbu.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
@@ -2264,7 +2497,6 @@ clean:
 	del /Q *.bsc *.def *.cod *.da *.bb *.bbg *.vc gmon.out 2>NUL
 	del /Q $(SQLITE3EXE) $(SQLITE3DLL) Replace.exe 2>NUL
 # <<mark>>
-	del /Q sqlite3.c sqlite3.h 2>NUL
 	del /Q opcodes.c opcodes.h 2>NUL
 	del /Q lemon.* lempar.c parse.* 2>NUL
 	del /Q mksourceid.* mkkeywordhash.* keywordhash.h 2>NUL
@@ -2281,11 +2513,16 @@ clean:
 	del /Q changeset.exe 2>NUL
 	del /Q showjournal.exe showstat4.exe showwal.exe speedtest1.exe 2>NUL
 	del /Q mptester.exe wordcount.exe rbu.exe srcck1.exe 2>NUL
-	del /Q sqlite3.c sqlite3-*.c 2>NUL
+	del /Q sqlite3.c sqlite3-*.c sqlite3.h 2>NUL
 	del /Q sqlite3rc.h 2>NUL
 	del /Q shell.c sqlite3ext.h sqlite3session.h 2>NUL
 	del /Q sqlite3_analyzer.exe sqlite3_analyzer.c 2>NUL
 	del /Q sqlite-*-output.vsix 2>NUL
 	del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe dbhash.exe 2>NUL
+	del /Q sqltclsh.* 2>NUL
+	del /Q dbfuzz.exe sessionfuzz.exe 2>NUL
+	del /Q kvtest.exe ossshell.exe scrub.exe 2>NUL
+	del /Q showshm.exe sqlite3_checker.* sqlite3_expert.exe 2>NUL
 	del /Q fts5.* fts5parse.* 2>NUL
+	del /Q lsm.h lsm1.c 2>NUL
 # <</mark>>

README.md

@@ -1,12 +1,14 @@
 <h1 align="center">SQLite Source Repository</h1>
 
-This repository contains the complete source code for the SQLite database
+This repository contains the complete source code for the 
-engine.  Some test scripts are also include.  However, many other test scripts
+[SQLite database engine](https://sqlite.org/).  Some test scripts 
+are also included.  However, many other test scripts
 and most of the documentation are managed separately.
 
-If you are reading this on a Git mirror someplace, you are doing it wrong.
+SQLite [does not use Git](https://sqlite.org/whynotgit.html).
-The [official repository](https://www.sqlite.org/src/) is better.  Go there
+If you are reading this on GitHub, then you are looking at an
-now.
+unofficial mirror. See <https://sqlite.org/src> for the official
+repository.
 
 ## Obtaining The Code
 
@@ -14,15 +16,17 @@ SQLite sources are managed using the
 [Fossil](https://www.fossil-scm.org/), a distributed version control system
 that was specifically designed to support SQLite development.
 If you do not want to use Fossil, you can download tarballs or ZIP
-archives as follows:
+archives or [SQLite archives](https://sqlite.org/cli.html#sqlar) as follows:
 
-  *  Lastest trunk check-in:
+  *  Lastest trunk check-in as
-     <https://www.sqlite.org/src/tarball/sqlite.tar.gz> or
+     [Tarball](https://www.sqlite.org/src/tarball/sqlite.tar.gz),
-     <https://www.sqlite.org/src/zip/sqlite.zip>.
+     [ZIP-archive](https://www.sqlite.org/src/zip/sqlite.zip), or
+     [SQLite-archive](https://www.sqlite.org/src/sqlar/sqlite.sqlar).
 
-  *  Latest release:
+  *  Latest release as
-     <https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=release> or
+     [Tarball](https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=release),
-     <https://www.sqlite.org/src/zip/sqlite.zip?r=release>.
+     [ZIP-archive](https://www.sqlite.org/src/zip/sqlite.zip?r=release), or
+     [SQLite-archive](https://www.sqlite.org/src/sqlar/sqlite.sqlar?r=release).
 
   *  For other check-ins, substitute an appropriate branch name or
      tag or hash prefix for "release" in the URLs of the previous
@@ -104,7 +108,6 @@ recommended.
 SQLite does not require [Tcl](http://www.tcl.tk/) to run, but a Tcl installation
 is required by the makefiles (including those for MSVC).  SQLite contains
 a lot of generated code and Tcl is used to do much of that code generation.
-The makefiles also require AWK.
 
 ## Source Code Tour
 
@@ -116,7 +119,7 @@ The **src/** also contains the "shell.c" file
 which is the main program for the "sqlite3.exe"
 [command-line shell](https://sqlite.org/cli.html) and
 the "tclsqlite.c" file which implements the
-[TCL bindings](https://sqlite.org/tclsqlite.html) for SQLite.
+[Tcl bindings](https://sqlite.org/tclsqlite.html) for SQLite.
 (Historical note:  SQLite began as a Tcl
 extension and only later escaped to the wild as an independent library.)
 
@@ -163,14 +166,14 @@ template for generating its parser.
 
 Lemon also generates the **parse.h** header file, at the same time it
 generates parse.c. But the parse.h header file is
-modified further (to add additional symbols) using the ./addopcodes.awk
+modified further (to add additional symbols) using the ./addopcodes.tcl
-AWK script.
+Tcl script.
 
 The **opcodes.h** header file contains macros that define the numbers
 corresponding to opcodes in the "VDBE" virtual machine.  The opcodes.h
 file is generated by the scanning the src/vdbe.c source file.  The
-AWK script at ./mkopcodeh.awk does this scan and generates opcodes.h.
+Tcl script at ./mkopcodeh.tcl does this scan and generates opcodes.h.
-A second AWK script, ./mkopcodec.awk, then scans opcodes.h to generate
+A second Tcl script, ./mkopcodec.tcl, then scans opcodes.h to generate
 the **opcodes.c** source file, which contains a reverse mapping from
 opcode-number to opcode-name that is used for EXPLAIN output.
 
@@ -207,8 +210,8 @@ The amalgamation source file is more than 200K lines long.  Some symbolic
 debuggers (most notably MSVC) are unable to deal with files longer than 64K
 lines.  To work around this, a separate Tcl script, tool/split-sqlite3c.tcl,
 can be run on the amalgamation to break it up into a single small C file
-called **sqlite3-all.c** that does #include on about five other files
+called **sqlite3-all.c** that does #include on about seven other files
-named **sqlite3-1.c**, **sqlite3-2.c**, ..., **sqlite3-5.c**.  In this way,
+named **sqlite3-1.c**, **sqlite3-2.c**, ..., **sqlite3-7.c**.  In this way,
 all of the source code is contained within a single translation unit so
 that the compiler can do extra cross-procedure optimization, but no
 individual source file exceeds 32K lines in length.
@@ -237,7 +240,8 @@ Key files:
      trying to understand how the library works internally.
 
   *  **sqliteInt.h** - this header file defines many of the data objects
-     used internally by SQLite.
+     used internally by SQLite.  In addition to "sqliteInt.h", some
+     subsystems have their own header files.
 
   *  **parse.y** - This file describes the LALR(1) grammar that SQLite uses
      to parse SQL statements, and the actions that are taken at each step
@@ -249,29 +253,44 @@ Key files:
      which defines internal data objects.  The rest of SQLite interacts
      with the VDBE through an interface defined by vdbe.h.
 
-  *  **where.c** - This file analyzes the WHERE clause and generates
+  *  **where.c** - This file (together with its helper files named
+     by "where*.c") analyzes the WHERE clause and generates
      virtual machine code to run queries efficiently.  This file is
      sometimes called the "query optimizer".  It has its own private
      header file, whereInt.h, that defines data objects used internally.
 
   *  **btree.c** - This file contains the implementation of the B-Tree
-     storage engine used by SQLite.
+     storage engine used by SQLite.  The interface to the rest of the system
+     is defined by "btree.h".  The "btreeInt.h" header defines objects
+     used internally by btree.c and not published to the rest of the system.
 
   *  **pager.c** - This file contains the "pager" implementation, the
-     module that implements transactions.
+     module that implements transactions.  The "pager.h" header file
+     defines the interface between pager.c and the rest of the system.
 
   *  **os_unix.c** and **os_win.c** - These two files implement the interface
      between SQLite and the underlying operating system using the run-time
      pluggable VFS interface.
 
-  *  **shell.c** - This file is not part of the core SQLite library.  This
+  *  **shell.c.in** - This file is not part of the core SQLite library.  This
      is the file that, when linked against sqlite3.a, generates the
-     "sqlite3.exe" command-line shell.
+     "sqlite3.exe" command-line shell.  The "shell.c.in" file is transformed
+     into "shell.c" as part of the build process.
 
   *  **tclsqlite.c** - This file implements the Tcl bindings for SQLite.  It
      is not part of the core SQLite library.  But as most of the tests in this
      repository are written in Tcl, the Tcl language bindings are important.
 
+  *  **test*.c** - Files in the src/ folder that begin with "test" go into
+     building the "testfixture.exe" program.  The testfixture.exe program is
+     an enhanced Tcl shell.  The testfixture.exe program runs scripts in the
+     test/ folder to validate the core SQLite code.  The testfixture program
+     (and some other test programs too) is build and run when you type
+     "make test".
+
+  *  **ext/misc/json1.c** - This file implements the various JSON functions
+     that are build into SQLite.
+
 There are many other source files.  Each has a succinct header comment that
 describes its purpose and role within the larger system.
 

VERSION

@@ -1 +1 @@
-3.21.0
+3.26.0

autoconf/Makefile.am

@@ -1,6 +1,5 @@
 
-AM_CFLAGS = @THREADSAFE_FLAGS@ @DYNAMIC_EXTENSION_FLAGS@ @FTS5_FLAGS@ @JSON1_FLAGS@ @SESSION_FLAGS@ -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE
+AM_CFLAGS = @BUILD_CFLAGS@ 
-
 lib_LTLIBRARIES = libsqlite3.la
 libsqlite3_la_SOURCES = sqlite3.c
 libsqlite3_la_LDFLAGS = -no-undefined -version-info 8:6:8
@@ -10,11 +9,11 @@ sqlite3_SOURCES = shell.c sqlite3.h
 EXTRA_sqlite3_SOURCES = sqlite3.c
 sqlite3_LDADD = @EXTRA_SHELL_OBJ@ @READLINE_LIBS@
 sqlite3_DEPENDENCIES = @EXTRA_SHELL_OBJ@
-sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS
+sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_STMTVTAB -DSQLITE_ENABLE_DBSTAT_VTAB $(SHELL_CFLAGS)
 
 include_HEADERS = sqlite3.h sqlite3ext.h
 
-EXTRA_DIST = sqlite3.1 tea Makefile.msc sqlite3.rc README.txt Replace.cs
+EXTRA_DIST = sqlite3.1 tea Makefile.msc sqlite3.rc README.txt Replace.cs Makefile.fallback
 pkgconfigdir = ${libdir}/pkgconfig
 pkgconfig_DATA = sqlite3.pc
 

autoconf/Makefile.fallback

@@ -0,0 +1,19 @@
+#!/usr/bin/make
+#
+# If the configure script does not work, then this Makefile is available
+# as a backup.  Manually configure the variables below.
+#
+# Note:  This makefile works out-of-the-box on MacOS 10.2 (Jaguar)
+#
+CC = gcc
+CFLAGS = -O0 -I.
+LIBS = -lz
+COPTS += -D_BSD_SOURCE
+COPTS += -DSQLITE_ENABLE_LOCKING_STYLE=0
+COPTS += -DSQLITE_THREADSAFE=0
+COPTS += -DSQLITE_OMIT_LOAD_EXTENSION
+COPTS += -DSQLITE_WITHOUT_ZONEMALLOC
+COPTS += -DSQLITE_ENABLE_RTREE
+
+sqlite3:	shell.c sqlite3.c
+	$(CC) $(CFLAGS) $(COPTS) -o sqlite3 shell.c sqlite3.c $(LIBS)

autoconf/Makefile.msc

@@ -277,6 +277,12 @@ SQLITE3EXEPDB = /pdb:sqlite3sh.pdb
 !IF $(MINIMAL_AMALGAMATION)==0
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_GEOPOLY=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_JSON1=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBSTAT_VTAB=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_INTROSPECTION_PRAGMAS=1
 !ENDIF
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1
 !ENDIF
@@ -561,6 +567,7 @@ SHELL_CORE_DEP =
 !ENDIF
 !ENDIF
 
+
 # This is the core library that the shell executable should link with.
 #
 !IFNDEF SHELL_CORE_LIB
@@ -808,7 +815,7 @@ LTLINK = $(TCC) -Fe$@
 # If requested, link to the RPCRT4 library.
 #
 !IF $(USE_RPCRT4_LIB)!=0
-LTLINK = $(LTLINK) rpcrt4.lib
+LTLIBS = $(LTLIBS) rpcrt4.lib
 !ENDIF
 
 # If a platform was set, force the linker to target that.
@@ -927,14 +934,24 @@ LIBRESOBJS =
 # when the shell is not being dynamically linked.
 #
 !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
-SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1
 !ENDIF
 
 
 # This is the default Makefile target.  The objects listed here
 # are what get build when you type just "make" with no arguments.
 #
-all:	dll shell
+core:	dll shell
+
+# Targets that require the Tcl library.
+#
+tcl:	$(ALL_TCL_TARGETS)
+
+# This Makefile target builds all of the standard binaries.
+#
+all:	core tcl
 
 # Dynamic link library section.
 #
@@ -954,11 +971,11 @@ Replace.exe:
 sqlite3.def:	Replace.exe $(LIBOBJ)
 	echo EXPORTS > sqlite3.def
 	dumpbin /all $(LIBOBJ) \
-		| .\Replace.exe "^\s+/EXPORT:_?(sqlite3(?:session|changeset|changegroup)?_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \
+		| .\Replace.exe "^\s+/EXPORT:_?(sqlite3(?:session|changeset|changegroup|rebaser)?_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \
 		| sort >> sqlite3.def
 
-$(SQLITE3EXE):	$(TOP)\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
+$(SQLITE3EXE):	shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
-	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\shell.c $(SHELL_CORE_SRC) \
+	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) shell.c $(SHELL_CORE_SRC) \
 		/link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS)
 
 
@@ -973,7 +990,7 @@ sqlite3.lo:	$(SQLITE3C)
 !IF $(USE_RC)!=0
 _HASHCHAR=^#
 !IF ![echo !IFNDEF VERSION > rcver.vc] && \
-    ![for /F "delims=" %V in ('type "$(SQLITE3H)" ^| find "$(_HASHCHAR)define SQLITE_VERSION "') do (echo VERSION = ^^%V >> rcver.vc)] && \
+    ![for /F "delims=" %V in ('type "$(SQLITE3H)" ^| "%SystemRoot%\System32\find.exe" "$(_HASHCHAR)define SQLITE_VERSION "') do (echo VERSION = ^^%V >> rcver.vc)] && \
     ![echo !ENDIF >> rcver.vc]
 !INCLUDE rcver.vc
 !ENDIF

autoconf/configure.ac

@@ -12,6 +12,7 @@
 AC_PREREQ(2.61)
 AC_INIT(sqlite, --SQLITE-VERSION--, http://www.sqlite.org)
 AC_CONFIG_SRCDIR([sqlite3.c])
+AC_CONFIG_AUX_DIR([.])
 
 # Use automake.
 AM_INIT_AUTOMAKE([foreign])
@@ -28,6 +29,7 @@ AC_CHECK_FUNCS([fdatasync usleep fullfsync localtime_r gmtime_r])
 AC_FUNC_STRERROR_R
 
 AC_CONFIG_FILES([Makefile sqlite3.pc])
+BUILD_CFLAGS=
 AC_SUBST(BUILD_CFLAGS)
 
 #-------------------------------------------------------------------------
@@ -85,13 +87,11 @@ AC_SUBST(READLINE_LIBS)
 AC_ARG_ENABLE(threadsafe, [AS_HELP_STRING(
   [--enable-threadsafe], [build a thread-safe library [default=yes]])], 
   [], [enable_threadsafe=yes])
-THREADSAFE_FLAGS=-DSQLITE_THREADSAFE=0
 if test x"$enable_threadsafe" != "xno"; then
-  THREADSAFE_FLAGS="-D_REENTRANT=1 -DSQLITE_THREADSAFE=1"
+  BUILD_CFLAGS="$BUILD_CFLAGS -D_REENTRANT=1 -DSQLITE_THREADSAFE=1"
   AC_SEARCH_LIBS(pthread_create, pthread)
   AC_SEARCH_LIBS(pthread_mutexattr_init, pthread)
 fi
-AC_SUBST(THREADSAFE_FLAGS)
 #-----------------------------------------------------------------------
 
 #-----------------------------------------------------------------------
@@ -103,36 +103,66 @@ AC_ARG_ENABLE(dynamic-extensions, [AS_HELP_STRING(
 if test x"$enable_dynamic_extensions" != "xno"; then
   AC_SEARCH_LIBS(dlopen, dl)
 else
-  DYNAMIC_EXTENSION_FLAGS=-DSQLITE_OMIT_LOAD_EXTENSION=1
+  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_OMIT_LOAD_EXTENSION=1"
 fi
 AC_MSG_CHECKING([for whether to support dynamic extensions])
 AC_MSG_RESULT($enable_dynamic_extensions)
-AC_SUBST(DYNAMIC_EXTENSION_FLAGS)
+#-----------------------------------------------------------------------
+
+#-----------------------------------------------------------------------
+#   --enable-fts4
+#
+AC_ARG_ENABLE(fts4, [AS_HELP_STRING(
+  [--enable-fts4], [include fts4 support [default=yes]])], 
+  [], [enable_fts4=yes])
+if test x"$enable_fts4" = "xyes"; then
+  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_FTS4"
+fi
+#-----------------------------------------------------------------------
+
+#-----------------------------------------------------------------------
+#   --enable-fts3
+#
+AC_ARG_ENABLE(fts3, [AS_HELP_STRING(
+  [--enable-fts3], [include fts3 support [default=no]])], 
+  [], [])
+if test x"$enable_fts3" = "xyes" -a x"$enable_fts4" = "xno"; then
+  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_FTS3"
+fi
 #-----------------------------------------------------------------------
 
 #-----------------------------------------------------------------------
 #   --enable-fts5
 #
 AC_ARG_ENABLE(fts5, [AS_HELP_STRING(
-  [--enable-fts5], [include fts5 support [default=no]])], 
+  [--enable-fts5], [include fts5 support [default=yes]])], 
-  [], [enable_fts5=no])
+  [], [enable_fts5=yes])
 if test x"$enable_fts5" = "xyes"; then
   AC_SEARCH_LIBS(log, m)
-  FTS5_FLAGS=-DSQLITE_ENABLE_FTS5
+  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_FTS5"
 fi
-AC_SUBST(FTS5_FLAGS)
 #-----------------------------------------------------------------------
 
 #-----------------------------------------------------------------------
 #   --enable-json1
 #
 AC_ARG_ENABLE(json1, [AS_HELP_STRING(
-  [--enable-json1], [include json1 support [default=no]])], 
+  [--enable-json1], [include json1 support [default=yes]])], 
-  [], [enable_json1=no])
+  [],[enable_json1=yes])
 if test x"$enable_json1" = "xyes"; then
-  JSON1_FLAGS=-DSQLITE_ENABLE_JSON1
+  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_JSON1"
+fi
+#-----------------------------------------------------------------------
+
+#-----------------------------------------------------------------------
+#   --enable-rtree
+#
+AC_ARG_ENABLE(rtree, [AS_HELP_STRING(
+  [--enable-rtree], [include rtree support [default=yes]])], 
+  [], [enable_rtree=yes])
+if test x"$enable_rtree" = "xyes"; then
+  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_RTREE"
 fi
-AC_SUBST(JSON1_FLAGS)
 #-----------------------------------------------------------------------
 
 #-----------------------------------------------------------------------
@@ -140,11 +170,22 @@ AC_SUBST(JSON1_FLAGS)
 #
 AC_ARG_ENABLE(session, [AS_HELP_STRING(
   [--enable-session], [enable the session extension [default=no]])], 
-  [], [enable_session=no])
+  [], [])
 if test x"$enable_session" = "xyes"; then
-  SESSION_FLAGS="-DSQLITE_ENABLE_SESSION -DSQLITE_ENABLE_PREUPDATE_HOOK"
+  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_SESSION -DSQLITE_ENABLE_PREUPDATE_HOOK"
+fi
+#-----------------------------------------------------------------------
+
+#-----------------------------------------------------------------------
+#   --enable-debug
+#
+AC_ARG_ENABLE(debug, [AS_HELP_STRING(
+  [--enable-debug], [build with debugging features enabled [default=no]])], 
+  [], [])
+if test x"$enable_debug" = "xyes"; then
+  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_DEBUG -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE"
+  CFLAGS="-g -O0"
 fi
-AC_SUBST(SESSION_FLAGS)
 #-----------------------------------------------------------------------
 
 #-----------------------------------------------------------------------
@@ -163,6 +204,12 @@ AC_SUBST(EXTRA_SHELL_OBJ)
 #-----------------------------------------------------------------------
 
 AC_CHECK_FUNCS(posix_fallocate)
+AC_CHECK_HEADERS(zlib.h,[
+  AC_SEARCH_LIBS(deflate,z,[BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_HAVE_ZLIB"])
+])
+
+AC_SEARCH_LIBS(system,,,[SHELL_CFLAGS="-DSQLITE_NOHAVE_SYSTEM"])
+AC_SUBST(SHELL_CFLAGS)
 
 #-----------------------------------------------------------------------
 # UPDATE: Maybe it's better if users just set CFLAGS before invoking

configure

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for sqlite 3.21.0.
+# Generated by GNU Autoconf 2.69 for sqlite 3.26.0.
 #
 #
 # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -726,8 +726,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='sqlite'
 PACKAGE_TARNAME='sqlite'
-PACKAGE_VERSION='3.21.0'
+PACKAGE_VERSION='3.26.0'
-PACKAGE_STRING='sqlite 3.21.0'
+PACKAGE_STRING='sqlite 3.26.0'
 PACKAGE_BUGREPORT=''
 PACKAGE_URL=''
 
@@ -772,6 +772,7 @@ LIBOBJS
 BUILD_CFLAGS
 USE_GCOV
 OPT_FEATURE_FLAGS
+HAVE_ZLIB
 USE_AMALGAMATION
 TARGET_DEBUG
 TARGET_HAVE_EDITLINE
@@ -910,6 +911,7 @@ enable_fts4
 enable_fts5
 enable_json1
 enable_update_limit
+enable_geopoly
 enable_rtree
 enable_session
 enable_gcov
@@ -1464,7 +1466,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures sqlite 3.21.0 to adapt to many kinds of systems.
+\`configure' configures sqlite 3.26.0 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1529,7 +1531,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of sqlite 3.21.0:";;
+     short | recursive ) echo "Configuration of sqlite 3.26.0:";;
    esac
   cat <<\_ACEOF
 
@@ -1562,6 +1564,7 @@ Optional Features:
   --enable-fts5           Enable the FTS5 extension
   --enable-json1          Enable the JSON1 extension
   --enable-update-limit   Enable the UPDATE/DELETE LIMIT clause
+  --enable-geopoly        Enable the GEOPOLY extension
   --enable-rtree          Enable the RTREE extension
   --enable-session        Enable the SESSION extension
   --enable-gcov           Enable coverage testing using gcov
@@ -1654,7 +1657,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-sqlite configure 3.21.0
+sqlite configure 3.26.0
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -2073,7 +2076,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by sqlite $as_me 3.21.0, which was
+It was created by sqlite $as_me 3.26.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -3931,13 +3934,13 @@ if ${lt_cv_nm_interface+:} false; then :
 else
   lt_cv_nm_interface="BSD nm"
   echo "int some_variable = 0;" > conftest.$ac_ext
-  (eval echo "\"\$as_me:3934: $ac_compile\"" >&5)
+  (eval echo "\"\$as_me:3937: $ac_compile\"" >&5)
   (eval "$ac_compile" 2>conftest.err)
   cat conftest.err >&5
-  (eval echo "\"\$as_me:3937: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
+  (eval echo "\"\$as_me:3940: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
   (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out)
   cat conftest.err >&5
-  (eval echo "\"\$as_me:3940: output\"" >&5)
+  (eval echo "\"\$as_me:3943: output\"" >&5)
   cat conftest.out >&5
   if $GREP 'External.*some_variable' conftest.out > /dev/null; then
     lt_cv_nm_interface="MS dumpbin"
@@ -5143,7 +5146,7 @@ ia64-*-hpux*)
   ;;
 *-*-irix6*)
   # Find out which ABI we are using.
-  echo '#line 5146 "configure"' > conftest.$ac_ext
+  echo '#line 5149 "configure"' > conftest.$ac_ext
   if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5
   (eval $ac_compile) 2>&5
   ac_status=$?
@@ -6668,11 +6671,11 @@ else
    -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
    -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
    -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:6671: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:6674: $lt_compile\"" >&5)
    (eval "$lt_compile" 2>conftest.err)
    ac_status=$?
    cat conftest.err >&5
-   echo "$as_me:6675: \$? = $ac_status" >&5
+   echo "$as_me:6678: \$? = $ac_status" >&5
    if (exit $ac_status) && test -s "$ac_outfile"; then
      # The compiler can only warn and ignore the option if not recognized
      # So say no if there are warnings other than the usual output.
@@ -7007,11 +7010,11 @@ else
    -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
    -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
    -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7010: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7013: $lt_compile\"" >&5)
    (eval "$lt_compile" 2>conftest.err)
    ac_status=$?
    cat conftest.err >&5
-   echo "$as_me:7014: \$? = $ac_status" >&5
+   echo "$as_me:7017: \$? = $ac_status" >&5
    if (exit $ac_status) && test -s "$ac_outfile"; then
      # The compiler can only warn and ignore the option if not recognized
      # So say no if there are warnings other than the usual output.
@@ -7112,11 +7115,11 @@ else
    -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
    -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
    -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7115: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7118: $lt_compile\"" >&5)
    (eval "$lt_compile" 2>out/conftest.err)
    ac_status=$?
    cat out/conftest.err >&5
-   echo "$as_me:7119: \$? = $ac_status" >&5
+   echo "$as_me:7122: \$? = $ac_status" >&5
    if (exit $ac_status) && test -s out/conftest2.$ac_objext
    then
      # The compiler can only warn and ignore the option if not recognized
@@ -7167,11 +7170,11 @@ else
    -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
    -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
    -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7170: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7173: $lt_compile\"" >&5)
    (eval "$lt_compile" 2>out/conftest.err)
    ac_status=$?
    cat out/conftest.err >&5
-   echo "$as_me:7174: \$? = $ac_status" >&5
+   echo "$as_me:7177: \$? = $ac_status" >&5
    if (exit $ac_status) && test -s out/conftest2.$ac_objext
    then
      # The compiler can only warn and ignore the option if not recognized
@@ -9547,7 +9550,7 @@ else
   lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
   lt_status=$lt_dlunknown
   cat > conftest.$ac_ext <<_LT_EOF
-#line 9550 "configure"
+#line 9553 "configure"
 #include "confdefs.h"
 
 #if HAVE_DLFCN_H
@@ -9643,7 +9646,7 @@ else
   lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
   lt_status=$lt_dlunknown
   cat > conftest.$ac_ext <<_LT_EOF
-#line 9646 "configure"
+#line 9649 "configure"
 #include "confdefs.h"
 
 #if HAVE_DLFCN_H
@@ -10454,8 +10457,6 @@ fi
 # Check whether --enable-threadsafe was given.
 if test "${enable_threadsafe+set}" = set; then :
   enableval=$enable_threadsafe;
-else
-  enable_threadsafe=yes
 fi
 
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support threadsafe operation" >&5
@@ -11248,12 +11249,10 @@ fi
 # check for debug enabled
 # Check whether --enable-debug was given.
 if test "${enable_debug+set}" = set; then :
-  enableval=$enable_debug; use_debug=$enableval
+  enableval=$enable_debug;
-else
-  use_debug=no
 fi
 
-if test "${use_debug}" = "yes" ; then
+if test "${enable_debug}" = "yes" ; then
   TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0"
 else
   TARGET_DEBUG="-DNDEBUG"
@@ -11264,26 +11263,98 @@ fi
 # See whether we should use the amalgamation to build
 # Check whether --enable-amalgamation was given.
 if test "${enable_amalgamation+set}" = set; then :
-  enableval=$enable_amalgamation; use_amalgamation=$enableval
+  enableval=$enable_amalgamation;
-else
-  use_amalgamation=yes
 fi
 
-if test "${use_amalgamation}" != "yes" ; then
+if test "${enable_amalgamation}" == "no" ; then
   USE_AMALGAMATION=0
 fi
 
 
+#########
+# Look for zlib.  Only needed by extensions and by the sqlite3.exe shell
+for ac_header in zlib.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "zlib.h" "ac_cv_header_zlib_h" "$ac_includes_default"
+if test "x$ac_cv_header_zlib_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_ZLIB_H 1
+_ACEOF
+
+fi
+
+done
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing deflate" >&5
+$as_echo_n "checking for library containing deflate... " >&6; }
+if ${ac_cv_search_deflate+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  ac_func_search_save_LIBS=$LIBS
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+/* Override any GCC internal prototype to avoid an error.
+   Use char because int might match the return type of a GCC
+   builtin and then its argument prototype would still apply.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+char deflate ();
+int
+main ()
+{
+return deflate ();
+  ;
+  return 0;
+}
+_ACEOF
+for ac_lib in '' z; do
+  if test -z "$ac_lib"; then
+    ac_res="none required"
+  else
+    ac_res=-l$ac_lib
+    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
+  fi
+  if ac_fn_c_try_link "$LINENO"; then :
+  ac_cv_search_deflate=$ac_res
+fi
+rm -f core conftest.err conftest.$ac_objext \
+    conftest$ac_exeext
+  if ${ac_cv_search_deflate+:} false; then :
+  break
+fi
+done
+if ${ac_cv_search_deflate+:} false; then :
+
+else
+  ac_cv_search_deflate=no
+fi
+rm conftest.$ac_ext
+LIBS=$ac_func_search_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_deflate" >&5
+$as_echo "$ac_cv_search_deflate" >&6; }
+ac_res=$ac_cv_search_deflate
+if test "$ac_res" != no; then :
+  test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
+  HAVE_ZLIB="-DSQLITE_HAVE_ZLIB=1"
+else
+  HAVE_ZLIB=""
+fi
+
+
+
 #########
 # See whether we should allow loadable extensions
 # Check whether --enable-load-extension was given.
 if test "${enable_load_extension+set}" = set; then :
-  enableval=$enable_load_extension; use_loadextension=$enableval
+  enableval=$enable_load_extension;
 else
-  use_loadextension=yes
+  enable_load_extension=yes
 fi
 
-if test "${use_loadextension}" = "yes" ; then
+if test "${enable_load_extension}" = "yes" ; then
   OPT_FEATURE_FLAGS=""
   { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing dlopen" >&5
 $as_echo_n "checking for library containing dlopen... " >&6; }
@@ -11350,9 +11421,7 @@ fi
 #
 # Check whether --enable-memsys5 was given.
 if test "${enable_memsys5+set}" = set; then :
-  enableval=$enable_memsys5; enable_memsys5=yes
+  enableval=$enable_memsys5;
-else
-  enable_memsys5=no
 fi
 
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support MEMSYS5" >&5
@@ -11367,9 +11436,7 @@ $as_echo "no" >&6; }
 fi
 # Check whether --enable-memsys3 was given.
 if test "${enable_memsys3+set}" = set; then :
-  enableval=$enable_memsys3; enable_memsys3=yes
+  enableval=$enable_memsys3;
-else
-  enable_memsys3=no
 fi
 
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support MEMSYS3" >&5
@@ -11387,9 +11454,7 @@ fi
 # See whether we should enable Full Text Search extensions
 # Check whether --enable-fts3 was given.
 if test "${enable_fts3+set}" = set; then :
-  enableval=$enable_fts3; enable_fts3=yes
+  enableval=$enable_fts3;
-else
-  enable_fts3=no
 fi
 
 if test "${enable_fts3}" = "yes" ; then
@@ -11397,9 +11462,7 @@ if test "${enable_fts3}" = "yes" ; then
 fi
 # Check whether --enable-fts4 was given.
 if test "${enable_fts4+set}" = set; then :
-  enableval=$enable_fts4; enable_fts4=yes
+  enableval=$enable_fts4;
-else
-  enable_fts4=no
 fi
 
 if test "${enable_fts4}" = "yes" ; then
@@ -11463,9 +11526,7 @@ fi
 fi
 # Check whether --enable-fts5 was given.
 if test "${enable_fts5+set}" = set; then :
-  enableval=$enable_fts5; enable_fts5=yes
+  enableval=$enable_fts5;
-else
-  enable_fts5=no
 fi
 
 if test "${enable_fts5}" = "yes" ; then
@@ -11532,9 +11593,7 @@ fi
 # See whether we should enable JSON1
 # Check whether --enable-json1 was given.
 if test "${enable_json1+set}" = set; then :
-  enableval=$enable_json1; enable_json1=yes
+  enableval=$enable_json1;
-else
-  enable_json1=no
 fi
 
 if test "${enable_json1}" = "yes" ; then
@@ -11546,22 +11605,32 @@ fi
 # statements.
 # Check whether --enable-update-limit was given.
 if test "${enable_update_limit+set}" = set; then :
-  enableval=$enable_update_limit; enable_udlimit=yes
+  enableval=$enable_update_limit;
-else
-  enable_udlimit=no
 fi
 
 if test "${enable_udlimit}" = "yes" ; then
   OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT"
 fi
 
+#########
+# See whether we should enable GEOPOLY
+# Check whether --enable-geopoly was given.
+if test "${enable_geopoly+set}" = set; then :
+  enableval=$enable_geopoly; enable_geopoly=yes
+else
+  enable_geopoly=no
+fi
+
+if test "${enable_geopoly}" = "yes" ; then
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_GEOPOLY"
+  enable_rtree=yes
+fi
+
 #########
 # See whether we should enable RTREE
 # Check whether --enable-rtree was given.
 if test "${enable_rtree+set}" = set; then :
-  enableval=$enable_rtree; enable_rtree=yes
+  enableval=$enable_rtree;
-else
-  enable_rtree=no
 fi
 
 if test "${enable_rtree}" = "yes" ; then
@@ -11572,9 +11641,7 @@ fi
 # See whether we should enable the SESSION extension
 # Check whether --enable-session was given.
 if test "${enable_session+set}" = set; then :
-  enableval=$enable_session; enable_session=yes
+  enableval=$enable_session;
-else
-  enable_session=no
 fi
 
 if test "${enable_session}" = "yes" ; then
@@ -11637,9 +11704,7 @@ BUILD_CFLAGS=$ac_temp_BUILD_CFLAGS
 # See whether we should use GCOV
 # Check whether --enable-gcov was given.
 if test "${enable_gcov+set}" = set; then :
-  enableval=$enable_gcov; use_gcov=$enableval
+  enableval=$enable_gcov;
-else
-  use_gcov=no
 fi
 
 if test "${use_gcov}" = "yes" ; then
@@ -12167,7 +12232,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by sqlite $as_me 3.21.0, which was
+This file was extended by sqlite $as_me 3.26.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -12233,7 +12298,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-sqlite config.status 3.21.0
+sqlite config.status 3.26.0
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 

configure.ac

@@ -182,7 +182,7 @@ AC_SUBST(BUILD_CC)
 # Do we want to support multithreaded use of sqlite
 #
 AC_ARG_ENABLE(threadsafe, 
-AC_HELP_STRING([--disable-threadsafe],[Disable mutexing]),,enable_threadsafe=yes)
+AC_HELP_STRING([--disable-threadsafe],[Disable mutexing]))
 AC_MSG_CHECKING([whether to support threadsafe operation])
 if test "$enable_threadsafe" = "no"; then
   SQLITE_THREADSAFE=0
@@ -557,9 +557,8 @@ AC_SEARCH_LIBS(fdatasync, [rt])
 
 #########
 # check for debug enabled
-AC_ARG_ENABLE(debug, AC_HELP_STRING([--enable-debug],[enable debugging & verbose explain]),
+AC_ARG_ENABLE(debug, AC_HELP_STRING([--enable-debug],[enable debugging & verbose explain]))
-      [use_debug=$enableval],[use_debug=no])
+if test "${enable_debug}" = "yes" ; then
-if test "${use_debug}" = "yes" ; then
   TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0"
 else
   TARGET_DEBUG="-DNDEBUG"
@@ -569,19 +568,23 @@ AC_SUBST(TARGET_DEBUG)
 #########
 # See whether we should use the amalgamation to build
 AC_ARG_ENABLE(amalgamation, AC_HELP_STRING([--disable-amalgamation],
-      [Disable the amalgamation and instead build all files separately]),
+      [Disable the amalgamation and instead build all files separately]))
-      [use_amalgamation=$enableval],[use_amalgamation=yes])
+if test "${enable_amalgamation}" == "no" ; then
-if test "${use_amalgamation}" != "yes" ; then
   USE_AMALGAMATION=0
 fi
 AC_SUBST(USE_AMALGAMATION)
 
+#########
+# Look for zlib.  Only needed by extensions and by the sqlite3.exe shell
+AC_CHECK_HEADERS(zlib.h)
+AC_SEARCH_LIBS(deflate, z, [HAVE_ZLIB="-DSQLITE_HAVE_ZLIB=1"], [HAVE_ZLIB=""])
+AC_SUBST(HAVE_ZLIB)
+
 #########
 # See whether we should allow loadable extensions
 AC_ARG_ENABLE(load-extension, AC_HELP_STRING([--disable-load-extension],
-      [Disable loading of external extensions]),
+      [Disable loading of external extensions]),,[enable_load_extension=yes])
-      [use_loadextension=$enableval],[use_loadextension=yes])
+if test "${enable_load_extension}" = "yes" ; then
-if test "${use_loadextension}" = "yes" ; then
   OPT_FEATURE_FLAGS=""
   AC_SEARCH_LIBS(dlopen, dl)
 else
@@ -592,8 +595,7 @@ fi
 # Do we want to support memsys3 and/or memsys5
 #
 AC_ARG_ENABLE(memsys5, 
-  AC_HELP_STRING([--enable-memsys5],[Enable MEMSYS5]),
+  AC_HELP_STRING([--enable-memsys5],[Enable MEMSYS5]))
-  [enable_memsys5=yes],[enable_memsys5=no])
 AC_MSG_CHECKING([whether to support MEMSYS5])
 if test "${enable_memsys5}" = "yes"; then
   OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS5"
@@ -602,8 +604,7 @@ else
   AC_MSG_RESULT([no])
 fi
 AC_ARG_ENABLE(memsys3, 
-  AC_HELP_STRING([--enable-memsys3],[Enable MEMSYS3]),
+  AC_HELP_STRING([--enable-memsys3],[Enable MEMSYS3]))
-  [enable_memsys3=yes],[enable_memsys3=no])
 AC_MSG_CHECKING([whether to support MEMSYS3])
 if test "${enable_memsys3}" = "yes" -a "${enable_memsys5}" = "no"; then
   OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS3"
@@ -615,21 +616,18 @@ fi
 #########
 # See whether we should enable Full Text Search extensions
 AC_ARG_ENABLE(fts3, AC_HELP_STRING([--enable-fts3],
-      [Enable the FTS3 extension]),
+      [Enable the FTS3 extension]))
-      [enable_fts3=yes],[enable_fts3=no])
 if test "${enable_fts3}" = "yes" ; then
   OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS3"
 fi
 AC_ARG_ENABLE(fts4, AC_HELP_STRING([--enable-fts4],
-      [Enable the FTS4 extension]),
+      [Enable the FTS4 extension]))
-      [enable_fts4=yes],[enable_fts4=no])
 if test "${enable_fts4}" = "yes" ; then
   OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS4"
   AC_SEARCH_LIBS([log],[m])
 fi
 AC_ARG_ENABLE(fts5, AC_HELP_STRING([--enable-fts5],
-      [Enable the FTS5 extension]),
+      [Enable the FTS5 extension]))
-      [enable_fts5=yes],[enable_fts5=no])
 if test "${enable_fts5}" = "yes" ; then
   OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS5"
   AC_SEARCH_LIBS([log],[m])
@@ -637,9 +635,7 @@ fi
 
 #########
 # See whether we should enable JSON1
-AC_ARG_ENABLE(json1, AC_HELP_STRING([--enable-json1],
+AC_ARG_ENABLE(json1, AC_HELP_STRING([--enable-json1],[Enable the JSON1 extension]))
-      [Enable the JSON1 extension]),
-      [enable_json1=yes],[enable_json1=no])
 if test "${enable_json1}" = "yes" ; then
   OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_JSON1"
 fi
@@ -648,17 +644,25 @@ fi
 # See whether we should enable the LIMIT clause on UPDATE and DELETE
 # statements.
 AC_ARG_ENABLE(update-limit, AC_HELP_STRING([--enable-update-limit],
-      [Enable the UPDATE/DELETE LIMIT clause]),
+      [Enable the UPDATE/DELETE LIMIT clause]))
-      [enable_udlimit=yes],[enable_udlimit=no])
 if test "${enable_udlimit}" = "yes" ; then
   OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT"
 fi
 
+#########
+# See whether we should enable GEOPOLY
+AC_ARG_ENABLE(geopoly, AC_HELP_STRING([--enable-geopoly],
+      [Enable the GEOPOLY extension]),
+      [enable_geopoly=yes],[enable_geopoly=no])
+if test "${enable_geopoly}" = "yes" ; then
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_GEOPOLY"
+  enable_rtree=yes
+fi
+
 #########
 # See whether we should enable RTREE
 AC_ARG_ENABLE(rtree, AC_HELP_STRING([--enable-rtree],
-      [Enable the RTREE extension]),
+      [Enable the RTREE extension]))
-      [enable_rtree=yes],[enable_rtree=no])
 if test "${enable_rtree}" = "yes" ; then
   OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_RTREE"
 fi
@@ -666,8 +670,7 @@ fi
 #########
 # See whether we should enable the SESSION extension
 AC_ARG_ENABLE(session, AC_HELP_STRING([--enable-session],
-      [Enable the SESSION extension]),
+      [Enable the SESSION extension]))
-      [enable_session=yes],[enable_session=no])
 if test "${enable_session}" = "yes" ; then
   OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_SESSION"
   OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_PREUPDATE_HOOK"
@@ -727,8 +730,7 @@ BUILD_CFLAGS=$ac_temp_BUILD_CFLAGS
 #########
 # See whether we should use GCOV
 AC_ARG_ENABLE(gcov, AC_HELP_STRING([--enable-gcov],
-      [Enable coverage testing using gcov]),
+      [Enable coverage testing using gcov]))
-      [use_gcov=$enableval],[use_gcov=no])
 if test "${use_gcov}" = "yes" ; then
   USE_GCOV=1
 else

doc/F2FS.txt

@@ -0,0 +1,87 @@
+
+SQLite's OS layer contains the following definitions used in F2FS related
+calls:
+
+#define F2FS_IOCTL_MAGIC        0xf5
+#define F2FS_IOC_START_ATOMIC_WRITE     _IO(F2FS_IOCTL_MAGIC, 1)
+#define F2FS_IOC_COMMIT_ATOMIC_WRITE    _IO(F2FS_IOCTL_MAGIC, 2)
+#define F2FS_IOC_START_VOLATILE_WRITE   _IO(F2FS_IOCTL_MAGIC, 3)
+#define F2FS_IOC_ABORT_VOLATILE_WRITE   _IO(F2FS_IOCTL_MAGIC, 5)
+#define F2FS_IOC_GET_FEATURES           _IOR(F2FS_IOCTL_MAGIC, 12, u32)
+#define F2FS_FEATURE_ATOMIC_WRITE       0x0004
+
+After opening a database file on Linux (including Android), SQLite determines
+whether or not a file supports F2FS atomic commits as follows:
+
+  u32 flags = 0;
+  rc = ioctl(fd, F2FS_IOC_GET_FEATURES, &flags);
+  if( rc==0 && (flags & F2FS_FEATURE_ATOMIC_WRITE) ){
+    /* File supports F2FS atomic commits */
+  }else{
+    /* File does NOT support F2FS atomic commits */
+  }
+
+where "fd" is the file-descriptor open on the database file.
+
+Usually, when writing to a database file that supports atomic commits, SQLite
+accumulates the entire transaction in heap memory, deferring all writes to the
+db file until the transaction is committed.
+
+When it is time to commit a transaction on a file that supports atomic
+commits, SQLite does:
+
+  /* Take an F_WRLCK lock on the database file. This prevents any other
+  ** SQLite clients from reading or writing the file until the lock
+  ** is released.  */
+  rc = fcntl(fd, F_SETLK, ...);
+  if( rc!=0 ) goto failed;
+
+  rc = ioctl(fd, F2FS_IOC_START_ATOMIC_WRITE);
+  if( rc!=0 ) goto fallback_to_legacy_journal_commit;
+
+  foreach (dirty page){
+    rc = write(fd, ...dirty page...);
+    if( rc!=0 ){
+      ioctl(fd, F2FS_IOC_ABORT_VOLATILE_WRITE);
+      goto fallback_to_legacy_journal_commit;
+    }
+  }
+
+  rc = ioctl(fd, F2FS_IOC_COMMIT_ATOMIC_WRITE);
+  if( rc!=0 ){
+    ioctl(fd, F2FS_IOC_ABORT_VOLATILE_WRITE);
+    goto fallback_to_legacy_journal_commit;
+  }
+
+  /* If we get there, the transaction has been successfully 
+  ** committed to persistent storage. The following call
+  ** relinquishes the F_WRLCK lock.  */
+  fcntl(fd, F_SETLK, ...);
+
+Assumptions:
+
+1. After either of the F2FS_IOC_ABORT_VOLATILE_WRITE calls return,
+   the database file is in the state that it was in before
+   F2FS_IOC_START_ATOMIC_WRITE was invoked. Even if the ioctl()
+   fails - we're ignoring the return code.
+
+   This is true regardless of the type of error that occurred in
+   ioctl() or write().
+
+2. If the system fails before the F2FS_IOC_COMMIT_ATOMIC_WRITE is
+   completed, then following a reboot the database file is in the
+   state that it was in before F2FS_IOC_START_ATOMIC_WRITE was invoked.
+   Or, if the write was commited right before the system failed, in a 
+   state indicating that all write() calls were successfully committed
+   to persistent storage before the failure occurred.
+
+3. If the process crashes before the F2FS_IOC_COMMIT_ATOMIC_WRITE is
+   completed then the file is automatically restored to the state that
+   it was in before F2FS_IOC_START_ATOMIC_WRITE was called. This occurs
+   before the posix advisory lock is automatically dropped - there is
+   no chance that another client will be able to read the file in a
+   half-committed state before the rollback operation occurs.
+
+
+
+

doc/lemon.html

@@ -99,6 +99,9 @@ Show only the basis for each parser state in the report file.
 <li><b>-c</b>
 Do not compress the generated action tables.  The parser will be a
 little larger and slower, but it will detect syntax errors sooner.
+<li><b>-d</b><i>directory</i>
+Write all output files into <i>directory</i>.  Normally, output files
+are written into the directory that contains the input grammar file.
 <li><b>-D<i>name</i></b>
 Define C preprocessor macro <i>name</i>.  This macro is usable by
 "<tt><a href='#pifdef'>%ifdef</a></tt>" and
@@ -679,6 +682,30 @@ of type "MyStruct*" and all action routines will have access to
 a variable named "pAbc" that is the value of the 4th parameter
 in the most recent call to Parse().</p>
 
+<p>The <tt>%extra_context</tt> directive works the same except that it
+is passed in on the ParseAlloc() or ParseInit() routines instead of
+on Parse().
+
+<a name='extractx'></a>
+<h4>The <tt>%extra_context</tt> directive</h4>
+
+The <tt>%extra_context</tt> directive instructs Lemon to add a 2th parameter
+to the parameter list of the ParseAlloc() and ParseInif() functions.  Lemon
+doesn't do anything itself with these extra argument, but it does
+store the value make it available to C-code action routines, destructors,
+and so forth.  For example, if the grammar file contains:</p>
+
+<p><pre>
+    %extra_context { MyStruct *pAbc }
+</pre></p>
+
+<p>Then the ParseAlloc() and ParseInit() functions will have an 2th parameter
+of type "MyStruct*" and all action routines will have access to
+a variable named "pAbc" that is the value of that 2th parameter.</p>
+
+<p>The <tt>%extra_argument</tt> directive works the same except that it
+is passed in on the Parse() routine instead of on ParseAlloc()/ParseInit().
+
 <a name='pfallback'></a>
 <h4>The <tt>%fallback</tt> directive</h4>
 
@@ -750,6 +777,9 @@ For example:</p>
 <p>This might be needed, for example, if some of the C actions in the
 grammar call functions that are prototyped in unistd.h.</p>
 
+<p>Use the <tt><a href="#pcode">%code</a></tt> directive to add code to
+the end of the generated parser.</p>
+
 <a name='pleft'></a>
 <h4>The <tt>%left</tt> directive</h4>
 

ext/expert/README.md

@@ -0,0 +1,83 @@
+## SQLite Expert Extension
+
+This folder contains code for a simple system to propose useful indexes
+given a database and a set of SQL queries. It works as follows:
+
+  1. The user database schema is copied to a temporary database.
+
+  1. All SQL queries are prepared against the temporary database.
+     Information regarding the WHERE and ORDER BY clauses, and other query
+     features that affect index selection are recorded.
+
+  1. The information gathered in step 2 is used to create candidate 
+     indexes - indexes that the planner might have made use of in the previous
+     step, had they been available.
+
+  1. A subset of the data in the user database is used to generate statistics
+     for all existing indexes and the candidate indexes generated in step 3
+     above.
+
+  1. The SQL queries are prepared a second time. If the planner uses any
+     of the indexes created in step 3, they are recommended to the user.
+
+# C API
+
+The SQLite expert C API is defined in sqlite3expert.h. Most uses will proceed
+as follows:
+
+  1. An sqlite3expert object is created by calling **sqlite3\_expert\_new()**.
+     A database handle opened by the user is passed as an argument.
+
+  1. The sqlite3expert object is configured with one or more SQL statements
+     by making one or more calls to **sqlite3\_expert\_sql()**. Each call may
+     specify a single SQL statement, or multiple statements separated by
+     semi-colons.
+  
+  1. Optionally, the **sqlite3\_expert\_config()** API may be used to 
+     configure the size of the data subset used to generate index statistics.
+     Using a smaller subset of the data can speed up the analysis.
+
+  1. **sqlite3\_expert\_analyze()** is called to run the analysis.
+
+  1. One or more calls are made to **sqlite3\_expert\_report()** to extract
+     components of the results of the analysis.
+
+  1. **sqlite3\_expert\_destroy()** is called to free all resources.
+
+Refer to comments in sqlite3expert.h for further details.
+
+# sqlite3_expert application
+
+The file "expert.c" contains the code for a command line application that
+uses the API described above. It can be compiled with (for example):
+
+<pre>
+  gcc -O2 sqlite3.c expert.c sqlite3expert.c -o sqlite3_expert
+</pre>
+
+Assuming the database is named "test.db", it can then be run to analyze a
+single query:
+
+<pre>
+  ./sqlite3_expert -sql &lt;sql-query&gt; test.db
+</pre>
+
+Or an entire text file worth of queries with:
+
+<pre>
+  ./sqlite3_expert -file &lt;text-file&gt; test.db
+</pre>
+
+By default, sqlite3\_expert generates index statistics using all the data in
+the user database. For a large database, this may be prohibitively time
+consuming. The "-sample" option may be used to configure sqlite3\_expert to
+generate statistics based on an integer percentage of the user database as
+follows:
+
+<pre>
+  # Generate statistics based on 25% of the user database rows:
+  ./sqlite3_expert -sample 25 -sql &lt;sql-query&gt; test.db
+
+  # Do not generate any statistics at all:
+  ./sqlite3_expert -sample 0 -sql &lt;sql-query&gt; test.db
+</pre>

ext/expert/expert.c

@@ -0,0 +1,156 @@
+/*
+** 2017 April 07
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+*/
+
+
+#include <sqlite3.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "sqlite3expert.h"
+
+
+static void option_requires_argument(const char *zOpt){
+  fprintf(stderr, "Option requires an argument: %s\n", zOpt);
+  exit(-3);
+}
+
+static int option_integer_arg(const char *zVal){
+  return atoi(zVal);
+}
+
+static void usage(char **argv){
+  fprintf(stderr, "\n");
+  fprintf(stderr, "Usage %s ?OPTIONS? DATABASE\n", argv[0]);
+  fprintf(stderr, "\n");
+  fprintf(stderr, "Options are:\n");
+  fprintf(stderr, "  -sql SQL   (analyze SQL statements passed as argument)\n");
+  fprintf(stderr, "  -file FILE (read SQL statements from file FILE)\n");
+  fprintf(stderr, "  -verbose LEVEL (integer verbosity level. default 1)\n");
+  fprintf(stderr, "  -sample PERCENT (percent of db to sample. default 100)\n");
+  exit(-1);
+}
+
+static int readSqlFromFile(sqlite3expert *p, const char *zFile, char **pzErr){
+  FILE *in = fopen(zFile, "rb");
+  long nIn;
+  size_t nRead;
+  char *pBuf;
+  int rc;
+  if( in==0 ){
+    *pzErr = sqlite3_mprintf("failed to open file %s\n", zFile);
+    return SQLITE_ERROR;
+  }
+  fseek(in, 0, SEEK_END);
+  nIn = ftell(in);
+  rewind(in);
+  pBuf = sqlite3_malloc64( nIn+1 );
+  nRead = fread(pBuf, nIn, 1, in);
+  fclose(in);
+  if( nRead!=1 ){
+    sqlite3_free(pBuf);
+    *pzErr = sqlite3_mprintf("failed to read file %s\n", zFile);
+    return SQLITE_ERROR;
+  }
+  pBuf[nIn] = 0;
+  rc = sqlite3_expert_sql(p, pBuf, pzErr);
+  sqlite3_free(pBuf);
+  return rc;
+}
+
+int main(int argc, char **argv){
+  const char *zDb;
+  int rc = 0;
+  char *zErr = 0;
+  int i;
+  int iVerbose = 1;               /* -verbose option */
+
+  sqlite3 *db = 0;
+  sqlite3expert *p = 0;
+
+  if( argc<2 ) usage(argv);
+  zDb = argv[argc-1];
+  if( zDb[0]=='-' ) usage(argv);
+  rc = sqlite3_open(zDb, &db);
+  if( rc!=SQLITE_OK ){
+    fprintf(stderr, "Cannot open db file: %s - %s\n", zDb, sqlite3_errmsg(db));
+    exit(-2);
+  }
+
+  p = sqlite3_expert_new(db, &zErr);
+  if( p==0 ){
+    fprintf(stderr, "Cannot run analysis: %s\n", zErr);
+    rc = 1;
+  }else{
+    for(i=1; i<(argc-1); i++){
+      char *zArg = argv[i];
+      int nArg;
+      if( zArg[0]=='-' && zArg[1]=='-' && zArg[2]!=0 ) zArg++;
+      nArg = (int)strlen(zArg);
+      if( nArg>=2 && 0==sqlite3_strnicmp(zArg, "-file", nArg) ){
+        if( ++i==(argc-1) ) option_requires_argument("-file");
+        rc = readSqlFromFile(p, argv[i], &zErr);
+      }
+
+      else if( nArg>=3 && 0==sqlite3_strnicmp(zArg, "-sql", nArg) ){
+        if( ++i==(argc-1) ) option_requires_argument("-sql");
+        rc = sqlite3_expert_sql(p, argv[i], &zErr);
+      }
+
+      else if( nArg>=3 && 0==sqlite3_strnicmp(zArg, "-sample", nArg) ){
+        int iSample;
+        if( ++i==(argc-1) ) option_requires_argument("-sample");
+        iSample = option_integer_arg(argv[i]);
+        sqlite3_expert_config(p, EXPERT_CONFIG_SAMPLE, iSample);
+      }
+
+      else if( nArg>=2 && 0==sqlite3_strnicmp(zArg, "-verbose", nArg) ){
+        if( ++i==(argc-1) ) option_requires_argument("-verbose");
+        iVerbose = option_integer_arg(argv[i]);
+      }
+
+      else{
+        usage(argv);
+      }
+    }
+  }
+
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_expert_analyze(p, &zErr);
+  }
+
+  if( rc==SQLITE_OK ){
+    int nQuery = sqlite3_expert_count(p);
+    if( iVerbose>0 ){
+      const char *zCand = sqlite3_expert_report(p,0,EXPERT_REPORT_CANDIDATES);
+      fprintf(stdout, "-- Candidates -------------------------------\n");
+      fprintf(stdout, "%s\n", zCand);
+    }
+    for(i=0; i<nQuery; i++){
+      const char *zSql = sqlite3_expert_report(p, i, EXPERT_REPORT_SQL);
+      const char *zIdx = sqlite3_expert_report(p, i, EXPERT_REPORT_INDEXES);
+      const char *zEQP = sqlite3_expert_report(p, i, EXPERT_REPORT_PLAN);
+      if( zIdx==0 ) zIdx = "(no new indexes)\n";
+      if( iVerbose>0 ){
+        fprintf(stdout, "-- Query %d ----------------------------------\n",i+1);
+        fprintf(stdout, "%s\n\n", zSql);
+      }
+      fprintf(stdout, "%s\n%s\n", zIdx, zEQP);
+    }
+  }else{
+    fprintf(stderr, "Error: %s\n", zErr ? zErr : "?");
+  }
+
+  sqlite3_expert_destroy(p);
+  sqlite3_free(zErr);
+  return rc;
+}

ext/expert/expert1.test

@@ -0,0 +1,382 @@
+# 2009 Nov 11
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# The focus of this file is testing the CLI shell tool. Specifically,
+# the ".recommend" command.
+#
+#
+
+# Test plan:
+#
+#
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+}
+source $testdir/tester.tcl
+set testprefix expert1
+
+if {[info commands sqlite3_expert_new]==""} {
+  finish_test
+  return
+}
+
+set CLI [test_binary_name sqlite3]
+set CMD [test_binary_name sqlite3_expert]
+
+proc squish {txt} {
+  regsub -all {[[:space:]]+} $txt { }
+}
+
+proc do_setup_rec_test {tn setup sql res} {
+  reset_db
+  db eval $setup
+  uplevel [list do_rec_test $tn $sql $res]
+}
+
+foreach {tn setup} {
+  1 {
+    if {![file executable $CMD]} { continue }
+
+    proc do_rec_test {tn sql res} {
+      set res [squish [string trim $res]]
+      set tst [subst -nocommands { 
+        squish [string trim [exec $::CMD -verbose 0 -sql {$sql;} test.db]]
+      }]
+      uplevel [list do_test $tn $tst $res]
+    }
+  }
+  2 {
+    if {[info commands sqlite3_expert_new]==""} { continue }
+
+    proc do_rec_test {tn sql res} {
+      set expert [sqlite3_expert_new db]
+      $expert sql $sql
+      $expert analyze
+
+      set result [list]
+      for {set i 0} {$i < [$expert count]} {incr i} {
+        set idx [string trim [$expert report $i indexes]]
+        if {$idx==""} {set idx "(no new indexes)"}
+        lappend result $idx
+        lappend result [string trim [$expert report $i plan]]
+      }
+
+      $expert destroy
+
+      set tst [subst -nocommands {set {} [squish [join {$result}]]}]
+      uplevel [list do_test $tn $tst [string trim [squish $res]]]
+    }
+  }
+  3 {
+    if {![file executable $CLI]} { continue }
+
+    proc do_rec_test {tn sql res} {
+      set res [squish [string trim $res]]
+      set tst [subst -nocommands { 
+        squish [string trim [exec $::CLI test.db ".expert" {$sql;}]]
+      }]
+      uplevel [list do_test $tn $tst $res]
+    }
+  }
+} {
+
+  eval $setup
+
+
+do_setup_rec_test $tn.1 { CREATE TABLE t1(a, b, c) } {
+  SELECT * FROM t1
+} {
+  (no new indexes)
+  SCAN TABLE t1
+}
+
+do_setup_rec_test $tn.2 {
+  CREATE TABLE t1(a, b, c);
+} {
+  SELECT * FROM t1 WHERE b>?;
+} {
+  CREATE INDEX t1_idx_00000062 ON t1(b);
+  SEARCH TABLE t1 USING INDEX t1_idx_00000062 (b>?)
+}
+
+do_setup_rec_test $tn.3 {
+  CREATE TABLE t1(a, b, c);
+} {
+  SELECT * FROM t1 WHERE b COLLATE nocase BETWEEN ? AND ?
+} {
+  CREATE INDEX t1_idx_3e094c27 ON t1(b COLLATE NOCASE);
+  SEARCH TABLE t1 USING INDEX t1_idx_3e094c27 (b>? AND b<?)
+}
+
+do_setup_rec_test $tn.4 {
+  CREATE TABLE t1(a, b, c);
+} {
+  SELECT a FROM t1 ORDER BY b;
+} {
+  CREATE INDEX t1_idx_00000062 ON t1(b);
+  SCAN TABLE t1 USING INDEX t1_idx_00000062
+}
+
+do_setup_rec_test $tn.5 {
+  CREATE TABLE t1(a, b, c);
+} {
+  SELECT a FROM t1 WHERE a=? ORDER BY b;
+} {
+  CREATE INDEX t1_idx_000123a7 ON t1(a, b);
+  SEARCH TABLE t1 USING COVERING INDEX t1_idx_000123a7 (a=?)
+}
+
+do_setup_rec_test $tn.6 {
+  CREATE TABLE t1(a, b, c);
+} {
+  SELECT min(a) FROM t1
+} {
+  CREATE INDEX t1_idx_00000061 ON t1(a);
+  SEARCH TABLE t1 USING COVERING INDEX t1_idx_00000061
+}
+
+do_setup_rec_test $tn.7 {
+  CREATE TABLE t1(a, b, c);
+} {
+  SELECT * FROM t1 ORDER BY a, b, c;
+} {
+  CREATE INDEX t1_idx_033e95fe ON t1(a, b, c);
+  SCAN TABLE t1 USING COVERING INDEX t1_idx_033e95fe
+}
+
+#do_setup_rec_test $tn.1.8 {
+#  CREATE TABLE t1(a, b, c);
+#} {
+#  SELECT * FROM t1 ORDER BY a ASC, b COLLATE nocase DESC, c ASC;
+#} {
+#  CREATE INDEX t1_idx_5be6e222 ON t1(a, b COLLATE NOCASE DESC, c);
+#  0|0|0|SCAN TABLE t1 USING COVERING INDEX t1_idx_5be6e222
+#}
+
+do_setup_rec_test $tn.8.1 {
+  CREATE TABLE t1(a COLLATE NOCase, b, c);
+} {
+  SELECT * FROM t1 WHERE a=?
+} {
+  CREATE INDEX t1_idx_00000061 ON t1(a);
+  SEARCH TABLE t1 USING INDEX t1_idx_00000061 (a=?)
+}
+do_setup_rec_test $tn.8.2 {
+  CREATE TABLE t1(a, b COLLATE nocase, c);
+} {
+  SELECT * FROM t1 ORDER BY a ASC, b DESC, c ASC;
+} {
+  CREATE INDEX t1_idx_5cb97285 ON t1(a, b DESC, c);
+  SCAN TABLE t1 USING COVERING INDEX t1_idx_5cb97285
+}
+
+
+# Tables with names that require quotes.
+#
+do_setup_rec_test $tn.9.1 {
+  CREATE TABLE "t t"(a, b, c);
+} {
+  SELECT * FROM "t t" WHERE a=?
+} {
+  CREATE INDEX 't t_idx_00000061' ON 't t'(a);
+  SEARCH TABLE t t USING INDEX t t_idx_00000061 (a=?) 
+}
+
+do_setup_rec_test $tn.9.2 {
+  CREATE TABLE "t t"(a, b, c);
+} {
+  SELECT * FROM "t t" WHERE b BETWEEN ? AND ?
+} {
+  CREATE INDEX 't t_idx_00000062' ON 't t'(b);
+  SEARCH TABLE t t USING INDEX t t_idx_00000062 (b>? AND b<?)
+}
+
+# Columns with names that require quotes.
+#
+do_setup_rec_test $tn.10.1 {
+  CREATE TABLE t3(a, "b b", c);
+} {
+  SELECT * FROM t3 WHERE "b b" = ?
+} {
+  CREATE INDEX t3_idx_00050c52 ON t3('b b');
+  SEARCH TABLE t3 USING INDEX t3_idx_00050c52 (b b=?)
+}
+
+do_setup_rec_test $tn.10.2 {
+  CREATE TABLE t3(a, "b b", c);
+} {
+  SELECT * FROM t3 ORDER BY "b b"
+} {
+  CREATE INDEX t3_idx_00050c52 ON t3('b b');
+  SCAN TABLE t3 USING INDEX t3_idx_00050c52
+}
+
+# Transitive constraints
+#
+do_setup_rec_test $tn.11.1 {
+  CREATE TABLE t5(a, b);
+  CREATE TABLE t6(c, d);
+} {
+  SELECT * FROM t5, t6 WHERE a=? AND b=c AND c=?
+} {
+  CREATE INDEX t5_idx_000123a7 ON t5(a, b);
+  CREATE INDEX t6_idx_00000063 ON t6(c);
+  SEARCH TABLE t6 USING INDEX t6_idx_00000063 (c=?) 
+  SEARCH TABLE t5 USING COVERING INDEX t5_idx_000123a7 (a=? AND b=?)
+}
+
+# OR terms.
+#
+do_setup_rec_test $tn.12.1 {
+  CREATE TABLE t7(a, b);
+} {
+  SELECT * FROM t7 WHERE a=? OR b=?
+} {
+  CREATE INDEX t7_idx_00000062 ON t7(b);
+  CREATE INDEX t7_idx_00000061 ON t7(a);
+  MULTI-INDEX OR
+    SEARCH TABLE t7 USING INDEX t7_idx_00000061 (a=?) 
+    SEARCH TABLE t7 USING INDEX t7_idx_00000062 (b=?)
+}
+
+# rowid terms.
+#
+do_setup_rec_test $tn.13.1 {
+  CREATE TABLE t8(a, b);
+} {
+  SELECT * FROM t8 WHERE rowid=?
+} {
+  (no new indexes)
+  SEARCH TABLE t8 USING INTEGER PRIMARY KEY (rowid=?)
+}
+do_setup_rec_test $tn.13.2 {
+  CREATE TABLE t8(a, b);
+} {
+  SELECT * FROM t8 ORDER BY rowid
+} {
+  (no new indexes)
+  SCAN TABLE t8
+}
+do_setup_rec_test $tn.13.3 {
+  CREATE TABLE t8(a, b);
+} {
+  SELECT * FROM t8 WHERE a=? ORDER BY rowid
+} {
+  CREATE INDEX t8_idx_00000061 ON t8(a); 
+  SEARCH TABLE t8 USING INDEX t8_idx_00000061 (a=?)
+}
+
+# Triggers
+#
+do_setup_rec_test $tn.14 {
+  CREATE TABLE t9(a, b, c);
+  CREATE TABLE t10(a, b, c);
+  CREATE TRIGGER t9t AFTER INSERT ON t9 BEGIN
+    UPDATE t10 SET a=new.a WHERE b = new.b;
+  END;
+} {
+  INSERT INTO t9 VALUES(?, ?, ?);
+} {
+  CREATE INDEX t10_idx_00000062 ON t10(b); 
+  SEARCH TABLE t10 USING INDEX t10_idx_00000062 (b=?)
+}
+
+do_setup_rec_test $tn.15 {
+  CREATE TABLE t1(a, b);
+  CREATE TABLE t2(c, d);
+
+  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
+  INSERT INTO t1 SELECT (i-1)/50, (i-1)/20 FROM s;
+
+  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
+  INSERT INTO t2 SELECT (i-1)/20, (i-1)/5 FROM s;
+} {
+  SELECT * FROM t2, t1 WHERE b=? AND d=? AND t2.rowid=t1.rowid
+} {
+  CREATE INDEX t2_idx_00000064 ON t2(d);
+  SEARCH TABLE t2 USING INDEX t2_idx_00000064 (d=?) 
+  SEARCH TABLE t1 USING INTEGER PRIMARY KEY (rowid=?)
+}
+
+do_setup_rec_test $tn.16 {
+  CREATE TABLE t1(a, b);
+} {
+  SELECT * FROM t1 WHERE b IS NOT NULL;
+} {
+  (no new indexes)
+  SCAN TABLE t1
+}
+
+}
+
+proc do_candidates_test {tn sql res} {
+  set res [squish [string trim $res]]
+
+  set expert [sqlite3_expert_new db]
+  $expert sql $sql
+  $expert analyze
+
+  set candidates [squish [string trim [$expert report 0 candidates]]]
+  $expert destroy
+
+  uplevel [list do_test $tn [list set {} $candidates] $res]
+}
+
+
+reset_db
+do_execsql_test 4.0 {
+  CREATE TABLE t1(a, b);
+  CREATE TABLE t2(c, d);
+
+  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
+  INSERT INTO t1 SELECT (i-1)/50, (i-1)/20 FROM s;
+
+  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
+  INSERT INTO t2 SELECT (i-1)/20, (i-1)/5 FROM s;
+}
+do_candidates_test 4.1 {
+  SELECT * FROM t1,t2 WHERE (b=? OR a=?) AND (c=? OR d=?)
+} {
+  CREATE INDEX t1_idx_00000062 ON t1(b); -- stat1: 100 20 
+  CREATE INDEX t1_idx_00000061 ON t1(a); -- stat1: 100 50 
+  CREATE INDEX t2_idx_00000063 ON t2(c); -- stat1: 100 20 
+  CREATE INDEX t2_idx_00000064 ON t2(d); -- stat1: 100 5
+}
+
+do_candidates_test 4.2 {
+  SELECT * FROM t1,t2 WHERE a=? AND b=? AND c=? AND d=?
+} {
+  CREATE INDEX t1_idx_000123a7 ON t1(a, b); -- stat1: 100 50 17
+  CREATE INDEX t2_idx_0001295b ON t2(c, d); -- stat1: 100 20 5
+}
+
+do_execsql_test 4.3 {
+  CREATE INDEX t1_idx_00000061 ON t1(a); -- stat1: 100 50 
+  CREATE INDEX t1_idx_00000062 ON t1(b); -- stat1: 100 20 
+  CREATE INDEX t1_idx_000123a7 ON t1(a, b); -- stat1: 100 50 16
+
+  CREATE INDEX t2_idx_00000063 ON t2(c); -- stat1: 100 20 
+  CREATE INDEX t2_idx_00000064 ON t2(d); -- stat1: 100 5
+  CREATE INDEX t2_idx_0001295b ON t2(c, d); -- stat1: 100 20 5
+
+  ANALYZE;
+  SELECT * FROM sqlite_stat1 ORDER BY 1, 2;
+} {
+  t1 t1_idx_00000061 {100 50} 
+  t1 t1_idx_00000062 {100 20}
+  t1 t1_idx_000123a7 {100 50 17}
+  t2 t2_idx_00000063 {100 20} 
+  t2 t2_idx_00000064 {100 5} 
+  t2 t2_idx_0001295b {100 20 5}
+}
+
+
+finish_test

ext/expert/sqlite3expert.h

@@ -0,0 +1,168 @@
+/*
+** 2017 April 07
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+*/
+
+
+#include "sqlite3.h"
+
+typedef struct sqlite3expert sqlite3expert;
+
+/*
+** Create a new sqlite3expert object.
+**
+** If successful, a pointer to the new object is returned and (*pzErr) set
+** to NULL. Or, if an error occurs, NULL is returned and (*pzErr) set to
+** an English-language error message. In this case it is the responsibility
+** of the caller to eventually free the error message buffer using
+** sqlite3_free().
+*/
+sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErr);
+
+/*
+** Configure an sqlite3expert object.
+**
+** EXPERT_CONFIG_SAMPLE:
+**   By default, sqlite3_expert_analyze() generates sqlite_stat1 data for
+**   each candidate index. This involves scanning and sorting the entire
+**   contents of each user database table once for each candidate index
+**   associated with the table. For large databases, this can be 
+**   prohibitively slow. This option allows the sqlite3expert object to
+**   be configured so that sqlite_stat1 data is instead generated based on a
+**   subset of each table, or so that no sqlite_stat1 data is used at all.
+**
+**   A single integer argument is passed to this option. If the value is less
+**   than or equal to zero, then no sqlite_stat1 data is generated or used by
+**   the analysis - indexes are recommended based on the database schema only.
+**   Or, if the value is 100 or greater, complete sqlite_stat1 data is
+**   generated for each candidate index (this is the default). Finally, if the
+**   value falls between 0 and 100, then it represents the percentage of user
+**   table rows that should be considered when generating sqlite_stat1 data.
+**
+**   Examples:
+**
+**     // Do not generate any sqlite_stat1 data
+**     sqlite3_expert_config(pExpert, EXPERT_CONFIG_SAMPLE, 0);
+**
+**     // Generate sqlite_stat1 data based on 10% of the rows in each table.
+**     sqlite3_expert_config(pExpert, EXPERT_CONFIG_SAMPLE, 10);
+*/
+int sqlite3_expert_config(sqlite3expert *p, int op, ...);
+
+#define EXPERT_CONFIG_SAMPLE 1    /* int */
+
+/*
+** Specify zero or more SQL statements to be included in the analysis.
+**
+** Buffer zSql must contain zero or more complete SQL statements. This
+** function parses all statements contained in the buffer and adds them
+** to the internal list of statements to analyze. If successful, SQLITE_OK
+** is returned and (*pzErr) set to NULL. Or, if an error occurs - for example
+** due to a error in the SQL - an SQLite error code is returned and (*pzErr)
+** may be set to point to an English language error message. In this case
+** the caller is responsible for eventually freeing the error message buffer
+** using sqlite3_free().
+**
+** If an error does occur while processing one of the statements in the
+** buffer passed as the second argument, none of the statements in the
+** buffer are added to the analysis.
+**
+** This function must be called before sqlite3_expert_analyze(). If a call
+** to this function is made on an sqlite3expert object that has already
+** been passed to sqlite3_expert_analyze() SQLITE_MISUSE is returned
+** immediately and no statements are added to the analysis.
+*/
+int sqlite3_expert_sql(
+  sqlite3expert *p,               /* From a successful sqlite3_expert_new() */
+  const char *zSql,               /* SQL statement(s) to add */
+  char **pzErr                    /* OUT: Error message (if any) */
+);
+
+
+/*
+** This function is called after the sqlite3expert object has been configured
+** with all SQL statements using sqlite3_expert_sql() to actually perform
+** the analysis. Once this function has been called, it is not possible to
+** add further SQL statements to the analysis.
+**
+** If successful, SQLITE_OK is returned and (*pzErr) is set to NULL. Or, if
+** an error occurs, an SQLite error code is returned and (*pzErr) set to 
+** point to a buffer containing an English language error message. In this
+** case it is the responsibility of the caller to eventually free the buffer
+** using sqlite3_free().
+**
+** If an error does occur within this function, the sqlite3expert object
+** is no longer useful for any purpose. At that point it is no longer
+** possible to add further SQL statements to the object or to re-attempt
+** the analysis. The sqlite3expert object must still be freed using a call
+** sqlite3_expert_destroy().
+*/
+int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr);
+
+/*
+** Return the total number of statements loaded using sqlite3_expert_sql().
+** The total number of SQL statements may be different from the total number
+** to calls to sqlite3_expert_sql().
+*/
+int sqlite3_expert_count(sqlite3expert*);
+
+/*
+** Return a component of the report.
+**
+** This function is called after sqlite3_expert_analyze() to extract the
+** results of the analysis. Each call to this function returns either a
+** NULL pointer or a pointer to a buffer containing a nul-terminated string.
+** The value passed as the third argument must be one of the EXPERT_REPORT_*
+** #define constants defined below.
+**
+** For some EXPERT_REPORT_* parameters, the buffer returned contains 
+** information relating to a specific SQL statement. In these cases that
+** SQL statement is identified by the value passed as the second argument.
+** SQL statements are numbered from 0 in the order in which they are parsed.
+** If an out-of-range value (less than zero or equal to or greater than the
+** value returned by sqlite3_expert_count()) is passed as the second argument
+** along with such an EXPERT_REPORT_* parameter, NULL is always returned.
+**
+** EXPERT_REPORT_SQL:
+**   Return the text of SQL statement iStmt.
+**
+** EXPERT_REPORT_INDEXES:
+**   Return a buffer containing the CREATE INDEX statements for all recommended
+**   indexes for statement iStmt. If there are no new recommeded indexes, NULL 
+**   is returned.
+**
+** EXPERT_REPORT_PLAN:
+**   Return a buffer containing the EXPLAIN QUERY PLAN output for SQL query
+**   iStmt after the proposed indexes have been added to the database schema.
+**
+** EXPERT_REPORT_CANDIDATES:
+**   Return a pointer to a buffer containing the CREATE INDEX statements 
+**   for all indexes that were tested (for all SQL statements). The iStmt
+**   parameter is ignored for EXPERT_REPORT_CANDIDATES calls.
+*/
+const char *sqlite3_expert_report(sqlite3expert*, int iStmt, int eReport);
+
+/*
+** Values for the third argument passed to sqlite3_expert_report().
+*/
+#define EXPERT_REPORT_SQL        1
+#define EXPERT_REPORT_INDEXES    2
+#define EXPERT_REPORT_PLAN       3
+#define EXPERT_REPORT_CANDIDATES 4
+
+/*
+** Free an (sqlite3expert*) handle and all associated resources. There 
+** should be one call to this function for each successful call to 
+** sqlite3-expert_new().
+*/
+void sqlite3_expert_destroy(sqlite3expert*);
+
+

ext/expert/test_expert.c

@@ -0,0 +1,220 @@
+/*
+** 2017 April 07
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+*/
+
+#if defined(SQLITE_TEST)
+
+#include "sqlite3expert.h"
+#include <assert.h>
+#include <string.h>
+
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#  ifndef SQLITE_TCLAPI
+#    define SQLITE_TCLAPI
+#  endif
+#endif
+
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+
+/*
+** Extract an sqlite3* db handle from the object passed as the second
+** argument. If successful, set *pDb to point to the db handle and return
+** TCL_OK. Otherwise, return TCL_ERROR.
+*/
+static int dbHandleFromObj(Tcl_Interp *interp, Tcl_Obj *pObj, sqlite3 **pDb){
+  Tcl_CmdInfo info;
+  if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(pObj), &info) ){
+    Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(pObj), 0);
+    return TCL_ERROR;
+  }
+
+  *pDb = *(sqlite3 **)info.objClientData;
+  return TCL_OK;
+}
+
+
+/*
+** Tclcmd:  $expert sql SQL
+**          $expert analyze
+**          $expert count
+**          $expert report STMT EREPORT
+**          $expert destroy
+*/
+static int SQLITE_TCLAPI testExpertCmd(
+  void *clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  sqlite3expert *pExpert = (sqlite3expert*)clientData;
+  struct Subcmd {
+    const char *zSub;
+    int nArg;
+    const char *zMsg;
+  } aSub[] = {
+    { "sql",       1, "TABLE",        }, /* 0 */
+    { "analyze",   0, "",             }, /* 1 */
+    { "count",     0, "",             }, /* 2 */
+    { "report",    2, "STMT EREPORT", }, /* 3 */
+    { "destroy",   0, "",             }, /* 4 */
+    { 0 }
+  };
+  int iSub;
+  int rc = TCL_OK;
+  char *zErr = 0;
+
+  if( objc<2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "SUBCOMMAND ...");
+    return TCL_ERROR;
+  }
+  rc = Tcl_GetIndexFromObjStruct(interp, 
+      objv[1], aSub, sizeof(aSub[0]), "sub-command", 0, &iSub
+  );
+  if( rc!=TCL_OK ) return rc;
+  if( objc!=2+aSub[iSub].nArg ){
+    Tcl_WrongNumArgs(interp, 2, objv, aSub[iSub].zMsg);
+    return TCL_ERROR;
+  }
+
+  switch( iSub ){
+    case 0: {      /* sql */
+      char *zArg = Tcl_GetString(objv[2]);
+      rc = sqlite3_expert_sql(pExpert, zArg, &zErr);
+      break;
+    }
+
+    case 1: {      /* analyze */
+      rc = sqlite3_expert_analyze(pExpert, &zErr);
+      break;
+    }
+
+    case 2: {      /* count */
+      int n = sqlite3_expert_count(pExpert);
+      Tcl_SetObjResult(interp, Tcl_NewIntObj(n));
+      break;
+    }
+
+    case 3: {      /* report */
+      const char *aEnum[] = {
+        "sql", "indexes", "plan", "candidates", 0
+      };
+      int iEnum;
+      int iStmt;
+      const char *zReport;
+
+      if( Tcl_GetIntFromObj(interp, objv[2], &iStmt) 
+       || Tcl_GetIndexFromObj(interp, objv[3], aEnum, "report", 0, &iEnum)
+      ){
+        return TCL_ERROR;
+      }
+
+      assert( EXPERT_REPORT_SQL==1 );
+      assert( EXPERT_REPORT_INDEXES==2 );
+      assert( EXPERT_REPORT_PLAN==3 );
+      assert( EXPERT_REPORT_CANDIDATES==4 );
+      zReport = sqlite3_expert_report(pExpert, iStmt, 1+iEnum);
+      Tcl_SetObjResult(interp, Tcl_NewStringObj(zReport, -1));
+      break;
+    }
+
+    default:       /* destroy */
+      assert( iSub==4 );     
+      Tcl_DeleteCommand(interp, Tcl_GetString(objv[0]));
+      break;
+  }
+
+  if( rc!=TCL_OK ){
+    if( zErr ){
+      Tcl_SetObjResult(interp, Tcl_NewStringObj(zErr, -1));
+    }else{
+      extern const char *sqlite3ErrName(int);
+      Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
+    }
+  }
+  sqlite3_free(zErr);
+  return rc;
+}
+
+static void SQLITE_TCLAPI testExpertDel(void *clientData){
+  sqlite3expert *pExpert = (sqlite3expert*)clientData;
+  sqlite3_expert_destroy(pExpert);
+}
+
+/*
+** sqlite3_expert_new DB
+*/
+static int SQLITE_TCLAPI test_sqlite3_expert_new(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  static int iCmd = 0;
+  sqlite3 *db;
+  char *zCmd = 0;
+  char *zErr = 0;
+  sqlite3expert *pExpert;
+  int rc = TCL_OK;
+
+  if( objc!=2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "DB");
+    return TCL_ERROR;
+  }
+  if( dbHandleFromObj(interp, objv[1], &db) ){
+    return TCL_ERROR;
+  }
+
+  zCmd = sqlite3_mprintf("sqlite3expert%d", ++iCmd);
+  if( zCmd==0 ){
+    Tcl_AppendResult(interp, "out of memory", (char*)0);
+    return TCL_ERROR;
+  }
+
+  pExpert = sqlite3_expert_new(db, &zErr);
+  if( pExpert==0 ){
+    Tcl_AppendResult(interp, zErr, (char*)0);
+    rc = TCL_ERROR;
+  }else{
+    void *p = (void*)pExpert;
+    Tcl_CreateObjCommand(interp, zCmd, testExpertCmd, p, testExpertDel);
+    Tcl_SetObjResult(interp, Tcl_NewStringObj(zCmd, -1));
+  }
+
+  sqlite3_free(zCmd);
+  sqlite3_free(zErr);
+  return rc;
+}
+
+#endif  /* ifndef SQLITE_OMIT_VIRTUALTABLE */
+
+int TestExpert_Init(Tcl_Interp *interp){
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+  struct Cmd {
+    const char *zCmd;
+    Tcl_ObjCmdProc *xProc;
+  } aCmd[] = {
+    { "sqlite3_expert_new", test_sqlite3_expert_new },
+  };
+  int i;
+
+  for(i=0; i<sizeof(aCmd)/sizeof(struct Cmd); i++){
+    struct Cmd *p = &aCmd[i];
+    Tcl_CreateObjCommand(interp, p->zCmd, p->xProc, 0, 0);
+  }
+#endif
+  return TCL_OK;
+}
+
+#endif

ext/fts3/fts3.c

@@ -1821,7 +1821,7 @@ static int fts3ScanInteriorNode(
   const char *zCsr = zNode;       /* Cursor to iterate through node */
   const char *zEnd = &zCsr[nNode];/* End of interior node buffer */
   char *zBuffer = 0;              /* Buffer to load terms into */
-  int nAlloc = 0;                 /* Size of allocated buffer */
+  i64 nAlloc = 0;                 /* Size of allocated buffer */
   int isFirstTerm = 1;            /* True when processing first term on page */
   sqlite3_int64 iChild;           /* Block id of child node to descend to */
 
@@ -1859,14 +1859,14 @@ static int fts3ScanInteriorNode(
     zCsr += fts3GetVarint32(zCsr, &nSuffix);
     
     assert( nPrefix>=0 && nSuffix>=0 );
-    if( &zCsr[nSuffix]>zEnd ){
+    if( nPrefix>zCsr-zNode || nSuffix>zEnd-zCsr ){
       rc = FTS_CORRUPT_VTAB;
       goto finish_scan;
     }
-    if( nPrefix+nSuffix>nAlloc ){
+    if( (i64)nPrefix+nSuffix>nAlloc ){
       char *zNew;
-      nAlloc = (nPrefix+nSuffix) * 2;
+      nAlloc = ((i64)nPrefix+nSuffix) * 2;
-      zNew = (char *)sqlite3_realloc(zBuffer, nAlloc);
+      zNew = (char *)sqlite3_realloc64(zBuffer, nAlloc);
       if( !zNew ){
         rc = SQLITE_NOMEM;
         goto finish_scan;
@@ -3808,7 +3808,7 @@ static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
   int rc = SQLITE_OK;
   UNUSED_PARAMETER(iSavepoint);
   assert( ((Fts3Table *)pVtab)->inTransaction );
-  assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint );
+  assert( ((Fts3Table *)pVtab)->mxSavepoint <= iSavepoint );
   TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint );
   if( ((Fts3Table *)pVtab)->bIgnoreSavepoint==0 ){
     rc = fts3SyncMethod(pVtab);
@@ -3846,8 +3846,23 @@ static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){
   return SQLITE_OK;
 }
 
+/*
+** Return true if zName is the extension on one of the shadow tables used
+** by this module.
+*/
+static int fts3ShadowName(const char *zName){
+  static const char *azName[] = {
+    "content", "docsize", "segdir", "segments", "stat", 
+  };
+  unsigned int i;
+  for(i=0; i<sizeof(azName)/sizeof(azName[0]); i++){
+    if( sqlite3_stricmp(zName, azName[i])==0 ) return 1;
+  }
+  return 0;
+}
+
 static const sqlite3_module fts3Module = {
-  /* iVersion      */ 2,
+  /* iVersion      */ 3,
   /* xCreate       */ fts3CreateMethod,
   /* xConnect      */ fts3ConnectMethod,
   /* xBestIndex    */ fts3BestIndexMethod,
@@ -3870,6 +3885,7 @@ static const sqlite3_module fts3Module = {
   /* xSavepoint    */ fts3SavepointMethod,
   /* xRelease      */ fts3ReleaseMethod,
   /* xRollbackTo   */ fts3RollbackToMethod,
+  /* xShadowName   */ fts3ShadowName,
 };
 
 /*
@@ -3963,7 +3979,7 @@ int sqlite3Fts3Init(sqlite3 *db){
 
 #ifdef SQLITE_TEST
   if( rc==SQLITE_OK ){
-    rc = sqlite3Fts3ExprInitTestInterface(db);
+    rc = sqlite3Fts3ExprInitTestInterface(db, pHash);
   }
 #endif
 
@@ -4150,6 +4166,7 @@ static int fts3EvalPhraseLoad(
   return rc;
 }
 
+#ifndef SQLITE_DISABLE_FTS4_DEFERRED
 /*
 ** This function is called on each phrase after the position lists for
 ** any deferred tokens have been loaded into memory. It updates the phrases
@@ -4253,6 +4270,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){
 
   return SQLITE_OK;
 }
+#endif /* SQLITE_DISABLE_FTS4_DEFERRED */
 
 /*
 ** Maximum number of tokens a phrase may have to be considered for the

ext/fts3/fts3Int.h

@@ -584,7 +584,7 @@ int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int,
 );
 void sqlite3Fts3ExprFree(Fts3Expr *);
 #ifdef SQLITE_TEST
-int sqlite3Fts3ExprInitTestInterface(sqlite3 *db);
+int sqlite3Fts3ExprInitTestInterface(sqlite3 *db, Fts3Hash*);
 int sqlite3Fts3InitTerm(sqlite3 *db);
 #endif
 

ext/fts3/fts3_aux.c

@@ -539,7 +539,8 @@ int sqlite3Fts3InitAux(sqlite3 *db){
      0,                           /* xRename       */
      0,                           /* xSavepoint    */
      0,                           /* xRelease      */
-     0                            /* xRollbackTo   */
+     0,                           /* xRollbackTo   */
+     0                            /* xShadowName   */
   };
   int rc;                         /* Return code */
 

ext/fts3/fts3_expr.c

@@ -1108,34 +1108,6 @@ void sqlite3Fts3ExprFree(Fts3Expr *pDel){
 
 #include <stdio.h>
 
-/*
-** Function to query the hash-table of tokenizers (see README.tokenizers).
-*/
-static int queryTestTokenizer(
-  sqlite3 *db, 
-  const char *zName,  
-  const sqlite3_tokenizer_module **pp
-){
-  int rc;
-  sqlite3_stmt *pStmt;
-  const char zSql[] = "SELECT fts3_tokenizer(?)";
-
-  *pp = 0;
-  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
-  if( rc!=SQLITE_OK ){
-    return rc;
-  }
-
-  sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
-  if( SQLITE_ROW==sqlite3_step(pStmt) ){
-    if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
-      memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
-    }
-  }
-
-  return sqlite3_finalize(pStmt);
-}
-
 /*
 ** Return a pointer to a buffer containing a text representation of the
 ** expression passed as the first argument. The buffer is obtained from
@@ -1203,12 +1175,12 @@ static char *exprToString(Fts3Expr *pExpr, char *zBuf){
 **
 **   SELECT fts3_exprtest('simple', 'Bill col2:Bloggs', 'col1', 'col2');
 */
-static void fts3ExprTest(
+static void fts3ExprTestCommon(
+  int bRebalance,
   sqlite3_context *context,
   int argc,
   sqlite3_value **argv
 ){
-  sqlite3_tokenizer_module const *pModule = 0;
   sqlite3_tokenizer *pTokenizer = 0;
   int rc;
   char **azCol = 0;
@@ -1218,7 +1190,9 @@ static void fts3ExprTest(
   int ii;
   Fts3Expr *pExpr;
   char *zBuf = 0;
-  sqlite3 *db = sqlite3_context_db_handle(context);
+  Fts3Hash *pHash = (Fts3Hash*)sqlite3_user_data(context);
+  const char *zTokenizer = 0;
+  char *zErr = 0;
 
   if( argc<3 ){
     sqlite3_result_error(context, 
@@ -1227,24 +1201,18 @@ static void fts3ExprTest(
     return;
   }
 
-  rc = queryTestTokenizer(db,
+  zTokenizer = (const char*)sqlite3_value_text(argv[0]);
-                          (const char *)sqlite3_value_text(argv[0]), &pModule);
+  rc = sqlite3Fts3InitTokenizer(pHash, zTokenizer, &pTokenizer, &zErr);
-  if( rc==SQLITE_NOMEM ){
+  if( rc!=SQLITE_OK ){
-    sqlite3_result_error_nomem(context);
+    if( rc==SQLITE_NOMEM ){
-    goto exprtest_out;
+      sqlite3_result_error_nomem(context);
-  }else if( !pModule ){
+    }else{
-    sqlite3_result_error(context, "No such tokenizer module", -1);
+      sqlite3_result_error(context, zErr, -1);
-    goto exprtest_out;
+    }
+    sqlite3_free(zErr);
+    return;
   }
 
-  rc = pModule->xCreate(0, 0, &pTokenizer);
-  assert( rc==SQLITE_NOMEM || rc==SQLITE_OK );
-  if( rc==SQLITE_NOMEM ){
-    sqlite3_result_error_nomem(context);
-    goto exprtest_out;
-  }
-  pTokenizer->pModule = pModule;
-
   zExpr = (const char *)sqlite3_value_text(argv[1]);
   nExpr = sqlite3_value_bytes(argv[1]);
   nCol = argc-2;
@@ -1257,7 +1225,7 @@ static void fts3ExprTest(
     azCol[ii] = (char *)sqlite3_value_text(argv[ii+2]);
   }
 
-  if( sqlite3_user_data(context) ){
+  if( bRebalance ){
     char *zDummy = 0;
     rc = sqlite3Fts3ExprParse(
         pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr, &zDummy
@@ -1283,23 +1251,38 @@ static void fts3ExprTest(
   sqlite3Fts3ExprFree(pExpr);
 
 exprtest_out:
-  if( pModule && pTokenizer ){
+  if( pTokenizer ){
-    rc = pModule->xDestroy(pTokenizer);
+    rc = pTokenizer->pModule->xDestroy(pTokenizer);
   }
   sqlite3_free(azCol);
 }
 
+static void fts3ExprTest(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  fts3ExprTestCommon(0, context, argc, argv);
+}
+static void fts3ExprTestRebalance(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  fts3ExprTestCommon(1, context, argc, argv);
+}
+
 /*
 ** Register the query expression parser test function fts3_exprtest() 
 ** with database connection db. 
 */
-int sqlite3Fts3ExprInitTestInterface(sqlite3* db){
+int sqlite3Fts3ExprInitTestInterface(sqlite3 *db, Fts3Hash *pHash){
   int rc = sqlite3_create_function(
-      db, "fts3_exprtest", -1, SQLITE_UTF8, 0, fts3ExprTest, 0, 0
+      db, "fts3_exprtest", -1, SQLITE_UTF8, (void*)pHash, fts3ExprTest, 0, 0
   );
   if( rc==SQLITE_OK ){
     rc = sqlite3_create_function(db, "fts3_exprtest_rebalance", 
-        -1, SQLITE_UTF8, (void *)1, fts3ExprTest, 0, 0
+        -1, SQLITE_UTF8, (void*)pHash, fts3ExprTestRebalance, 0, 0
     );
   }
   return rc;

ext/fts3/fts3_term.c

@@ -361,7 +361,8 @@ int sqlite3Fts3InitTerm(sqlite3 *db){
      0,                           /* xRename       */
      0,                           /* xSavepoint    */
      0,                           /* xRelease      */
-     0                            /* xRollbackTo   */
+     0,                           /* xRollbackTo   */
+     0                            /* xShadowName   */
   };
   int rc;                         /* Return code */
 

ext/fts3/fts3_tokenize_vtab.c

@@ -443,7 +443,8 @@ int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash){
      0,                           /* xRename       */
      0,                           /* xSavepoint    */
      0,                           /* xRelease      */
-     0                            /* xRollbackTo   */
+     0,                           /* xRollbackTo   */
+     0                            /* xShadowName   */
   };
   int rc;                         /* Return code */
 

ext/fts3/fts3_write.c

@@ -1374,15 +1374,19 @@ static int fts3SegReaderNext(
   ** safe (no risk of overread) even if the node data is corrupted. */
   pNext += fts3GetVarint32(pNext, &nPrefix);
   pNext += fts3GetVarint32(pNext, &nSuffix);
-  if( nPrefix<0 || nSuffix<=0 
+  if( nSuffix<=0 
-   || &pNext[nSuffix]>&pReader->aNode[pReader->nNode] 
+   || (&pReader->aNode[pReader->nNode] - pNext)<nSuffix
+   || nPrefix>pReader->nTermAlloc
   ){
     return FTS_CORRUPT_VTAB;
   }
 
-  if( nPrefix+nSuffix>pReader->nTermAlloc ){
+  /* Both nPrefix and nSuffix were read by fts3GetVarint32() and so are
-    int nNew = (nPrefix+nSuffix)*2;
+  ** between 0 and 0x7FFFFFFF. But the sum of the two may cause integer
-    char *zNew = sqlite3_realloc(pReader->zTerm, nNew);
+  ** overflow - hence the (i64) casts.  */
+  if( (i64)nPrefix+nSuffix>(i64)pReader->nTermAlloc ){
+    i64 nNew = ((i64)nPrefix+nSuffix)*2;
+    char *zNew = sqlite3_realloc64(pReader->zTerm, nNew);
     if( !zNew ){
       return SQLITE_NOMEM;
     }
@@ -1404,7 +1408,7 @@ static int fts3SegReaderNext(
   ** b-tree node. And that the final byte of the doclist is 0x00. If either 
   ** of these statements is untrue, then the data structure is corrupt.
   */
-  if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode] 
+  if( (&pReader->aNode[pReader->nNode] - pReader->aDoclist)<pReader->nDoclist
    || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1])
   ){
     return FTS_CORRUPT_VTAB;
@@ -1908,6 +1912,7 @@ static int fts3WriteSegment(
     sqlite3_bind_blob(pStmt, 2, z, n, SQLITE_STATIC);
     sqlite3_step(pStmt);
     rc = sqlite3_reset(pStmt);
+    sqlite3_bind_null(pStmt, 2);
   }
   return rc;
 }
@@ -1964,6 +1969,7 @@ static int fts3WriteSegdir(
     sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC);
     sqlite3_step(pStmt);
     rc = sqlite3_reset(pStmt);
+    sqlite3_bind_null(pStmt, 6);
   }
   return rc;
 }
@@ -3443,6 +3449,7 @@ static void fts3UpdateDocTotals(
   sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, SQLITE_STATIC);
   sqlite3_step(pStmt);
   *pRC = sqlite3_reset(pStmt);
+  sqlite3_bind_null(pStmt, 2);
   sqlite3_free(a);
 }
 
@@ -3727,6 +3734,9 @@ static int nodeReaderNext(NodeReader *p){
     }
     p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nSuffix);
 
+    if( nPrefix>p->iOff || nSuffix>p->nNode-p->iOff ){
+      return SQLITE_CORRUPT_VTAB;
+    }
     blobGrowBuffer(&p->term, nPrefix+nSuffix, &rc);
     if( rc==SQLITE_OK ){
       memcpy(&p->term.a[nPrefix], &p->aNode[p->iOff], nSuffix);
@@ -3734,6 +3744,9 @@ static int nodeReaderNext(NodeReader *p){
       p->iOff += nSuffix;
       if( p->iChild==0 ){
         p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &p->nDoclist);
+        if( (p->nNode-p->iOff)<p->nDoclist ){
+          return SQLITE_CORRUPT_VTAB;
+        }
         p->aDoclist = &p->aNode[p->iOff];
         p->iOff += p->nDoclist;
       }
@@ -3741,7 +3754,6 @@ static int nodeReaderNext(NodeReader *p){
   }
 
   assert( p->iOff<=p->nNode );
-
   return rc;
 }
 
@@ -4631,6 +4643,7 @@ static int fts3TruncateSegment(
       sqlite3_bind_int(pChomp, 4, iIdx);
       sqlite3_step(pChomp);
       rc = sqlite3_reset(pChomp);
+      sqlite3_bind_null(pChomp, 2);
     }
   }
 
@@ -4710,6 +4723,7 @@ static int fts3IncrmergeHintStore(Fts3Table *p, Blob *pHint){
     sqlite3_bind_blob(pReplace, 2, pHint->a, pHint->n, SQLITE_STATIC);
     sqlite3_step(pReplace);
     rc = sqlite3_reset(pReplace);
+    sqlite3_bind_null(pReplace, 2);
   }
 
   return rc;
@@ -5524,7 +5538,6 @@ int sqlite3Fts3UpdateMethod(
 ){
   Fts3Table *p = (Fts3Table *)pVtab;
   int rc = SQLITE_OK;             /* Return Code */
-  int isRemove = 0;               /* True for an UPDATE or DELETE */
   u32 *aSzIns = 0;                /* Sizes of inserted documents */
   u32 *aSzDel = 0;                /* Sizes of deleted documents */
   int nChng = 0;                  /* Net change in number of documents */
@@ -5622,7 +5635,6 @@ int sqlite3Fts3UpdateMethod(
   if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
     assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER );
     rc = fts3DeleteByRowid(p, apVal[0], &nChng, aSzDel);
-    isRemove = 1;
   }
   
   /* If this is an INSERT or UPDATE operation, insert the new record. */
@@ -5634,7 +5646,7 @@ int sqlite3Fts3UpdateMethod(
         rc = FTS_CORRUPT_VTAB;
       }
     }
-    if( rc==SQLITE_OK && (!isRemove || *pRowid!=p->iPrevDocid ) ){
+    if( rc==SQLITE_OK ){
       rc = fts3PendingTermsDocid(p, 0, iLangid, *pRowid);
     }
     if( rc==SQLITE_OK ){

ext/fts3/unicode/mkunicode.tcl

@@ -529,6 +529,263 @@ proc print_fold {zFunc} {
   puts "\}"
 }
 
+proc code {txt} {
+  set txt [string trimright $txt]
+  set txt [string trimleft $txt "\n"]
+  set n [expr {[string length $txt] - [string length [string trim $txt]]}]
+  set ret ""
+  foreach L [split $txt "\n"] {
+    append ret "[string range $L $n end]\n"
+  }
+  return [uplevel "subst -nocommands {$ret}"]
+}
+
+proc intarray {lInt} {
+  set ret ""
+  set n [llength $lInt]
+  for {set i 0} {$i < $n} {incr i 10} {
+    append ret "\n    "
+    foreach int [lrange $lInt $i [expr $i+9]] {
+      append ret [format "%-7s" "$int, "]
+    }
+  }
+  append ret "\n  "
+  set ret
+}
+
+proc categories_switch {Cvar first lSecond} {
+  upvar $Cvar C
+  set ret ""
+  append ret "case '$first':\n"
+  append ret "          switch( zCat\[1\] ){\n"
+  foreach s $lSecond {
+    append ret "            case '$s': aArray\[$C($first$s)\] = 1; break;\n"
+  }
+  append ret "            case '*': \n"
+  foreach s $lSecond {
+    append ret "              aArray\[$C($first$s)\] = 1;\n"
+  }
+  append ret "              break;\n"
+  append ret "            default: return 1;"
+  append ret "          }\n"
+  append ret "          break;\n"
+}
+
+# Argument is a list. Each element of which is itself a list of two elements:
+#
+#   * the codepoint
+#   * the category
+#
+# List elements are sorted in order of codepoint.
+#
+proc print_categories {lMap} {
+  set categories {
+    Cc Cf Cn Cs
+    Ll Lm Lo Lt Lu
+    Mc Me Mn
+    Nd Nl No
+    Pc Pd Pe Pf Pi Po Ps
+    Sc Sk Sm So
+    Zl Zp Zs
+
+    LC Co
+  }
+
+  for {set i 0} {$i < [llength $categories]} {incr i} {
+    set C([lindex $categories $i]) [expr 1+$i]
+  }
+
+  set caseC [categories_switch C C {c f n s o}]
+  set caseL [categories_switch C L {l m o t u C}]
+  set caseM [categories_switch C M {c e n}]
+  set caseN [categories_switch C N {d l o}]
+  set caseP [categories_switch C P {c d e f i o s}]
+  set caseS [categories_switch C S {c k m o}]
+  set caseZ [categories_switch C Z {l p s}]
+
+  set nCat [expr [llength [array names C]] + 1]
+  puts [code {
+    int sqlite3Fts5UnicodeNCat(void) { 
+      return $nCat;
+    }
+
+    int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ 
+      aArray[0] = 1;
+      switch( zCat[0] ){
+        $caseC
+        $caseL
+        $caseM
+        $caseN
+        $caseP
+        $caseS
+        $caseZ
+      }
+      return 0;
+    }
+  }]
+
+  set nRepeat 0
+  set first   [lindex $lMap 0 0]
+  set class   [lindex $lMap 0 1]
+  set prev -1
+
+  set CASE(0) "Lu"
+  set CASE(1) "Ll"
+
+  foreach m $lMap {
+    foreach {codepoint cl} $m {}
+    set codepoint [expr "0x$codepoint"]
+    if {$codepoint>=(1<<20)} continue
+
+    set bNew 0
+    if {$codepoint!=($prev+1)} {
+      set bNew 1
+    } elseif {
+      $cl==$class || ($class=="LC" && $cl==$CASE([expr $nRepeat & 0x01]))
+    } {
+      incr nRepeat
+    } elseif {$class=="Lu" && $nRepeat==1 && $cl=="Ll"} {
+      set class LC
+      incr nRepeat
+    } else {
+      set bNew 1
+    }
+    if {$bNew} {
+      lappend lEntries [list $first $class $nRepeat]
+      set nRepeat 1
+      set first $codepoint
+      set class $cl
+    }
+    set prev $codepoint
+  }
+  if {$nRepeat>0} {
+    lappend lEntries [list $first $class $nRepeat]
+  }
+
+  set aBlock [list 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
+  set aMap [list]
+  foreach e $lEntries {
+    foreach {cp class nRepeat} $e {}
+    set block [expr ($cp>>16)]
+    if {$block>0 && [lindex $aBlock $block]==0} {
+      for {set i 1} {$i<=$block} {incr i} {
+        if {[lindex $aBlock $i]==0} {
+          lset aBlock $i [llength $aMap]
+        }
+      }
+    }
+    lappend aMap [expr {$cp & 0xFFFF}]
+    lappend aData [expr {($nRepeat << 5) + $C($class)}]
+  }
+  for {set i 1} {$i<[llength $aBlock]} {incr i} {
+    if {[lindex $aBlock $i]==0} {
+      lset aBlock $i [llength $aMap]
+    }
+  }
+
+  set aBlockArray [intarray $aBlock]
+  set aMapArray [intarray $aMap]
+  set aDataArray [intarray $aData]
+  puts [code {
+    static u16 aFts5UnicodeBlock[] = {$aBlockArray};
+    static u16 aFts5UnicodeMap[] = {$aMapArray};
+    static u16 aFts5UnicodeData[] = {$aDataArray};
+
+    int sqlite3Fts5UnicodeCategory(int iCode) { 
+      int iRes = -1;
+      int iHi;
+      int iLo;
+      int ret;
+      u16 iKey;
+
+      if( iCode>=(1<<20) ){
+        return 0;
+      }
+      iLo = aFts5UnicodeBlock[(iCode>>16)];
+      iHi = aFts5UnicodeBlock[1+(iCode>>16)];
+      iKey = (iCode & 0xFFFF);
+      while( iHi>iLo ){
+        int iTest = (iHi + iLo) / 2;
+        assert( iTest>=iLo && iTest<iHi );
+        if( iKey>=aFts5UnicodeMap[iTest] ){
+          iRes = iTest;
+          iLo = iTest+1;
+        }else{
+          iHi = iTest;
+        }
+      }
+
+      if( iRes<0 ) return 0;
+      if( iKey>=(aFts5UnicodeMap[iRes]+(aFts5UnicodeData[iRes]>>5)) ) return 0;
+      ret = aFts5UnicodeData[iRes] & 0x1F;
+      if( ret!=$C(LC) ) return ret;
+      return ((iKey - aFts5UnicodeMap[iRes]) & 0x01) ? $C(Ll) : $C(Lu);
+    }
+
+    void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){
+      int i = 0;
+      int iTbl = 0;
+      while( i<128 ){
+        int bToken = aArray[ aFts5UnicodeData[iTbl] & 0x1F ];
+        int n = (aFts5UnicodeData[iTbl] >> 5) + i;
+        for(; i<128 && i<n; i++){
+          aAscii[i] = bToken;
+        }
+        iTbl++;
+      }
+    }
+  }]
+}
+
+proc print_test_categories {lMap} {
+
+  set lCP [list]
+  foreach e $lMap {
+    foreach {cp cat} $e {}
+    if {[expr 0x$cp] < (1<<20)} {
+      lappend lCP "{0x$cp, \"$cat\"}, "
+    }
+  }
+
+  set aCP "\n"
+  for {set i 0} {$i < [llength $lCP]} {incr i 4} {
+    append aCP "    [join [lrange $lCP $i $i+3]]\n"
+  }
+
+
+  puts [code {
+    static int categories_test (int *piCode){
+      struct Codepoint {
+        int iCode;
+        const char *zCat;
+      } aCP[] = {$aCP};
+      int i;
+      int iCP = 0;
+
+      for(i=0; i<1000000; i++){
+        u8 aArray[40];
+        int cat = 0;
+        int c = 0;
+        memset(aArray, 0, sizeof(aArray));
+        if( aCP[iCP].iCode==i ){
+          sqlite3Fts5UnicodeCatParse(aCP[iCP].zCat, aArray);
+          iCP++;
+        }else{
+          aArray[0] = 1;
+        }
+
+        c = sqlite3Fts5UnicodeCategory(i);
+        if( aArray[c]==0 ){
+          *piCode = i;
+          return 1;
+        }
+      }
+
+      return 0;
+    }
+  }]
+}
+
 proc print_fold_test {zFunc mappings} {
   global tl_lookup_table
 
@@ -605,15 +862,21 @@ proc print_test_main {} {
   puts "#include <stdio.h>"
   puts ""
   puts "int main(int argc, char **argv)\{"
-  puts "  int r1, r2;"
+  puts "  int r1, r2, r3;"
   puts "  int code;"
+  puts "  r3 = 0;"
   puts "  r1 = isalnum_test(&code);"
   puts "  if( r1 ) printf(\"isalnum(): Problem with code %d\\n\",code);"
   puts "  else printf(\"isalnum(): test passed\\n\");"
   puts "  r2 = fold_test(&code);"
   puts "  if( r2 ) printf(\"fold(): Problem with code %d\\n\",code);"
   puts "  else printf(\"fold(): test passed\\n\");"
-  puts "  return (r1 || r2);"
+  if {$::generate_fts5_code} {
+    puts "  r3 = categories_test(&code);"
+    puts "  if( r3 ) printf(\"categories(): Problem with code %d\\n\",code);"
+    puts "  else printf(\"categories(): test passed\\n\");"
+  }
+  puts "  return (r1 || r2 || r3);"
   puts "\}"
 }
 
@@ -651,10 +914,18 @@ for {set i 0} {$i < [llength $argv]-2} {incr i} {
 
 print_fileheader
 
+if {$::generate_test_code} {
+  puts "typedef unsigned short int u16;"
+  puts "typedef unsigned char u8;"
+  puts "#include <string.h>"
+}
+
 # Print the isalnum() function to stdout.
 #
 set lRange [an_load_separator_ranges]
-print_isalnum ${function_prefix}UnicodeIsalnum $lRange
+if {$generate_fts5_code==0} {
+  print_isalnum ${function_prefix}UnicodeIsalnum $lRange
+}
 
 # Leave a gap between the two generated C functions.
 #
@@ -677,12 +948,21 @@ puts ""
 #
 print_fold ${function_prefix}UnicodeFold
 
+if {$generate_fts5_code} {
+  puts ""
+  puts ""
+  print_categories [cc_load_unicodedata_text ${unicodedata.txt}]
+}
+
 # Print the test routines and main() function to stdout, if -test 
 # was specified.
 #
 if {$::generate_test_code} {
-  print_test_isalnum ${function_prefix}UnicodeIsalnum $lRange
+  if {$generate_fts5_code==0} {
+    print_test_isalnum ${function_prefix}UnicodeIsalnum $lRange
+  }
   print_fold_test ${function_prefix}UnicodeFold $mappings
+  print_test_categories [cc_load_unicodedata_text ${unicodedata.txt}]
   print_test_main 
 }
 

ext/fts3/unicode/parseunicode.tcl

@@ -143,4 +143,40 @@ proc tl_load_casefolding_txt {zName} {
   }
 }
 
+proc cc_load_unicodedata_text {zName} {
+  set fd [open $zName]
+  set lField {
+    code
+    character_name
+    general_category
+    canonical_combining_classes
+    bidirectional_category
+    character_decomposition_mapping
+    decimal_digit_value
+    digit_value
+    numeric_value
+    mirrored
+    unicode_1_name
+    iso10646_comment_field
+    uppercase_mapping
+    lowercase_mapping
+    titlecase_mapping
+  }
+  set lRet [list]
+
+  while { ![eof $fd] } {
+    set line [gets $fd]
+    if {$line == ""} continue
+
+    set fields [split $line ";"]
+    if {[llength $fields] != [llength $lField]} { error "parse error: $line" }
+    foreach $lField $fields {}
+
+    lappend lRet [list $code $general_category]
+  }
+
+  close $fd
+  set lRet
+}
+
 

ext/fts5/fts5.h

@@ -444,7 +444,7 @@ struct Fts5ExtensionApi {
 **            This way, even if the tokenizer does not provide synonyms
 **            when tokenizing query text (it should not - to do would be
 **            inefficient), it doesn't matter if the user queries for 
-**            'first + place' or '1st + place', as there are entires in the
+**            'first + place' or '1st + place', as there are entries in the
 **            FTS index corresponding to both forms of the first token.
 **   </ol>
 **
@@ -472,7 +472,7 @@ struct Fts5ExtensionApi {
 **   extra data to the FTS index or require FTS5 to query for multiple terms,
 **   so it is efficient in terms of disk space and query speed. However, it
 **   does not support prefix queries very well. If, as suggested above, the
-**   token "first" is subsituted for "1st" by the tokenizer, then the query:
+**   token "first" is substituted for "1st" by the tokenizer, then the query:
 **
 **   <codeblock>
 **     ... MATCH '1s*'</codeblock>

ext/fts5/fts5Int.h

@@ -722,6 +722,8 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm(
   int bPrefix
 );
 
+void sqlite3Fts5ParseSetCaret(Fts5ExprPhrase*);
+
 Fts5ExprNearset *sqlite3Fts5ParseNearset(
   Fts5Parse*, 
   Fts5ExprNearset*,
@@ -782,9 +784,12 @@ int sqlite3Fts5VocabInit(Fts5Global*, sqlite3*);
 /**************************************************************************
 ** Interface to automatically generated code in fts5_unicode2.c. 
 */
-int sqlite3Fts5UnicodeIsalnum(int c);
 int sqlite3Fts5UnicodeIsdiacritic(int c);
 int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic);
+
+int sqlite3Fts5UnicodeCatParse(const char*, u8*);
+int sqlite3Fts5UnicodeCategory(int iCode);
+void sqlite3Fts5UnicodeAscii(u8*, u8*);
 /*
 ** End of interface to code in fts5_unicode2.c.
 **************************************************************************/

ext/fts5/fts5_aux.c

@@ -358,6 +358,16 @@ static int fts5SnippetScore(
   return rc;
 }
 
+/*
+** Return the value in pVal interpreted as utf-8 text. Except, if pVal 
+** contains a NULL value, return a pointer to a static string zero
+** bytes in length instead of a NULL pointer.
+*/
+static const char *fts5ValueToText(sqlite3_value *pVal){
+  const char *zRet = (const char*)sqlite3_value_text(pVal);
+  return zRet ? zRet : "";
+}
+
 /*
 ** Implementation of snippet() function.
 */
@@ -393,9 +403,9 @@ static void fts5SnippetFunction(
   nCol = pApi->xColumnCount(pFts);
   memset(&ctx, 0, sizeof(HighlightContext));
   iCol = sqlite3_value_int(apVal[0]);
-  ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]);
+  ctx.zOpen = fts5ValueToText(apVal[1]);
-  ctx.zClose = (const char*)sqlite3_value_text(apVal[2]);
+  ctx.zClose = fts5ValueToText(apVal[2]);
-  zEllips = (const char*)sqlite3_value_text(apVal[3]);
+  zEllips = fts5ValueToText(apVal[3]);
   nToken = sqlite3_value_int(apVal[4]);
 
   iBestCol = (iCol>=0 ? iCol : 0);

ext/fts5/fts5_expr.c

@@ -36,6 +36,7 @@ void sqlite3Fts5Parser(void*, int, Fts5Token, Fts5Parse*);
 #include <stdio.h>
 void sqlite3Fts5ParserTrace(FILE*, char*);
 #endif
+int sqlite3Fts5ParserFallback(int);
 
 
 struct Fts5Expr {
@@ -87,7 +88,8 @@ struct Fts5ExprNode {
 ** or term prefix.
 */
 struct Fts5ExprTerm {
-  int bPrefix;                    /* True for a prefix term */
+  u8 bPrefix;                     /* True for a prefix term */
+  u8 bFirst;                      /* True if token must be first in column */
   char *zTerm;                    /* nul-terminated term */
   Fts5IndexIter *pIter;           /* Iterator for this term */
   Fts5ExprTerm *pSynonym;         /* Pointer to first in list of synonyms */
@@ -168,6 +170,7 @@ static int fts5ExprGetToken(
     case '+':  tok = FTS5_PLUS;  break;
     case '*':  tok = FTS5_STAR;  break;
     case '-':  tok = FTS5_MINUS; break;
+    case '^':  tok = FTS5_CARET; break;
     case '\0': tok = FTS5_EOF;   break;
 
     case '"': {
@@ -427,6 +430,7 @@ static int fts5ExprPhraseIsMatch(
   Fts5PoslistReader *aIter = aStatic;
   int i;
   int rc = SQLITE_OK;
+  int bFirst = pPhrase->aTerm[0].bFirst;
   
   fts5BufferZero(&pPhrase->poslist);
 
@@ -481,8 +485,10 @@ static int fts5ExprPhraseIsMatch(
     }while( bMatch==0 );
 
     /* Append position iPos to the output */
-    rc = sqlite3Fts5PoslistWriterAppend(&pPhrase->poslist, &writer, iPos);
+    if( bFirst==0 || FTS5_POS2OFFSET(iPos)==0 ){
-    if( rc!=SQLITE_OK ) goto ismatch_out;
+      rc = sqlite3Fts5PoslistWriterAppend(&pPhrase->poslist, &writer, iPos);
+      if( rc!=SQLITE_OK ) goto ismatch_out;
+    }
 
     for(i=0; i<pPhrase->nTerm; i++){
       if( sqlite3Fts5PoslistReaderNext(&aIter[i]) ) goto ismatch_out;
@@ -736,7 +742,9 @@ static int fts5ExprNearTest(
     ** phrase is not a match, break out of the loop early.  */
     for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
       Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
-      if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){
+      if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym 
+       || pNear->pColset || pPhrase->aTerm[0].bFirst
+      ){
         int bMatch = 0;
         rc = fts5ExprPhraseIsMatch(pNode, pPhrase, &bMatch);
         if( bMatch==0 ) break;
@@ -917,6 +925,7 @@ static int fts5ExprNodeTest_STRING(
   assert( pNear->nPhrase>1 
        || pNear->apPhrase[0]->nTerm>1 
        || pNear->apPhrase[0]->aTerm[0].pSynonym
+       || pNear->apPhrase[0]->aTerm[0].bFirst
   );
 
   /* Initialize iLast, the "lastest" rowid any iterator points to. If the
@@ -1441,6 +1450,16 @@ static void fts5ExprPhraseFree(Fts5ExprPhrase *pPhrase){
   }
 }
 
+/*
+** Set the "bFirst" flag on the first token of the phrase passed as the
+** only argument.
+*/
+void sqlite3Fts5ParseSetCaret(Fts5ExprPhrase *pPhrase){
+  if( pPhrase && pPhrase->nTerm ){
+    pPhrase->aTerm[0].bFirst = 1;
+  }
+}
+
 /*
 ** If argument pNear is NULL, then a new Fts5ExprNearset object is allocated
 ** and populated with pPhrase. Or, if pNear is not NULL, phrase pPhrase is
@@ -1658,7 +1677,7 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm(
       ** no token characters at all. (e.g ... MATCH '""'). */
       sCtx.pPhrase = sqlite3Fts5MallocZero(&pParse->rc, sizeof(Fts5ExprPhrase));
     }else if( sCtx.pPhrase->nTerm ){
-      sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = bPrefix;
+      sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = (u8)bPrefix;
     }
     pParse->apPhrase[pParse->nPhrase-1] = sCtx.pPhrase;
   }
@@ -1719,6 +1738,7 @@ int sqlite3Fts5ExprClonePhrase(
       }
       if( rc==SQLITE_OK ){
         sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix;
+        sCtx.pPhrase->aTerm[i].bFirst = pOrig->aTerm[i].bFirst;
       }
     }
   }else{
@@ -1737,7 +1757,10 @@ int sqlite3Fts5ExprClonePhrase(
     pNew->pRoot->pNear->nPhrase = 1;
     sCtx.pPhrase->pNode = pNew->pRoot;
 
-    if( pOrig->nTerm==1 && pOrig->aTerm[0].pSynonym==0 ){
+    if( pOrig->nTerm==1 
+     && pOrig->aTerm[0].pSynonym==0 
+     && pOrig->aTerm[0].bFirst==0 
+    ){
       pNew->pRoot->eType = FTS5_TERM;
       pNew->pRoot->xNext = fts5ExprNodeNext_TERM;
     }else{
@@ -2011,6 +2034,7 @@ static void fts5ExprAssignXNext(Fts5ExprNode *pNode){
       Fts5ExprNearset *pNear = pNode->pNear;
       if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 
        && pNear->apPhrase[0]->aTerm[0].pSynonym==0
+       && pNear->apPhrase[0]->aTerm[0].bFirst==0
       ){
         pNode->eType = FTS5_TERM;
         pNode->xNext = fts5ExprNodeNext_TERM;
@@ -2097,20 +2121,23 @@ Fts5ExprNode *sqlite3Fts5ParseNode(
           }
         }
 
-        if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL 
+        if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL ){
-         && (pNear->nPhrase!=1 || pNear->apPhrase[0]->nTerm>1)
+          Fts5ExprPhrase *pPhrase = pNear->apPhrase[0];
-        ){
+          if( pNear->nPhrase!=1 
-          assert( pParse->rc==SQLITE_OK );
+           || pPhrase->nTerm>1
-          pParse->rc = SQLITE_ERROR;
+           || (pPhrase->nTerm>0 && pPhrase->aTerm[0].bFirst)
-          assert( pParse->zErr==0 );
+          ){
-          pParse->zErr = sqlite3_mprintf(
+            assert( pParse->rc==SQLITE_OK );
-              "fts5: %s queries are not supported (detail!=full)", 
+            pParse->rc = SQLITE_ERROR;
-              pNear->nPhrase==1 ? "phrase": "NEAR"
+            assert( pParse->zErr==0 );
-          );
+            pParse->zErr = sqlite3_mprintf(
-          sqlite3_free(pRet);
+                "fts5: %s queries are not supported (detail!=full)", 
-          pRet = 0;
+                pNear->nPhrase==1 ? "phrase": "NEAR"
+                );
+            sqlite3_free(pRet);
+            pRet = 0;
+          }
         }
-
       }else{
         fts5ExprAddChildren(pRet, pLeft);
         fts5ExprAddChildren(pRet, pRight);
@@ -2514,14 +2541,19 @@ static void fts5ExprIsAlnum(
   sqlite3_value **apVal           /* Function arguments */
 ){
   int iCode;
+  u8 aArr[32];
   if( nArg!=1 ){
     sqlite3_result_error(pCtx, 
         "wrong number of arguments to function fts5_isalnum", -1
     );
     return;
   }
+  memset(aArr, 0, sizeof(aArr));
+  sqlite3Fts5UnicodeCatParse("L*", aArr);
+  sqlite3Fts5UnicodeCatParse("N*", aArr);
+  sqlite3Fts5UnicodeCatParse("Co", aArr);
   iCode = sqlite3_value_int(apVal[0]);
-  sqlite3_result_int(pCtx, sqlite3Fts5UnicodeIsalnum(iCode));
+  sqlite3_result_int(pCtx, aArr[sqlite3Fts5UnicodeCategory(iCode)]);
 }
 
 static void fts5ExprFold(
@@ -2565,10 +2597,12 @@ int sqlite3Fts5ExprInit(Fts5Global *pGlobal, sqlite3 *db){
     rc = sqlite3_create_function(db, p->z, -1, SQLITE_UTF8, pCtx, p->x, 0, 0);
   }
 
-  /* Avoid a warning indicating that sqlite3Fts5ParserTrace() is unused */
+  /* Avoid warnings indicating that sqlite3Fts5ParserTrace() and
+  ** sqlite3Fts5ParserFallback() are unused */
 #ifndef NDEBUG
   (void)sqlite3Fts5ParserTrace;
 #endif
+  (void)sqlite3Fts5ParserFallback;
 
   return rc;
 }

ext/fts5/fts5_index.c

@@ -758,6 +758,7 @@ static void fts5DataWrite(Fts5Index *p, i64 iRowid, const u8 *pData, int nData){
   sqlite3_bind_blob(p->pWriter, 2, pData, nData, SQLITE_STATIC);
   sqlite3_step(p->pWriter);
   p->rc = sqlite3_reset(p->pWriter);
+  sqlite3_bind_null(p->pWriter, 2);
 }
 
 /*
@@ -2386,6 +2387,7 @@ static void fts5SegIterSeekInit(
     bDlidx = (val & 0x0001);
   }
   p->rc = sqlite3_reset(pIdxSelect);
+  sqlite3_bind_null(pIdxSelect, 2);
 
   if( iPg<pSeg->pgnoFirst ){
     iPg = pSeg->pgnoFirst;
@@ -3598,6 +3600,7 @@ static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){
           sqlite3_bind_blob(pIdxSelect, 2, aBlob, 2, SQLITE_STATIC);
           assert( sqlite3_step(pIdxSelect)!=SQLITE_ROW );
           p->rc = sqlite3_reset(pIdxSelect);
+          sqlite3_bind_null(pIdxSelect, 2);
         }
       }
 #endif
@@ -3724,6 +3727,7 @@ static void fts5WriteFlushBtree(Fts5Index *p, Fts5SegWriter *pWriter){
     sqlite3_bind_int64(p->pIdxWriter, 3, bFlag + ((i64)pWriter->iBtPage<<1));
     sqlite3_step(p->pIdxWriter);
     p->rc = sqlite3_reset(p->pIdxWriter);
+    sqlite3_bind_null(p->pIdxWriter, 2);
   }
   pWriter->iBtPage = 0;
 }
@@ -4909,7 +4913,13 @@ static void fts5MergePrefixLists(
     Fts5Buffer out = {0, 0, 0};
     Fts5Buffer tmp = {0, 0, 0};
 
-    if( sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n) ) return;
+    /* The maximum size of the output is equal to the sum of the two 
+    ** input sizes + 1 varint (9 bytes). The extra varint is because if the
+    ** first rowid in one input is a large negative number, and the first in
+    ** the other a non-negative number, the delta for the non-negative
+    ** number will be larger on disk than the literal integer value
+    ** was.  */
+    if( sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n + 9) ) return;
     fts5DoclistIterInit(p1, &i1);
     fts5DoclistIterInit(p2, &i2);
 
@@ -5003,6 +5013,7 @@ static void fts5MergePrefixLists(
       fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid);
       fts5BufferSafeAppendBlob(&out, i2.aPoslist, i2.aEof - i2.aPoslist);
     }
+    assert( out.n<=(p1->n+p2->n+9) );
 
     fts5BufferSet(&p->rc, p1, out.n, out.p);
     fts5BufferFree(&tmp);
@@ -5250,7 +5261,10 @@ int sqlite3Fts5IndexCharlenToBytelen(
   for(i=0; i<nChar; i++){
     if( n>=nByte ) return 0;      /* Input contains fewer than nChar chars */
     if( (unsigned char)p[n++]>=0xc0 ){
-      while( (p[n] & 0xc0)==0x80 ) n++;
+      while( (p[n] & 0xc0)==0x80 ){
+        n++;
+        if( n>=nByte ) break;
+      }
     }
   }
   return n;
@@ -5388,7 +5402,7 @@ int sqlite3Fts5IndexQuery(
       fts5CloseReader(p);
     }
 
-    *ppIter = &pRet->base;
+    *ppIter = (Fts5IndexIter*)pRet;
     sqlite3Fts5BufferFree(&buf);
   }
   return fts5IndexReturn(p);

ext/fts5/fts5_main.c

@@ -280,7 +280,7 @@ static void fts5CheckTransactionState(Fts5Table *p, int op, int iSavepoint){
     case FTS5_SAVEPOINT:
       assert( p->ts.eState==1 );
       assert( iSavepoint>=0 );
-      assert( iSavepoint>p->ts.iSavepoint );
+      assert( iSavepoint>=p->ts.iSavepoint );
       p->ts.iSavepoint = iSavepoint;
       break;
       
@@ -535,6 +535,12 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
   aColMap[1] = nCol;
   aColMap[2] = nCol+1;
 
+  assert( SQLITE_INDEX_CONSTRAINT_EQ<SQLITE_INDEX_CONSTRAINT_MATCH );
+  assert( SQLITE_INDEX_CONSTRAINT_GT<SQLITE_INDEX_CONSTRAINT_MATCH );
+  assert( SQLITE_INDEX_CONSTRAINT_LE<SQLITE_INDEX_CONSTRAINT_MATCH );
+  assert( SQLITE_INDEX_CONSTRAINT_GE<SQLITE_INDEX_CONSTRAINT_MATCH );
+  assert( SQLITE_INDEX_CONSTRAINT_LE<SQLITE_INDEX_CONSTRAINT_MATCH );
+
   /* Set idxFlags flags for all WHERE clause terms that will be used. */
   for(i=0; i<pInfo->nConstraint; i++){
     struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
@@ -553,11 +559,11 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
         pInfo->estimatedCost = 1e50;
         return SQLITE_OK;
       }
-    }else{
+    }else if( p->op<=SQLITE_INDEX_CONSTRAINT_MATCH ){
       int j;
       for(j=1; j<ArraySize(aConstraint); j++){
         struct Constraint *pC = &aConstraint[j];
-        if( iCol==aColMap[pC->iCol] && p->op & pC->op && p->usable ){
+        if( iCol==aColMap[pC->iCol] && (p->op & pC->op) && p->usable ){
           pC->iConsIndex = i;
           idxFlags |= pC->fts5op;
         }
@@ -1199,6 +1205,13 @@ static int fts5FilterMethod(
     assert( nVal==0 && pMatch==0 && bOrderByRank==0 && bDesc==0 );
     assert( pCsr->iLastRowid==LARGEST_INT64 );
     assert( pCsr->iFirstRowid==SMALLEST_INT64 );
+    if( pTab->pSortCsr->bDesc ){
+      pCsr->iLastRowid = pTab->pSortCsr->iFirstRowid;
+      pCsr->iFirstRowid = pTab->pSortCsr->iLastRowid;
+    }else{
+      pCsr->iLastRowid = pTab->pSortCsr->iLastRowid;
+      pCsr->iFirstRowid = pTab->pSortCsr->iFirstRowid;
+    }
     pCsr->ePlan = FTS5_PLAN_SOURCE;
     pCsr->pExpr = pTab->pSortCsr->pExpr;
     rc = fts5CursorFirst(pTab, pCsr, bDesc);
@@ -2632,9 +2645,24 @@ static void fts5SourceIdFunc(
   sqlite3_result_text(pCtx, "--FTS5-SOURCE-ID--", -1, SQLITE_TRANSIENT);
 }
 
+/*
+** Return true if zName is the extension on one of the shadow tables used
+** by this module.
+*/
+static int fts5ShadowName(const char *zName){
+  static const char *azName[] = {
+    "config", "content", "data", "docsize", "idx"
+  };
+  unsigned int i;
+  for(i=0; i<sizeof(azName)/sizeof(azName[0]); i++){
+    if( sqlite3_stricmp(zName, azName[i])==0 ) return 1;
+  }
+  return 0;
+}
+
 static int fts5Init(sqlite3 *db){
   static const sqlite3_module fts5Mod = {
-    /* iVersion      */ 2,
+    /* iVersion      */ 3,
     /* xCreate       */ fts5CreateMethod,
     /* xConnect      */ fts5ConnectMethod,
     /* xBestIndex    */ fts5BestIndexMethod,
@@ -2657,6 +2685,7 @@ static int fts5Init(sqlite3 *db){
     /* xSavepoint    */ fts5SavepointMethod,
     /* xRelease      */ fts5ReleaseMethod,
     /* xRollbackTo   */ fts5RollbackToMethod,
+    /* xShadowName   */ fts5ShadowName
   };
 
   int rc;

ext/fts5/fts5_storage.c

@@ -458,6 +458,7 @@ static int fts5StorageInsertDocsize(
       sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
       sqlite3_step(pReplace);
       rc = sqlite3_reset(pReplace);
+      sqlite3_bind_null(pReplace, 2);
     }
   }
   return rc;
@@ -1118,6 +1119,7 @@ int sqlite3Fts5StorageConfigValue(
     }
     sqlite3_step(pReplace);
     rc = sqlite3_reset(pReplace);
+    sqlite3_bind_null(pReplace, 1);
   }
   if( rc==SQLITE_OK && pVal ){
     int iNew = p->pConfig->iCookie + 1;

ext/fts5/fts5_tcl.c

@@ -433,7 +433,7 @@ static int SQLITE_TCLAPI xF5tApi(
       int iVal;
       int bClear;
       if( Tcl_GetBooleanFromObj(interp, objv[2], &bClear) ) return TCL_ERROR;
-      iVal = ((char*)p->pApi->xGetAuxdata(p->pFts, bClear) - (char*)0);
+      iVal = (int)((char*)p->pApi->xGetAuxdata(p->pFts, bClear) - (char*)0);
       Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal));
       break;
     }
@@ -482,7 +482,7 @@ static int SQLITE_TCLAPI xF5tApi(
 
       rc = p->pApi->xPhraseFirstColumn(p->pFts, iPhrase, &iter, &iCol);
       if( rc!=SQLITE_OK ){
-        Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
+        Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
         return TCL_ERROR;
       }
       for( ; iCol>=0; p->pApi->xPhraseNextColumn(p->pFts, &iter, &iCol)){
@@ -924,7 +924,7 @@ static int SQLITE_TCLAPI f5tTokenizerReturn(
 
   rc = p->xToken(p->pCtx, tflags, zToken, nToken, iStart, iEnd);
   Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
-  return TCL_OK;
+  return rc==SQLITE_OK ? TCL_OK : TCL_ERROR;
 
  usage:
   Tcl_WrongNumArgs(interp, 1, objv, "?-colocated? TEXT START END");

ext/fts5/fts5_test_tok.c

@@ -471,7 +471,8 @@ int sqlite3Fts5TestRegisterTok(sqlite3 *db, fts5_api *pApi){
      0,                           /* xRename       */
      0,                           /* xSavepoint    */
      0,                           /* xRelease      */
-     0                            /* xRollbackTo   */
+     0,                           /* xRollbackTo   */
+     0                            /* xShadowName   */
   };
   int rc;                         /* Return code */
 

ext/fts5/fts5_tokenize.c

@@ -237,6 +237,8 @@ struct Unicode61Tokenizer {
   int bRemoveDiacritic;           /* True if remove_diacritics=1 is set */
   int nException;
   int *aiException;
+
+  unsigned char aCategory[32];    /* True for token char categories */
 };
 
 static int fts5UnicodeAddExceptions(
@@ -261,7 +263,7 @@ static int fts5UnicodeAddExceptions(
         if( iCode<128 ){
           p->aTokenChar[iCode] = (unsigned char)bTokenChars;
         }else{
-          bToken = sqlite3Fts5UnicodeIsalnum(iCode);
+          bToken = p->aCategory[sqlite3Fts5UnicodeCategory(iCode)];
           assert( (bToken==0 || bToken==1) ); 
           assert( (bTokenChars==0 || bTokenChars==1) );
           if( bToken!=bTokenChars && sqlite3Fts5UnicodeIsdiacritic(iCode)==0 ){
@@ -322,6 +324,21 @@ static void fts5UnicodeDelete(Fts5Tokenizer *pTok){
   return;
 }
 
+static int unicodeSetCategories(Unicode61Tokenizer *p, const char *zCat){
+  const char *z = zCat;
+
+  while( *z ){
+    while( *z==' ' || *z=='\t' ) z++;
+    if( *z && sqlite3Fts5UnicodeCatParse(z, p->aCategory) ){
+      return SQLITE_ERROR;
+    }
+    while( *z!=' ' && *z!='\t' && *z!='\0' ) z++;
+  }
+
+  sqlite3Fts5UnicodeAscii(p->aCategory, p->aTokenChar);
+  return SQLITE_OK;
+}
+
 /*
 ** Create a "unicode61" tokenizer.
 */
@@ -340,15 +357,28 @@ static int fts5UnicodeCreate(
   }else{
     p = (Unicode61Tokenizer*)sqlite3_malloc(sizeof(Unicode61Tokenizer));
     if( p ){
+      const char *zCat = "L* N* Co";
       int i;
       memset(p, 0, sizeof(Unicode61Tokenizer));
-      memcpy(p->aTokenChar, aAsciiTokenChar, sizeof(aAsciiTokenChar));
+
       p->bRemoveDiacritic = 1;
       p->nFold = 64;
       p->aFold = sqlite3_malloc(p->nFold * sizeof(char));
       if( p->aFold==0 ){
         rc = SQLITE_NOMEM;
       }
+
+      /* Search for a "categories" argument */
+      for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
+        if( 0==sqlite3_stricmp(azArg[i], "categories") ){
+          zCat = azArg[i+1];
+        }
+      }
+
+      if( rc==SQLITE_OK ){
+        rc = unicodeSetCategories(p, zCat);
+      }
+
       for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
         const char *zArg = azArg[i+1];
         if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
@@ -362,10 +392,14 @@ static int fts5UnicodeCreate(
         }else
         if( 0==sqlite3_stricmp(azArg[i], "separators") ){
           rc = fts5UnicodeAddExceptions(p, zArg, 0);
+        }else
+        if( 0==sqlite3_stricmp(azArg[i], "categories") ){
+          /* no-op */
         }else{
           rc = SQLITE_ERROR;
         }
       }
+
     }else{
       rc = SQLITE_NOMEM;
     }
@@ -384,8 +418,10 @@ static int fts5UnicodeCreate(
 ** character (not a separator).
 */
 static int fts5UnicodeIsAlnum(Unicode61Tokenizer *p, int iCode){
-  assert( (sqlite3Fts5UnicodeIsalnum(iCode) & 0xFFFFFFFE)==0 );
+  return (
-  return sqlite3Fts5UnicodeIsalnum(iCode) ^ fts5UnicodeIsException(p, iCode);
+    p->aCategory[sqlite3Fts5UnicodeCategory(iCode)]
+    ^ fts5UnicodeIsException(p, iCode)
+  );
 }
 
 static int fts5UnicodeTokenize(

ext/fts5/fts5_unicode2.c

@@ -18,135 +18,6 @@
 
 #include <assert.h>
 
-/*
-** Return true if the argument corresponds to a unicode codepoint
-** classified as either a letter or a number. Otherwise false.
-**
-** The results are undefined if the value passed to this function
-** is less than zero.
-*/
-int sqlite3Fts5UnicodeIsalnum(int c){
-  /* Each unsigned integer in the following array corresponds to a contiguous
-  ** range of unicode codepoints that are not either letters or numbers (i.e.
-  ** codepoints for which this function should return 0).
-  **
-  ** The most significant 22 bits in each 32-bit value contain the first 
-  ** codepoint in the range. The least significant 10 bits are used to store
-  ** the size of the range (always at least 1). In other words, the value 
-  ** ((C<<22) + N) represents a range of N codepoints starting with codepoint 
-  ** C. It is not possible to represent a range larger than 1023 codepoints 
-  ** using this format.
-  */
-  static const unsigned int aEntry[] = {
-    0x00000030, 0x0000E807, 0x00016C06, 0x0001EC2F, 0x0002AC07,
-    0x0002D001, 0x0002D803, 0x0002EC01, 0x0002FC01, 0x00035C01,
-    0x0003DC01, 0x000B0804, 0x000B480E, 0x000B9407, 0x000BB401,
-    0x000BBC81, 0x000DD401, 0x000DF801, 0x000E1002, 0x000E1C01,
-    0x000FD801, 0x00120808, 0x00156806, 0x00162402, 0x00163C01,
-    0x00164437, 0x0017CC02, 0x00180005, 0x00181816, 0x00187802,
-    0x00192C15, 0x0019A804, 0x0019C001, 0x001B5001, 0x001B580F,
-    0x001B9C07, 0x001BF402, 0x001C000E, 0x001C3C01, 0x001C4401,
-    0x001CC01B, 0x001E980B, 0x001FAC09, 0x001FD804, 0x00205804,
-    0x00206C09, 0x00209403, 0x0020A405, 0x0020C00F, 0x00216403,
-    0x00217801, 0x0023901B, 0x00240004, 0x0024E803, 0x0024F812,
-    0x00254407, 0x00258804, 0x0025C001, 0x00260403, 0x0026F001,
-    0x0026F807, 0x00271C02, 0x00272C03, 0x00275C01, 0x00278802,
-    0x0027C802, 0x0027E802, 0x00280403, 0x0028F001, 0x0028F805,
-    0x00291C02, 0x00292C03, 0x00294401, 0x0029C002, 0x0029D401,
-    0x002A0403, 0x002AF001, 0x002AF808, 0x002B1C03, 0x002B2C03,
-    0x002B8802, 0x002BC002, 0x002C0403, 0x002CF001, 0x002CF807,
-    0x002D1C02, 0x002D2C03, 0x002D5802, 0x002D8802, 0x002DC001,
-    0x002E0801, 0x002EF805, 0x002F1803, 0x002F2804, 0x002F5C01,
-    0x002FCC08, 0x00300403, 0x0030F807, 0x00311803, 0x00312804,
-    0x00315402, 0x00318802, 0x0031FC01, 0x00320802, 0x0032F001,
-    0x0032F807, 0x00331803, 0x00332804, 0x00335402, 0x00338802,
-    0x00340802, 0x0034F807, 0x00351803, 0x00352804, 0x00355C01,
-    0x00358802, 0x0035E401, 0x00360802, 0x00372801, 0x00373C06,
-    0x00375801, 0x00376008, 0x0037C803, 0x0038C401, 0x0038D007,
-    0x0038FC01, 0x00391C09, 0x00396802, 0x003AC401, 0x003AD006,
-    0x003AEC02, 0x003B2006, 0x003C041F, 0x003CD00C, 0x003DC417,
-    0x003E340B, 0x003E6424, 0x003EF80F, 0x003F380D, 0x0040AC14,
-    0x00412806, 0x00415804, 0x00417803, 0x00418803, 0x00419C07,
-    0x0041C404, 0x0042080C, 0x00423C01, 0x00426806, 0x0043EC01,
-    0x004D740C, 0x004E400A, 0x00500001, 0x0059B402, 0x005A0001,
-    0x005A6C02, 0x005BAC03, 0x005C4803, 0x005CC805, 0x005D4802,
-    0x005DC802, 0x005ED023, 0x005F6004, 0x005F7401, 0x0060000F,
-    0x0062A401, 0x0064800C, 0x0064C00C, 0x00650001, 0x00651002,
-    0x0066C011, 0x00672002, 0x00677822, 0x00685C05, 0x00687802,
-    0x0069540A, 0x0069801D, 0x0069FC01, 0x006A8007, 0x006AA006,
-    0x006C0005, 0x006CD011, 0x006D6823, 0x006E0003, 0x006E840D,
-    0x006F980E, 0x006FF004, 0x00709014, 0x0070EC05, 0x0071F802,
-    0x00730008, 0x00734019, 0x0073B401, 0x0073C803, 0x00770027,
-    0x0077F004, 0x007EF401, 0x007EFC03, 0x007F3403, 0x007F7403,
-    0x007FB403, 0x007FF402, 0x00800065, 0x0081A806, 0x0081E805,
-    0x00822805, 0x0082801A, 0x00834021, 0x00840002, 0x00840C04,
-    0x00842002, 0x00845001, 0x00845803, 0x00847806, 0x00849401,
-    0x00849C01, 0x0084A401, 0x0084B801, 0x0084E802, 0x00850005,
-    0x00852804, 0x00853C01, 0x00864264, 0x00900027, 0x0091000B,
-    0x0092704E, 0x00940200, 0x009C0475, 0x009E53B9, 0x00AD400A,
-    0x00B39406, 0x00B3BC03, 0x00B3E404, 0x00B3F802, 0x00B5C001,
-    0x00B5FC01, 0x00B7804F, 0x00B8C00C, 0x00BA001A, 0x00BA6C59,
-    0x00BC00D6, 0x00BFC00C, 0x00C00005, 0x00C02019, 0x00C0A807,
-    0x00C0D802, 0x00C0F403, 0x00C26404, 0x00C28001, 0x00C3EC01,
-    0x00C64002, 0x00C6580A, 0x00C70024, 0x00C8001F, 0x00C8A81E,
-    0x00C94001, 0x00C98020, 0x00CA2827, 0x00CB003F, 0x00CC0100,
-    0x01370040, 0x02924037, 0x0293F802, 0x02983403, 0x0299BC10,
-    0x029A7C01, 0x029BC008, 0x029C0017, 0x029C8002, 0x029E2402,
-    0x02A00801, 0x02A01801, 0x02A02C01, 0x02A08C09, 0x02A0D804,
-    0x02A1D004, 0x02A20002, 0x02A2D011, 0x02A33802, 0x02A38012,
-    0x02A3E003, 0x02A4980A, 0x02A51C0D, 0x02A57C01, 0x02A60004,
-    0x02A6CC1B, 0x02A77802, 0x02A8A40E, 0x02A90C01, 0x02A93002,
-    0x02A97004, 0x02A9DC03, 0x02A9EC01, 0x02AAC001, 0x02AAC803,
-    0x02AADC02, 0x02AAF802, 0x02AB0401, 0x02AB7802, 0x02ABAC07,
-    0x02ABD402, 0x02AF8C0B, 0x03600001, 0x036DFC02, 0x036FFC02,
-    0x037FFC01, 0x03EC7801, 0x03ECA401, 0x03EEC810, 0x03F4F802,
-    0x03F7F002, 0x03F8001A, 0x03F88007, 0x03F8C023, 0x03F95013,
-    0x03F9A004, 0x03FBFC01, 0x03FC040F, 0x03FC6807, 0x03FCEC06,
-    0x03FD6C0B, 0x03FF8007, 0x03FFA007, 0x03FFE405, 0x04040003,
-    0x0404DC09, 0x0405E411, 0x0406400C, 0x0407402E, 0x040E7C01,
-    0x040F4001, 0x04215C01, 0x04247C01, 0x0424FC01, 0x04280403,
-    0x04281402, 0x04283004, 0x0428E003, 0x0428FC01, 0x04294009,
-    0x0429FC01, 0x042CE407, 0x04400003, 0x0440E016, 0x04420003,
-    0x0442C012, 0x04440003, 0x04449C0E, 0x04450004, 0x04460003,
-    0x0446CC0E, 0x04471404, 0x045AAC0D, 0x0491C004, 0x05BD442E,
-    0x05BE3C04, 0x074000F6, 0x07440027, 0x0744A4B5, 0x07480046,
-    0x074C0057, 0x075B0401, 0x075B6C01, 0x075BEC01, 0x075C5401,
-    0x075CD401, 0x075D3C01, 0x075DBC01, 0x075E2401, 0x075EA401,
-    0x075F0C01, 0x07BBC002, 0x07C0002C, 0x07C0C064, 0x07C2800F,
-    0x07C2C40E, 0x07C3040F, 0x07C3440F, 0x07C4401F, 0x07C4C03C,
-    0x07C5C02B, 0x07C7981D, 0x07C8402B, 0x07C90009, 0x07C94002,
-    0x07CC0021, 0x07CCC006, 0x07CCDC46, 0x07CE0014, 0x07CE8025,
-    0x07CF1805, 0x07CF8011, 0x07D0003F, 0x07D10001, 0x07D108B6,
-    0x07D3E404, 0x07D4003E, 0x07D50004, 0x07D54018, 0x07D7EC46,
-    0x07D9140B, 0x07DA0046, 0x07DC0074, 0x38000401, 0x38008060,
-    0x380400F0,
-  };
-  static const unsigned int aAscii[4] = {
-    0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001,
-  };
-
-  if( (unsigned int)c<128 ){
-    return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 );
-  }else if( (unsigned int)c<(1<<22) ){
-    unsigned int key = (((unsigned int)c)<<10) | 0x000003FF;
-    int iRes = 0;
-    int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
-    int iLo = 0;
-    while( iHi>=iLo ){
-      int iTest = (iHi + iLo) / 2;
-      if( key >= aEntry[iTest] ){
-        iRes = iTest;
-        iLo = iTest+1;
-      }else{
-        iHi = iTest-1;
-      }
-    }
-    assert( aEntry[0]<key );
-    assert( key>=aEntry[iRes] );
-    return (((unsigned int)c) >= ((aEntry[iRes]>>10) + (aEntry[iRes]&0x3FF)));
-  }
-  return 1;
-}
 
 
 /*
@@ -358,3 +229,536 @@ int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic){
 
   return ret;
 }
+
+
+#if 0
+int sqlite3Fts5UnicodeNCat(void) { 
+  return 32;
+}
+#endif
+
+int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ 
+  aArray[0] = 1;
+  switch( zCat[0] ){
+    case 'C':
+          switch( zCat[1] ){
+            case 'c': aArray[1] = 1; break;
+            case 'f': aArray[2] = 1; break;
+            case 'n': aArray[3] = 1; break;
+            case 's': aArray[4] = 1; break;
+            case 'o': aArray[31] = 1; break;
+            case '*': 
+              aArray[1] = 1;
+              aArray[2] = 1;
+              aArray[3] = 1;
+              aArray[4] = 1;
+              aArray[31] = 1;
+              break;
+            default: return 1;          }
+          break;
+
+    case 'L':
+          switch( zCat[1] ){
+            case 'l': aArray[5] = 1; break;
+            case 'm': aArray[6] = 1; break;
+            case 'o': aArray[7] = 1; break;
+            case 't': aArray[8] = 1; break;
+            case 'u': aArray[9] = 1; break;
+            case 'C': aArray[30] = 1; break;
+            case '*': 
+              aArray[5] = 1;
+              aArray[6] = 1;
+              aArray[7] = 1;
+              aArray[8] = 1;
+              aArray[9] = 1;
+              aArray[30] = 1;
+              break;
+            default: return 1;          }
+          break;
+
+    case 'M':
+          switch( zCat[1] ){
+            case 'c': aArray[10] = 1; break;
+            case 'e': aArray[11] = 1; break;
+            case 'n': aArray[12] = 1; break;
+            case '*': 
+              aArray[10] = 1;
+              aArray[11] = 1;
+              aArray[12] = 1;
+              break;
+            default: return 1;          }
+          break;
+
+    case 'N':
+          switch( zCat[1] ){
+            case 'd': aArray[13] = 1; break;
+            case 'l': aArray[14] = 1; break;
+            case 'o': aArray[15] = 1; break;
+            case '*': 
+              aArray[13] = 1;
+              aArray[14] = 1;
+              aArray[15] = 1;
+              break;
+            default: return 1;          }
+          break;
+
+    case 'P':
+          switch( zCat[1] ){
+            case 'c': aArray[16] = 1; break;
+            case 'd': aArray[17] = 1; break;
+            case 'e': aArray[18] = 1; break;
+            case 'f': aArray[19] = 1; break;
+            case 'i': aArray[20] = 1; break;
+            case 'o': aArray[21] = 1; break;
+            case 's': aArray[22] = 1; break;
+            case '*': 
+              aArray[16] = 1;
+              aArray[17] = 1;
+              aArray[18] = 1;
+              aArray[19] = 1;
+              aArray[20] = 1;
+              aArray[21] = 1;
+              aArray[22] = 1;
+              break;
+            default: return 1;          }
+          break;
+
+    case 'S':
+          switch( zCat[1] ){
+            case 'c': aArray[23] = 1; break;
+            case 'k': aArray[24] = 1; break;
+            case 'm': aArray[25] = 1; break;
+            case 'o': aArray[26] = 1; break;
+            case '*': 
+              aArray[23] = 1;
+              aArray[24] = 1;
+              aArray[25] = 1;
+              aArray[26] = 1;
+              break;
+            default: return 1;          }
+          break;
+
+    case 'Z':
+          switch( zCat[1] ){
+            case 'l': aArray[27] = 1; break;
+            case 'p': aArray[28] = 1; break;
+            case 's': aArray[29] = 1; break;
+            case '*': 
+              aArray[27] = 1;
+              aArray[28] = 1;
+              aArray[29] = 1;
+              break;
+            default: return 1;          }
+          break;
+
+  }
+  return 0;
+}
+
+static u16 aFts5UnicodeBlock[] = {
+    0,     1471,  1753,  1760,  1760,  1760,  1760,  1760,  1760,  1760,  
+    1760,  1760,  1760,  1760,  1760,  1763,  1765,  
+  };
+static u16 aFts5UnicodeMap[] = {
+    0,     32,    33,    36,    37,    40,    41,    42,    43,    44,    
+    45,    46,    48,    58,    60,    63,    65,    91,    92,    93,    
+    94,    95,    96,    97,    123,   124,   125,   126,   127,   160,   
+    161,   162,   166,   167,   168,   169,   170,   171,   172,   173,   
+    174,   175,   176,   177,   178,   180,   181,   182,   184,   185,   
+    186,   187,   188,   191,   192,   215,   216,   223,   247,   248,   
+    256,   312,   313,   329,   330,   377,   383,   385,   387,   388,   
+    391,   394,   396,   398,   402,   403,   405,   406,   409,   412,   
+    414,   415,   417,   418,   423,   427,   428,   431,   434,   436,   
+    437,   440,   442,   443,   444,   446,   448,   452,   453,   454,   
+    455,   456,   457,   458,   459,   460,   461,   477,   478,   496,   
+    497,   498,   499,   500,   503,   505,   506,   564,   570,   572,   
+    573,   575,   577,   580,   583,   584,   592,   660,   661,   688,   
+    706,   710,   722,   736,   741,   748,   749,   750,   751,   768,   
+    880,   884,   885,   886,   890,   891,   894,   900,   902,   903,   
+    904,   908,   910,   912,   913,   931,   940,   975,   977,   978,   
+    981,   984,   1008,  1012,  1014,  1015,  1018,  1020,  1021,  1072,  
+    1120,  1154,  1155,  1160,  1162,  1217,  1231,  1232,  1329,  1369,  
+    1370,  1377,  1417,  1418,  1423,  1425,  1470,  1471,  1472,  1473,  
+    1475,  1476,  1478,  1479,  1488,  1520,  1523,  1536,  1542,  1545,  
+    1547,  1548,  1550,  1552,  1563,  1566,  1568,  1600,  1601,  1611,  
+    1632,  1642,  1646,  1648,  1649,  1748,  1749,  1750,  1757,  1758,  
+    1759,  1765,  1767,  1769,  1770,  1774,  1776,  1786,  1789,  1791,  
+    1792,  1807,  1808,  1809,  1810,  1840,  1869,  1958,  1969,  1984,  
+    1994,  2027,  2036,  2038,  2039,  2042,  2048,  2070,  2074,  2075,  
+    2084,  2085,  2088,  2089,  2096,  2112,  2137,  2142,  2208,  2210,  
+    2276,  2304,  2307,  2308,  2362,  2363,  2364,  2365,  2366,  2369,  
+    2377,  2381,  2382,  2384,  2385,  2392,  2402,  2404,  2406,  2416,  
+    2417,  2418,  2425,  2433,  2434,  2437,  2447,  2451,  2474,  2482,  
+    2486,  2492,  2493,  2494,  2497,  2503,  2507,  2509,  2510,  2519,  
+    2524,  2527,  2530,  2534,  2544,  2546,  2548,  2554,  2555,  2561,  
+    2563,  2565,  2575,  2579,  2602,  2610,  2613,  2616,  2620,  2622,  
+    2625,  2631,  2635,  2641,  2649,  2654,  2662,  2672,  2674,  2677,  
+    2689,  2691,  2693,  2703,  2707,  2730,  2738,  2741,  2748,  2749,  
+    2750,  2753,  2759,  2761,  2763,  2765,  2768,  2784,  2786,  2790,  
+    2800,  2801,  2817,  2818,  2821,  2831,  2835,  2858,  2866,  2869,  
+    2876,  2877,  2878,  2879,  2880,  2881,  2887,  2891,  2893,  2902,  
+    2903,  2908,  2911,  2914,  2918,  2928,  2929,  2930,  2946,  2947,  
+    2949,  2958,  2962,  2969,  2972,  2974,  2979,  2984,  2990,  3006,  
+    3008,  3009,  3014,  3018,  3021,  3024,  3031,  3046,  3056,  3059,  
+    3065,  3066,  3073,  3077,  3086,  3090,  3114,  3125,  3133,  3134,  
+    3137,  3142,  3146,  3157,  3160,  3168,  3170,  3174,  3192,  3199,  
+    3202,  3205,  3214,  3218,  3242,  3253,  3260,  3261,  3262,  3263,  
+    3264,  3270,  3271,  3274,  3276,  3285,  3294,  3296,  3298,  3302,  
+    3313,  3330,  3333,  3342,  3346,  3389,  3390,  3393,  3398,  3402,  
+    3405,  3406,  3415,  3424,  3426,  3430,  3440,  3449,  3450,  3458,  
+    3461,  3482,  3507,  3517,  3520,  3530,  3535,  3538,  3542,  3544,  
+    3570,  3572,  3585,  3633,  3634,  3636,  3647,  3648,  3654,  3655,  
+    3663,  3664,  3674,  3713,  3716,  3719,  3722,  3725,  3732,  3737,  
+    3745,  3749,  3751,  3754,  3757,  3761,  3762,  3764,  3771,  3773,  
+    3776,  3782,  3784,  3792,  3804,  3840,  3841,  3844,  3859,  3860,  
+    3861,  3864,  3866,  3872,  3882,  3892,  3893,  3894,  3895,  3896,  
+    3897,  3898,  3899,  3900,  3901,  3902,  3904,  3913,  3953,  3967,  
+    3968,  3973,  3974,  3976,  3981,  3993,  4030,  4038,  4039,  4046,  
+    4048,  4053,  4057,  4096,  4139,  4141,  4145,  4146,  4152,  4153,  
+    4155,  4157,  4159,  4160,  4170,  4176,  4182,  4184,  4186,  4190,  
+    4193,  4194,  4197,  4199,  4206,  4209,  4213,  4226,  4227,  4229,  
+    4231,  4237,  4238,  4239,  4240,  4250,  4253,  4254,  4256,  4295,  
+    4301,  4304,  4347,  4348,  4349,  4682,  4688,  4696,  4698,  4704,  
+    4746,  4752,  4786,  4792,  4800,  4802,  4808,  4824,  4882,  4888,  
+    4957,  4960,  4969,  4992,  5008,  5024,  5120,  5121,  5741,  5743,  
+    5760,  5761,  5787,  5788,  5792,  5867,  5870,  5888,  5902,  5906,  
+    5920,  5938,  5941,  5952,  5970,  5984,  5998,  6002,  6016,  6068,  
+    6070,  6071,  6078,  6086,  6087,  6089,  6100,  6103,  6104,  6107,  
+    6108,  6109,  6112,  6128,  6144,  6150,  6151,  6155,  6158,  6160,  
+    6176,  6211,  6212,  6272,  6313,  6314,  6320,  6400,  6432,  6435,  
+    6439,  6441,  6448,  6450,  6451,  6457,  6464,  6468,  6470,  6480,  
+    6512,  6528,  6576,  6593,  6600,  6608,  6618,  6622,  6656,  6679,  
+    6681,  6686,  6688,  6741,  6742,  6743,  6744,  6752,  6753,  6754,  
+    6755,  6757,  6765,  6771,  6783,  6784,  6800,  6816,  6823,  6824,  
+    6912,  6916,  6917,  6964,  6965,  6966,  6971,  6972,  6973,  6978,  
+    6979,  6981,  6992,  7002,  7009,  7019,  7028,  7040,  7042,  7043,  
+    7073,  7074,  7078,  7080,  7082,  7083,  7084,  7086,  7088,  7098,  
+    7142,  7143,  7144,  7146,  7149,  7150,  7151,  7154,  7164,  7168,  
+    7204,  7212,  7220,  7222,  7227,  7232,  7245,  7248,  7258,  7288,  
+    7294,  7360,  7376,  7379,  7380,  7393,  7394,  7401,  7405,  7406,  
+    7410,  7412,  7413,  7424,  7468,  7531,  7544,  7545,  7579,  7616,  
+    7676,  7680,  7830,  7838,  7936,  7944,  7952,  7960,  7968,  7976,  
+    7984,  7992,  8000,  8008,  8016,  8025,  8027,  8029,  8031,  8033,  
+    8040,  8048,  8064,  8072,  8080,  8088,  8096,  8104,  8112,  8118,  
+    8120,  8124,  8125,  8126,  8127,  8130,  8134,  8136,  8140,  8141,  
+    8144,  8150,  8152,  8157,  8160,  8168,  8173,  8178,  8182,  8184,  
+    8188,  8189,  8192,  8203,  8208,  8214,  8216,  8217,  8218,  8219,  
+    8221,  8222,  8223,  8224,  8232,  8233,  8234,  8239,  8240,  8249,  
+    8250,  8251,  8255,  8257,  8260,  8261,  8262,  8263,  8274,  8275,  
+    8276,  8277,  8287,  8288,  8298,  8304,  8305,  8308,  8314,  8317,  
+    8318,  8319,  8320,  8330,  8333,  8334,  8336,  8352,  8400,  8413,  
+    8417,  8418,  8421,  8448,  8450,  8451,  8455,  8456,  8458,  8459,  
+    8462,  8464,  8467,  8468,  8469,  8470,  8472,  8473,  8478,  8484,  
+    8485,  8486,  8487,  8488,  8489,  8490,  8494,  8495,  8496,  8500,  
+    8501,  8505,  8506,  8508,  8510,  8512,  8517,  8519,  8522,  8523,  
+    8524,  8526,  8527,  8528,  8544,  8579,  8581,  8585,  8592,  8597,  
+    8602,  8604,  8608,  8609,  8611,  8612,  8614,  8615,  8622,  8623,  
+    8654,  8656,  8658,  8659,  8660,  8661,  8692,  8960,  8968,  8972,  
+    8992,  8994,  9001,  9002,  9003,  9084,  9085,  9115,  9140,  9180,  
+    9186,  9216,  9280,  9312,  9372,  9450,  9472,  9655,  9656,  9665,  
+    9666,  9720,  9728,  9839,  9840,  9985,  10088, 10089, 10090, 10091, 
+    10092, 10093, 10094, 10095, 10096, 10097, 10098, 10099, 10100, 10101, 
+    10102, 10132, 10176, 10181, 10182, 10183, 10214, 10215, 10216, 10217, 
+    10218, 10219, 10220, 10221, 10222, 10223, 10224, 10240, 10496, 10627, 
+    10628, 10629, 10630, 10631, 10632, 10633, 10634, 10635, 10636, 10637, 
+    10638, 10639, 10640, 10641, 10642, 10643, 10644, 10645, 10646, 10647, 
+    10648, 10649, 10712, 10713, 10714, 10715, 10716, 10748, 10749, 10750, 
+    11008, 11056, 11077, 11079, 11088, 11264, 11312, 11360, 11363, 11365, 
+    11367, 11374, 11377, 11378, 11380, 11381, 11383, 11388, 11390, 11393, 
+    11394, 11492, 11493, 11499, 11503, 11506, 11513, 11517, 11518, 11520, 
+    11559, 11565, 11568, 11631, 11632, 11647, 11648, 11680, 11688, 11696, 
+    11704, 11712, 11720, 11728, 11736, 11744, 11776, 11778, 11779, 11780, 
+    11781, 11782, 11785, 11786, 11787, 11788, 11789, 11790, 11799, 11800, 
+    11802, 11803, 11804, 11805, 11806, 11808, 11809, 11810, 11811, 11812, 
+    11813, 11814, 11815, 11816, 11817, 11818, 11823, 11824, 11834, 11904, 
+    11931, 12032, 12272, 12288, 12289, 12292, 12293, 12294, 12295, 12296, 
+    12297, 12298, 12299, 12300, 12301, 12302, 12303, 12304, 12305, 12306, 
+    12308, 12309, 12310, 12311, 12312, 12313, 12314, 12315, 12316, 12317, 
+    12318, 12320, 12321, 12330, 12334, 12336, 12337, 12342, 12344, 12347, 
+    12348, 12349, 12350, 12353, 12441, 12443, 12445, 12447, 12448, 12449, 
+    12539, 12540, 12543, 12549, 12593, 12688, 12690, 12694, 12704, 12736, 
+    12784, 12800, 12832, 12842, 12872, 12880, 12881, 12896, 12928, 12938, 
+    12977, 12992, 13056, 13312, 19893, 19904, 19968, 40908, 40960, 40981, 
+    40982, 42128, 42192, 42232, 42238, 42240, 42508, 42509, 42512, 42528, 
+    42538, 42560, 42606, 42607, 42608, 42611, 42612, 42622, 42623, 42624, 
+    42655, 42656, 42726, 42736, 42738, 42752, 42775, 42784, 42786, 42800, 
+    42802, 42864, 42865, 42873, 42878, 42888, 42889, 42891, 42896, 42912, 
+    43000, 43002, 43003, 43010, 43011, 43014, 43015, 43019, 43020, 43043, 
+    43045, 43047, 43048, 43056, 43062, 43064, 43065, 43072, 43124, 43136, 
+    43138, 43188, 43204, 43214, 43216, 43232, 43250, 43256, 43259, 43264, 
+    43274, 43302, 43310, 43312, 43335, 43346, 43359, 43360, 43392, 43395, 
+    43396, 43443, 43444, 43446, 43450, 43452, 43453, 43457, 43471, 43472, 
+    43486, 43520, 43561, 43567, 43569, 43571, 43573, 43584, 43587, 43588, 
+    43596, 43597, 43600, 43612, 43616, 43632, 43633, 43639, 43642, 43643, 
+    43648, 43696, 43697, 43698, 43701, 43703, 43705, 43710, 43712, 43713, 
+    43714, 43739, 43741, 43742, 43744, 43755, 43756, 43758, 43760, 43762, 
+    43763, 43765, 43766, 43777, 43785, 43793, 43808, 43816, 43968, 44003, 
+    44005, 44006, 44008, 44009, 44011, 44012, 44013, 44016, 44032, 55203, 
+    55216, 55243, 55296, 56191, 56319, 57343, 57344, 63743, 63744, 64112, 
+    64256, 64275, 64285, 64286, 64287, 64297, 64298, 64312, 64318, 64320, 
+    64323, 64326, 64434, 64467, 64830, 64831, 64848, 64914, 65008, 65020, 
+    65021, 65024, 65040, 65047, 65048, 65049, 65056, 65072, 65073, 65075, 
+    65077, 65078, 65079, 65080, 65081, 65082, 65083, 65084, 65085, 65086, 
+    65087, 65088, 65089, 65090, 65091, 65092, 65093, 65095, 65096, 65097, 
+    65101, 65104, 65108, 65112, 65113, 65114, 65115, 65116, 65117, 65118, 
+    65119, 65122, 65123, 65124, 65128, 65129, 65130, 65136, 65142, 65279, 
+    65281, 65284, 65285, 65288, 65289, 65290, 65291, 65292, 65293, 65294, 
+    65296, 65306, 65308, 65311, 65313, 65339, 65340, 65341, 65342, 65343, 
+    65344, 65345, 65371, 65372, 65373, 65374, 65375, 65376, 65377, 65378, 
+    65379, 65380, 65382, 65392, 65393, 65438, 65440, 65474, 65482, 65490, 
+    65498, 65504, 65506, 65507, 65508, 65509, 65512, 65513, 65517, 65529, 
+    65532, 0,     13,    40,    60,    63,    80,    128,   256,   263,   
+    311,   320,   373,   377,   394,   400,   464,   509,   640,   672,   
+    768,   800,   816,   833,   834,   842,   896,   927,   928,   968,   
+    976,   977,   1024,  1064,  1104,  1184,  2048,  2056,  2058,  2103,  
+    2108,  2111,  2135,  2136,  2304,  2326,  2335,  2336,  2367,  2432,  
+    2494,  2560,  2561,  2565,  2572,  2576,  2581,  2585,  2616,  2623,  
+    2624,  2640,  2656,  2685,  2687,  2816,  2873,  2880,  2904,  2912,  
+    2936,  3072,  3680,  4096,  4097,  4098,  4099,  4152,  4167,  4178,  
+    4198,  4224,  4226,  4227,  4272,  4275,  4279,  4281,  4283,  4285,  
+    4286,  4304,  4336,  4352,  4355,  4391,  4396,  4397,  4406,  4416,  
+    4480,  4482,  4483,  4531,  4534,  4543,  4545,  4549,  4560,  5760,  
+    5803,  5804,  5805,  5806,  5808,  5814,  5815,  5824,  8192,  9216,  
+    9328,  12288, 26624, 28416, 28496, 28497, 28559, 28563, 45056, 53248, 
+    53504, 53545, 53605, 53607, 53610, 53613, 53619, 53627, 53635, 53637, 
+    53644, 53674, 53678, 53760, 53826, 53829, 54016, 54112, 54272, 54298, 
+    54324, 54350, 54358, 54376, 54402, 54428, 54430, 54434, 54437, 54441, 
+    54446, 54454, 54459, 54461, 54469, 54480, 54506, 54532, 54535, 54541, 
+    54550, 54558, 54584, 54587, 54592, 54598, 54602, 54610, 54636, 54662, 
+    54688, 54714, 54740, 54766, 54792, 54818, 54844, 54870, 54896, 54922, 
+    54952, 54977, 54978, 55003, 55004, 55010, 55035, 55036, 55061, 55062, 
+    55068, 55093, 55094, 55119, 55120, 55126, 55151, 55152, 55177, 55178, 
+    55184, 55209, 55210, 55235, 55236, 55242, 55246, 60928, 60933, 60961, 
+    60964, 60967, 60969, 60980, 60985, 60987, 60994, 60999, 61001, 61003, 
+    61005, 61009, 61012, 61015, 61017, 61019, 61021, 61023, 61025, 61028, 
+    61031, 61036, 61044, 61049, 61054, 61056, 61067, 61089, 61093, 61099, 
+    61168, 61440, 61488, 61600, 61617, 61633, 61649, 61696, 61712, 61744, 
+    61808, 61926, 61968, 62016, 62032, 62208, 62256, 62263, 62336, 62368, 
+    62406, 62432, 62464, 62528, 62530, 62713, 62720, 62784, 62800, 62971, 
+    63045, 63104, 63232, 0,     42710, 42752, 46900, 46912, 47133, 63488, 
+    1,     32,    256,   0,     65533, 
+  };
+static u16 aFts5UnicodeData[] = {
+    1025,  61,    117,   55,    117,   54,    50,    53,    57,    53,    
+    49,    85,    333,   85,    121,   85,    841,   54,    53,    50,    
+    56,    48,    56,    837,   54,    57,    50,    57,    1057,  61,    
+    53,    151,   58,    53,    56,    58,    39,    52,    57,    34,    
+    58,    56,    58,    57,    79,    56,    37,    85,    56,    47,    
+    39,    51,    111,   53,    745,   57,    233,   773,   57,    261,   
+    1822,  37,    542,   37,    1534,  222,   69,    73,    37,    126,   
+    126,   73,    69,    137,   37,    73,    37,    105,   101,   73,    
+    37,    73,    37,    190,   158,   37,    126,   126,   73,    37,    
+    126,   94,    37,    39,    94,    69,    135,   41,    40,    37,    
+    41,    40,    37,    41,    40,    37,    542,   37,    606,   37,    
+    41,    40,    37,    126,   73,    37,    1886,  197,   73,    37,    
+    73,    69,    126,   105,   37,    286,   2181,  39,    869,   582,   
+    152,   390,   472,   166,   248,   38,    56,    38,    568,   3596,  
+    158,   38,    56,    94,    38,    101,   53,    88,    41,    53,    
+    105,   41,    73,    37,    553,   297,   1125,  94,    37,    105,   
+    101,   798,   133,   94,    57,    126,   94,    37,    1641,  1541,  
+    1118,  58,    172,   75,    1790,  478,   37,    2846,  1225,  38,    
+    213,   1253,  53,    49,    55,    1452,  49,    44,    53,    76,    
+    53,    76,    53,    44,    871,   103,   85,    162,   121,   85,    
+    55,    85,    90,    364,   53,    85,    1031,  38,    327,   684,   
+    333,   149,   71,    44,    3175,  53,    39,    236,   34,    58,    
+    204,   70,    76,    58,    140,   71,    333,   103,   90,    39,    
+    469,   34,    39,    44,    967,   876,   2855,  364,   39,    333,   
+    1063,  300,   70,    58,    117,   38,    711,   140,   38,    300,   
+    38,    108,   38,    172,   501,   807,   108,   53,    39,    359,   
+    876,   108,   42,    1735,  44,    42,    44,    39,    106,   268,   
+    138,   44,    74,    39,    236,   327,   76,    85,    333,   53,    
+    38,    199,   231,   44,    74,    263,   71,    711,   231,   39,    
+    135,   44,    39,    106,   140,   74,    74,    44,    39,    42,    
+    71,    103,   76,    333,   71,    87,    207,   58,    55,    76,    
+    42,    199,   71,    711,   231,   71,    71,    71,    44,    106,   
+    76,    76,    108,   44,    135,   39,    333,   76,    103,   44,    
+    76,    42,    295,   103,   711,   231,   71,    167,   44,    39,    
+    106,   172,   76,    42,    74,    44,    39,    71,    76,    333,   
+    53,    55,    44,    74,    263,   71,    711,   231,   71,    167,   
+    44,    39,    42,    44,    42,    140,   74,    74,    44,    44,    
+    42,    71,    103,   76,    333,   58,    39,    207,   44,    39,    
+    199,   103,   135,   71,    39,    71,    71,    103,   391,   74,    
+    44,    74,    106,   106,   44,    39,    42,    333,   111,   218,   
+    55,    58,    106,   263,   103,   743,   327,   167,   39,    108,   
+    138,   108,   140,   76,    71,    71,    76,    333,   239,   58,    
+    74,    263,   103,   743,   327,   167,   44,    39,    42,    44,    
+    170,   44,    74,    74,    76,    74,    39,    71,    76,    333,   
+    71,    74,    263,   103,   1319,  39,    106,   140,   106,   106,   
+    44,    39,    42,    71,    76,    333,   207,   58,    199,   74,    
+    583,   775,   295,   39,    231,   44,    106,   108,   44,    266,   
+    74,    53,    1543,  44,    71,    236,   55,    199,   38,    268,   
+    53,    333,   85,    71,    39,    71,    39,    39,    135,   231,   
+    103,   39,    39,    71,    135,   44,    71,    204,   76,    39,    
+    167,   38,    204,   333,   135,   39,    122,   501,   58,    53,    
+    122,   76,    218,   333,   335,   58,    44,    58,    44,    58,    
+    44,    54,    50,    54,    50,    74,    263,   1159,  460,   42,    
+    172,   53,    76,    167,   364,   1164,  282,   44,    218,   90,    
+    181,   154,   85,    1383,  74,    140,   42,    204,   42,    76,    
+    74,    76,    39,    333,   213,   199,   74,    76,    135,   108,   
+    39,    106,   71,    234,   103,   140,   423,   44,    74,    76,    
+    202,   44,    39,    42,    333,   106,   44,    90,    1225,  41,    
+    41,    1383,  53,    38,    10631, 135,   231,   39,    135,   1319,  
+    135,   1063,  135,   231,   39,    135,   487,   1831,  135,   2151,  
+    108,   309,   655,   519,   346,   2727,  49,    19847, 85,    551,   
+    61,    839,   54,    50,    2407,  117,   110,   423,   135,   108,   
+    583,   108,   85,    583,   76,    423,   103,   76,    1671,  76,    
+    42,    236,   266,   44,    74,    364,   117,   38,    117,   55,    
+    39,    44,    333,   335,   213,   49,    149,   108,   61,    333,   
+    1127,  38,    1671,  1319,  44,    39,    2247,  935,   108,   138,   
+    76,    106,   74,    44,    202,   108,   58,    85,    333,   967,   
+    167,   1415,  554,   231,   74,    333,   47,    1114,  743,   76,    
+    106,   85,    1703,  42,    44,    42,    236,   44,    42,    44,    
+    74,    268,   202,   332,   44,    333,   333,   245,   38,    213,   
+    140,   42,    1511,  44,    42,    172,   42,    44,    170,   44,    
+    74,    231,   333,   245,   346,   300,   314,   76,    42,    967,   
+    42,    140,   74,    76,    42,    44,    74,    71,    333,   1415,  
+    44,    42,    76,    106,   44,    42,    108,   74,    149,   1159,  
+    266,   268,   74,    76,    181,   333,   103,   333,   967,   198,   
+    85,    277,   108,   53,    428,   42,    236,   135,   44,    135,   
+    74,    44,    71,    1413,  2022,  421,   38,    1093,  1190,  1260,  
+    140,   4830,  261,   3166,  261,   265,   197,   201,   261,   265,   
+    261,   265,   197,   201,   261,   41,    41,    41,    94,    229,   
+    265,   453,   261,   264,   261,   264,   261,   264,   165,   69,    
+    137,   40,    56,    37,    120,   101,   69,    137,   40,    120,   
+    133,   69,    137,   120,   261,   169,   120,   101,   69,    137,   
+    40,    88,    381,   162,   209,   85,    52,    51,    54,    84,    
+    51,    54,    52,    277,   59,    60,    162,   61,    309,   52,    
+    51,    149,   80,    117,   57,    54,    50,    373,   57,    53,    
+    48,    341,   61,    162,   194,   47,    38,    207,   121,   54,    
+    50,    38,    335,   121,   54,    50,    422,   855,   428,   139,   
+    44,    107,   396,   90,    41,    154,   41,    90,    37,    105,   
+    69,    105,   37,    58,    41,    90,    57,    169,   218,   41,    
+    58,    41,    58,    41,    58,    137,   58,    37,    137,   37,    
+    135,   37,    90,    69,    73,    185,   94,    101,   58,    57,    
+    90,    37,    58,    527,   1134,  94,    142,   47,    185,   186,   
+    89,    154,   57,    90,    57,    90,    57,    250,   57,    1018,  
+    89,    90,    57,    58,    57,    1018,  8601,  282,   153,   666,   
+    89,    250,   54,    50,    2618,  57,    986,   825,   1306,  217,   
+    602,   1274,  378,   1935,  2522,  719,   5882,  57,    314,   57,    
+    1754,  281,   3578,  57,    4634,  3322,  54,    50,    54,    50,    
+    54,    50,    54,    50,    54,    50,    54,    50,    54,    50,    
+    975,   1434,  185,   54,    50,    1017,  54,    50,    54,    50,    
+    54,    50,    54,    50,    54,    50,    537,   8218,  4217,  54,    
+    50,    54,    50,    54,    50,    54,    50,    54,    50,    54,    
+    50,    54,    50,    54,    50,    54,    50,    54,    50,    54,    
+    50,    2041,  54,    50,    54,    50,    1049,  54,    50,    8281,  
+    1562,  697,   90,    217,   346,   1513,  1509,  126,   73,    69,    
+    254,   105,   37,    94,    37,    94,    165,   70,    105,   37,    
+    3166,  37,    218,   158,   108,   94,    149,   47,    85,    1221,  
+    37,    37,    1799,  38,    53,    44,    743,   231,   231,   231,   
+    231,   231,   231,   231,   231,   1036,  85,    52,    51,    52,    
+    51,    117,   52,    51,    53,    52,    51,    309,   49,    85,    
+    49,    53,    52,    51,    85,    52,    51,    54,    50,    54,    
+    50,    54,    50,    54,    50,    181,   38,    341,   81,    858,   
+    2874,  6874,  410,   61,    117,   58,    38,    39,    46,    54,    
+    50,    54,    50,    54,    50,    54,    50,    54,    50,    90,    
+    54,    50,    54,    50,    54,    50,    54,    50,    49,    54,    
+    82,    58,    302,   140,   74,    49,    166,   90,    110,   38,    
+    39,    53,    90,    2759,  76,    88,    70,    39,    49,    2887,  
+    53,    102,   39,    1319,  3015,  90,    143,   346,   871,   1178,  
+    519,   1018,  335,   986,   271,   58,    495,   1050,  335,   1274,  
+    495,   2042,  8218,  39,    39,    2074,  39,    39,    679,   38,    
+    36583, 1786,  1287,  198,   85,    8583,  38,    117,   519,   333,   
+    71,    1502,  39,    44,    107,   53,    332,   53,    38,    798,   
+    44,    2247,  334,   76,    213,   760,   294,   88,    478,   69,    
+    2014,  38,    261,   190,   350,   38,    88,    158,   158,   382,   
+    70,    37,    231,   44,    103,   44,    135,   44,    743,   74,    
+    76,    42,    154,   207,   90,    55,    58,    1671,  149,   74,    
+    1607,  522,   44,    85,    333,   588,   199,   117,   39,    333,   
+    903,   268,   85,    743,   364,   74,    53,    935,   108,   42,    
+    1511,  44,    74,    140,   74,    44,    138,   437,   38,    333,   
+    85,    1319,  204,   74,    76,    74,    76,    103,   44,    263,   
+    44,    42,    333,   149,   519,   38,    199,   122,   39,    42,    
+    1543,  44,    39,    108,   71,    76,    167,   76,    39,    44,    
+    39,    71,    38,    85,    359,   42,    76,    74,    85,    39,    
+    70,    42,    44,    199,   199,   199,   231,   231,   1127,  74,    
+    44,    74,    44,    74,    53,    42,    44,    333,   39,    39,    
+    743,   1575,  36,    68,    68,    36,    63,    63,    11719, 3399,  
+    229,   165,   39,    44,    327,   57,    423,   167,   39,    71,    
+    71,    3463,  536,   11623, 54,    50,    2055,  1735,  391,   55,    
+    58,    524,   245,   54,    50,    53,    236,   53,    81,    80,    
+    54,    50,    54,    50,    54,    50,    54,    50,    54,    50,    
+    54,    50,    54,    50,    54,    50,    85,    54,    50,    149,   
+    112,   117,   149,   49,    54,    50,    54,    50,    54,    50,    
+    117,   57,    49,    121,   53,    55,    85,    167,   4327,  34,    
+    117,   55,    117,   54,    50,    53,    57,    53,    49,    85,    
+    333,   85,    121,   85,    841,   54,    53,    50,    56,    48,    
+    56,    837,   54,    57,    50,    57,    54,    50,    53,    54,    
+    50,    85,    327,   38,    1447,  70,    999,   199,   199,   199,   
+    103,   87,    57,    56,    58,    87,    58,    153,   90,    98,    
+    90,    391,   839,   615,   71,    487,   455,   3943,  117,   1455,  
+    314,   1710,  143,   570,   47,    410,   1466,  44,    935,   1575,  
+    999,   143,   551,   46,    263,   46,    967,   53,    1159,  263,   
+    53,    174,   1289,  1285,  2503,  333,   199,   39,    1415,  71,    
+    39,    743,   53,    271,   711,   207,   53,    839,   53,    1799,  
+    71,    39,    108,   76,    140,   135,   103,   871,   108,   44,    
+    271,   309,   935,   79,    53,    1735,  245,   711,   271,   615,   
+    271,   2343,  1007,  42,    44,    42,    1703,  492,   245,   655,   
+    333,   76,    42,    1447,  106,   140,   74,    76,    85,    34,    
+    149,   807,   333,   108,   1159,  172,   42,    268,   333,   149,   
+    76,    42,    1543,  106,   300,   74,    135,   149,   333,   1383,  
+    44,    42,    44,    74,    204,   42,    44,    333,   28135, 3182,  
+    149,   34279, 18215, 2215,  39,    1482,  140,   422,   71,    7898,  
+    1274,  1946,  74,    108,   122,   202,   258,   268,   90,    236,   
+    986,   140,   1562,  2138,  108,   58,    2810,  591,   841,   837,   
+    841,   229,   581,   841,   837,   41,    73,    41,    73,    137,   
+    265,   133,   37,    229,   357,   841,   837,   73,    137,   265,   
+    233,   837,   73,    137,   169,   41,    233,   837,   841,   837,   
+    841,   837,   841,   837,   841,   837,   841,   837,   841,   901,   
+    809,   57,    805,   57,    197,   809,   57,    805,   57,    197,   
+    809,   57,    805,   57,    197,   809,   57,    805,   57,    197,   
+    809,   57,    805,   57,    197,   94,    1613,  135,   871,   71,    
+    39,    39,    327,   135,   39,    39,    39,    39,    39,    39,    
+    103,   71,    39,    39,    39,    39,    39,    39,    71,    39,    
+    135,   231,   135,   135,   39,    327,   551,   103,   167,   551,   
+    89,    1434,  3226,  506,   474,   506,   506,   367,   1018,  1946,  
+    1402,  954,   1402,  314,   90,    1082,  218,   2266,  666,   1210,  
+    186,   570,   2042,  58,    5850,  154,   2010,  154,   794,   2266,  
+    378,   2266,  3738,  39,    39,    39,    39,    39,    39,    17351, 
+    34,    3074,  7692,  63,    63,    
+  };
+
+int sqlite3Fts5UnicodeCategory(int iCode) { 
+  int iRes = -1;
+  int iHi;
+  int iLo;
+  int ret;
+  u16 iKey;
+
+  if( iCode>=(1<<20) ){
+    return 0;
+  }
+  iLo = aFts5UnicodeBlock[(iCode>>16)];
+  iHi = aFts5UnicodeBlock[1+(iCode>>16)];
+  iKey = (iCode & 0xFFFF);
+  while( iHi>iLo ){
+    int iTest = (iHi + iLo) / 2;
+    assert( iTest>=iLo && iTest<iHi );
+    if( iKey>=aFts5UnicodeMap[iTest] ){
+      iRes = iTest;
+      iLo = iTest+1;
+    }else{
+      iHi = iTest;
+    }
+  }
+
+  if( iRes<0 ) return 0;
+  if( iKey>=(aFts5UnicodeMap[iRes]+(aFts5UnicodeData[iRes]>>5)) ) return 0;
+  ret = aFts5UnicodeData[iRes] & 0x1F;
+  if( ret!=30 ) return ret;
+  return ((iKey - aFts5UnicodeMap[iRes]) & 0x01) ? 5 : 9;
+}
+
+void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){
+  int i = 0;
+  int iTbl = 0;
+  while( i<128 ){
+    int bToken = aArray[ aFts5UnicodeData[iTbl] & 0x1F ];
+    int n = (aFts5UnicodeData[iTbl] >> 5) + i;
+    for(; i<128 && i<n; i++){
+      aAscii[i] = (u8)bToken;
+    }
+    iTbl++;
+  }
+}
+

ext/fts5/fts5_vocab.c

@@ -431,6 +431,8 @@ static int fts5VocabInstanceNext(Fts5VocabCursor *pCsr){
   i64 *pp = &pCsr->iInstPos;
   int *po = &pCsr->iInstOff;
   
+  assert( sqlite3Fts5IterEof(pIter)==0 );
+  assert( pCsr->bEof==0 );
   while( eDetail==FTS5_DETAIL_NONE
       || sqlite3Fts5PoslistNext64(pIter->pData, pIter->nData, po, pp) 
   ){
@@ -440,7 +442,7 @@ static int fts5VocabInstanceNext(Fts5VocabCursor *pCsr){
     rc = sqlite3Fts5IterNextScan(pCsr->pIter);
     if( rc==SQLITE_OK ){
       rc = fts5VocabInstanceNewTerm(pCsr);
-      if( eDetail==FTS5_DETAIL_NONE ) break;
+      if( pCsr->bEof || eDetail==FTS5_DETAIL_NONE ) break;
     }
     if( rc ){
       pCsr->bEof = 1;
@@ -755,10 +757,9 @@ int sqlite3Fts5VocabInit(Fts5Global *pGlobal, sqlite3 *db){
     /* xSavepoint    */ 0,
     /* xRelease      */ 0,
     /* xRollbackTo   */ 0,
+    /* xShadowName   */ 0
   };
   void *p = (void*)pGlobal;
 
   return sqlite3_create_module_v2(db, "fts5vocab", &fts5Vocab, p, 0);
 }
-
-

ext/fts5/fts5parse.y

@@ -148,7 +148,11 @@ cnearset(A) ::= colset(X) COLON nearset(Y). {
 %destructor nearset { sqlite3Fts5ParseNearsetFree($$); }
 %destructor nearphrases { sqlite3Fts5ParseNearsetFree($$); }
 
-nearset(A) ::= phrase(X). { A = sqlite3Fts5ParseNearset(pParse, 0, X); }
+nearset(A) ::= phrase(Y). { A = sqlite3Fts5ParseNearset(pParse, 0, Y); }
+nearset(A) ::= CARET phrase(Y). { 
+  sqlite3Fts5ParseSetCaret(Y);
+  A = sqlite3Fts5ParseNearset(pParse, 0, Y); 
+}
 nearset(A) ::= STRING(X) LP nearphrases(Y) neardist_opt(Z) RP. {
   sqlite3Fts5ParseNear(pParse, &X);
   sqlite3Fts5ParseSetDistance(pParse, Y, &Z);
@@ -189,6 +193,5 @@ phrase(A) ::= STRING(Y) star_opt(Z). {
 ** Optional "*" character.
 */
 %type star_opt {int}
-
 star_opt(A) ::= STAR. { A = 1; }
 star_opt(A) ::= . { A = 0; }

ext/fts5/test/fts5aa.test

@@ -409,6 +409,7 @@ do_test 14.3 {
 do_execsql_test 15.0 {
   INSERT INTO t1(t1) VALUES('integrity-check');
 }
+sqlite3_db_config db DEFENSIVE 0
 do_execsql_test 15.1 {
   UPDATE t1_content SET c1 = 'xyz xyz xyz xyz xyz abc' WHERE rowid = 1;
 }
@@ -591,7 +592,19 @@ do_execsql_test 22.1 {
   SELECT rowid FROM t9('a*')
 } {1}
 
+#-------------------------------------------------------------------------
+do_execsql_test 23.0 {
+  CREATE VIRTUAL TABLE t10 USING fts5(x, detail=%DETAIL%);
+  CREATE TABLE t11(x);
+}
+do_execsql_test 23.1 {
+  SELECT * FROM t11, t10 WHERE t11.x = t10.x AND t10.rowid IS NULL;
+}
+do_execsql_test 23.2 {
+  SELECT * FROM t11, t10 WHERE t10.rowid IS NULL;
 }
 
+}
 
+expand_all_sql db
 finish_test

ext/fts5/test/fts5af.test

@@ -175,6 +175,16 @@ do_execsql_test 5.1 {
   SELECT snippet(p1, 0, '[', ']', '...', 6) FROM p1('x');
 } {{[x] a a a a a...}}
 
+do_execsql_test 5.2 {
+  SELECT snippet(p1, 0, '[', ']', NULL, 6) FROM p1('x');
+} {{[x] a a a a a}}
+do_execsql_test 5.3 {
+  SELECT snippet(p1, 0, NULL, ']', '...', 6) FROM p1('x');
+} {{x] a a a a a...}}
+do_execsql_test 5.4 {
+  SELECT snippet(p1, 0, '[', NULL, '...', 6) FROM p1('x');
+} {{[x a a a a a...}}
+
 } ;# foreach_detail_mode 
 
 finish_test

ext/fts5/test/fts5cat.test

@@ -0,0 +1,59 @@
+# 2016 Jan 15
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+ifcapable !fts5 { finish_test ; return }
+set ::testprefix fts5cat
+
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize="unicode61 categories 'L*'");
+  INSERT INTO t1 VALUES ('Unlike1option2values3and4column5names');
+}
+
+do_execsql_test 1.1 {
+  SELECT rowid FROM t1('option');
+} {1}
+
+do_execsql_test 1.2 {
+  CREATE VIRTUAL TABLE t2 USING fts5(x);
+  CREATE VIRTUAL TABLE t2t USING fts5vocab(t2, row);
+
+  CREATE VIRTUAL TABLE t3 USING fts5(
+    x, tokenize="unicode61 categories 'L* N* Co Mn'"
+  );
+  CREATE VIRTUAL TABLE t3t USING fts5vocab(t3, row);
+
+  CREATE VIRTUAL TABLE t4 USING fts5(
+    x, tokenize="unicode61 categories 'L* N* Co M*'"
+  );
+  CREATE VIRTUAL TABLE t4t USING fts5vocab(t4, row);
+
+  INSERT INTO t2 VALUES ('สนามกีฬา');
+  INSERT INTO t3 VALUES ('สนามกีฬา');
+  INSERT INTO t4 VALUES ('สนามกีฬา');
+}
+
+do_execsql_test 1.3 {
+  SELECT * FROM t2t
+} {สนามก 1 1 ฬา 1 1}
+
+do_execsql_test 1.4 {
+  SELECT * FROM t3t
+} {สนามกีฬา 1 1}
+
+do_execsql_test 1.5 {
+  SELECT * FROM t4t
+} {สนามกีฬา 1 1}
+
+
+finish_test

ext/fts5/test/fts5connect.test

@@ -244,4 +244,3 @@ foreach {tn sql res} {
 }
 
 finish_test
-

ext/fts5/test/fts5corrupt.test

@@ -41,6 +41,7 @@ db_save
 do_execsql_test 1.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
 set segid [lindex [fts5_level_segids t1] 0]
 
+sqlite3_db_config db DEFENSIVE 0
 do_test 1.3 {
   execsql {
     DELETE FROM t1_data WHERE rowid = fts5_rowid('segment', $segid, 4);
@@ -50,6 +51,7 @@ do_test 1.3 {
 
 do_test 1.4 {
   db_restore_and_reopen
+  sqlite3_db_config db DEFENSIVE 0
   execsql {
     UPDATE t1_data set block = X'00000000' || substr(block, 5) WHERE
     rowid = fts5_rowid('segment', $segid, 4);
@@ -89,7 +91,7 @@ do_execsql_test 3.0 {
 do_execsql_test 3.1 {
   SELECT * FROM t3 WHERE t3 MATCH 'o'
 } {{one o} {three o} {five o}}
-
+sqlite3_db_config db DEFENSIVE 0
 do_catchsql_test 3.1 {
   DELETE FROM t3_content WHERE rowid = 3;
   SELECT * FROM t3 WHERE t3 MATCH 'o';

ext/fts5/test/fts5corrupt2.test

@@ -99,6 +99,7 @@ foreach {tno stmt} {
 set lrowid [db one {SELECT max(rowid) FROM t1_data WHERE (rowid & $mask)=0}] 
 set nbyte [db one {SELECT length(block) FROM t1_data WHERE rowid=$lrowid}]
 set all [db eval {SELECT rowid FROM t1}]
+sqlite3_db_config db DEFENSIVE 0
 for {set i [expr $nbyte-2]} {$i>=0} {incr i -1} {
   do_execsql_test 2.$i.1 {
     BEGIN;
@@ -248,6 +249,7 @@ foreach {tn hdr} {
 
 #--------------------------------------------------------------------
 reset_db
+sqlite3_db_config db DEFENSIVE 0
 do_execsql_test 6.1 {
   CREATE VIRTUAL TABLE x5 USING fts5(tt);
   INSERT INTO x5 VALUES('a');

ext/fts5/test/fts5corrupt3.test

@@ -51,6 +51,7 @@ do_test 1.1 {
   set {} {}
 } {} 
 
+sqlite3_db_config db DEFENSIVE 0
 for {set i 0} {$i < $L} {incr i} {
   do_test 1.2.$i {
     catchsql {
@@ -86,6 +87,7 @@ do_execsql_test 2.2 {
 #
 reset_db
 do_test 3.0 { create_t1 } {}
+sqlite3_db_config db DEFENSIVE 0
 
 do_execsql_test 3.1 {
   SELECT count(*) FROM t1_data;
@@ -158,6 +160,7 @@ do_3_test 3.10
 # Test that segments that end unexpectedly are identified as corruption.
 #
 reset_db
+sqlite3_db_config db DEFENSIVE 0
 do_test 4.0 {
   execsql { 
     CREATE VIRTUAL TABLE t1 USING fts5(x);
@@ -182,6 +185,7 @@ for {set i 1} {1} {incr i} {
 
   db close
   sqlite3 db test.db
+  sqlite3_db_config db DEFENSIVE 0
 
   db eval {
     BEGIN;
@@ -257,6 +261,7 @@ foreach rowid [db eval {SELECT rowid FROM x1_data WHERE rowid>100}] {
 #------------------------------------------------------------------------
 #
 reset_db
+sqlite3_db_config db DEFENSIVE 0
 do_execsql_test 6.1.0 {
   CREATE VIRTUAL TABLE t1 USING fts5(a);
   INSERT INTO t1 VALUES('bbbbb ccccc');
@@ -273,6 +278,7 @@ do_catchsql_test 6.1.2 {
 
 #-------
 reset_db
+sqlite3_db_config db DEFENSIVE 0
 do_execsql_test 6.2.0 {
   CREATE VIRTUAL TABLE t1 USING fts5(a);
   INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
@@ -288,6 +294,7 @@ do_catchsql_test 6.2.2 {
 
 #-------
 reset_db
+sqlite3_db_config db DEFENSIVE 0
 do_execsql_test 6.3.0 {
   CREATE VIRTUAL TABLE t1 USING fts5(a);
   INSERT INTO t1 VALUES('abc abcdef abcdefghi');
@@ -362,6 +369,7 @@ do_test 7.0 {
   }
 } {}
 
+sqlite3_db_config db DEFENSIVE 0
 do_test 7.1 {
   foreach i [db eval { SELECT rowid FROM t5_data WHERE rowid>100 }] {
     db eval BEGIN  
@@ -383,6 +391,7 @@ do_execsql_test 8.1 {
   INSERT INTO t1 VALUES('one', 'two');
 }
 
+sqlite3_db_config db DEFENSIVE 0
 do_test 9.1.1 {
   set    blob "12345678"    ;# cookie
   append blob "0105"        ;# 1 level, total of 5 segments

ext/fts5/test/fts5fault6.test

@@ -253,7 +253,7 @@ do_faultsim_test 5.2 -faults oom* -prep {
     SELECT rowid, mit(matchinfo(t1, 'x')) FROM t1 WHERE t1 MATCH 'a AND c'
   }
 } -test {
-  faultsim_test_result [list 0 $::res]
+  faultsim_test_result [list 0 $::res] {1 {SQL logic error}}
 }
 
 do_faultsim_test 5.3 -faults oom* -prep {
@@ -264,7 +264,7 @@ do_faultsim_test 5.3 -faults oom* -prep {
     SELECT count(*) FROM t1 WHERE t1 MATCH 'd AND e AND f'
   }
 } -test {
-  faultsim_test_result {0 29}
+  faultsim_test_result {0 29} {1 {SQL logic error}}
 }
 
 do_faultsim_test 5.4 -faults oom* -prep {
@@ -275,7 +275,7 @@ do_faultsim_test 5.4 -faults oom* -prep {
     SELECT count(*) FROM t1 WHERE t1 MATCH 'x + e'
   }
 } -test {
-  faultsim_test_result {0 1}
+  faultsim_test_result {0 1} {1 {SQL logic error}}
 }
 
 #-------------------------------------------------------------------------

ext/fts5/test/fts5fault9.test

@@ -24,6 +24,8 @@ ifcapable !fts5 {
 
 foreach_detail_mode $testprefix {
 
+if {"%DETAIL%" != "none"} continue
+
 fts5_aux_test_functions db
 
 do_execsql_test 1.0 {
@@ -98,14 +100,16 @@ do_faultsim_test 4.1 -faults oom-t* -body {
   execsql { SELECT rowid, fts5_test_collist(t4) FROM t4('2') }
 } -test {
   faultsim_test_result \
-      {0 {1 {0.0 0.1 0.2} 2 {0.0 0.1 0.2} 3 {0.0 0.1 0.2}}} {1 SQLITE_NOMEM}
+      {0 {1 {0.0 0.1 0.2} 2 {0.0 0.1 0.2} 3 {0.0 0.1 0.2}}} \
+      {1 SQLITE_NOMEM} {1 SQLITE_ERROR} {1 {SQL logic error}}
 }
 
 do_faultsim_test 4.2 -faults oom-t* -body {
   execsql { SELECT rowid, fts5_test_collist(t4) FROM t4('a5 OR b5 OR c5') }
 } -test {
   faultsim_test_result \
-      {0 {4 {0.0 0.1 0.2} 5 {1.0 1.1 1.2} 6 {2.0 2.1 2.2}}} {1 SQLITE_NOMEM}
+      {0 {4 {0.0 0.1 0.2} 5 {1.0 1.1 1.2} 6 {2.0 2.1 2.2}}} \
+      {1 SQLITE_NOMEM} {1 SQLITE_ERROR} {1 {SQL logic error}}
 }
 
 

ext/fts5/test/fts5faultB.test

@@ -130,5 +130,22 @@ do_faultsim_test 4.2 -faults oom* -body {
   faultsim_test_result {0 {2 3}}
 }
 
+#-------------------------------------------------------------------------
+# Test OOM injection while parsing a CARET expression
+#
+reset_db
+do_execsql_test 5.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a);
+  INSERT INTO t1 VALUES('a b c d');  -- 1
+  INSERT INTO t1 VALUES('d a b c');  -- 2
+  INSERT INTO t1 VALUES('c d a b');  -- 3
+  INSERT INTO t1 VALUES('b c d a');  -- 4
+}
+do_faultsim_test 5.1 -faults oom* -body {
+  execsql { SELECT rowid FROM t1('^a OR ^b') }
+} -test {
+  faultsim_test_result {0 {1 4}}
+}
+
 
 finish_test

ext/fts5/test/fts5first.test

@@ -0,0 +1,95 @@
+# 2017 November 25
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5first
+
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(a, b);
+}
+
+foreach {tn expr ok} {
+  1 {^abc}           1
+  2 {^abc + def}     1
+  3 {^ "abc def"}    1
+  4 {^"abc def"}     1
+  5 {abc ^def}       1
+  6 {abc + ^def}     0
+  7 {abc ^+ def}     0
+  8 {"^abc"}         1
+  9 {NEAR(^abc def)} 0
+} {
+  set res(0) {/1 {fts5: syntax error near .*}/}
+  set res(1) {0 {}}
+
+  do_catchsql_test 1.$tn { SELECT * FROM x1($expr) } $res($ok)
+}
+
+#-------------------------------------------------------------------------
+# 
+do_execsql_test 2.0 {
+  INSERT INTO x1 VALUES('a b c', 'b c a');
+}
+
+foreach {tn expr match} {
+  1 {^a} 1
+  2 {^b} 1
+  3 {^c} 0
+  4 {^a + b} 1
+  5 {^b + c} 1
+  6 {^c + a} 0
+  7 {^"c a"} 0
+  8 {a:^a} 1
+  9 {a:^b} 0
+  10 {a:^"a b"} 1
+} {
+  do_execsql_test 2.$tn { SELECT EXISTS (SELECT rowid FROM x1($expr)) } $match
+}
+
+#-------------------------------------------------------------------------
+# 
+do_execsql_test 3.0 {
+  DELETE FROM x1;
+  INSERT INTO x1 VALUES('b a', 'c a');
+  INSERT INTO x1 VALUES('a a', 'c c');
+  INSERT INTO x1 VALUES('a b', 'a a');
+}
+fts5_aux_test_functions db
+
+foreach {tn expr expect} {
+  1 {^a} {{2 1}}
+  2 {^c AND ^b} {{0 2} {1 0}}
+} {
+  do_execsql_test 3.$tn {
+    SELECT fts5_test_queryphrase(x1) FROM x1($expr) LIMIT 1
+  } [list $expect]
+}
+
+#-------------------------------------------------------------------------
+# 
+do_execsql_test 3.1 {
+  CREATE VIRTUAL TABLE x2 USING fts5(a, b, c, detail=column);
+}
+
+do_catchsql_test 3.2 {
+  SELECT * FROM x2('a + b');
+} {1 {fts5: phrase queries are not supported (detail!=full)}}
+
+do_catchsql_test 3.3 {
+  SELECT * FROM x2('^a');
+} {1 {fts5: phrase queries are not supported (detail!=full)}}
+finish_test

ext/fts5/test/fts5integrity.test

@@ -71,6 +71,7 @@ do_execsql_test 4.1 {
   INSERT INTO aa(aa) VALUES('integrity-check'); 
 }
 
+sqlite3_db_config db DEFENSIVE 0
 do_catchsql_test 4.2 { 
   BEGIN;
     UPDATE aa_docsize SET sz = X'44' WHERE rowid = 3;

ext/fts5/test/fts5plan.test

@@ -29,38 +29,37 @@ do_execsql_test 1.0 {
 do_eqp_test 1.1 {
   SELECT * FROM t1, f1 WHERE f1 MATCH t1.x
 } {
-  0 0 0 {SCAN TABLE t1} 
+  QUERY PLAN
-  0 1 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 65537:}
+  |--SCAN TABLE t1
+  `--SCAN TABLE f1 VIRTUAL TABLE INDEX 65537:
 }
 
 do_eqp_test 1.2 {
   SELECT * FROM t1, f1 WHERE f1 > t1.x
 } {
-  0 0 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 0:}
+  QUERY PLAN
-  0 1 0 {SCAN TABLE t1} 
+  |--SCAN TABLE f1 VIRTUAL TABLE INDEX 0:
+  `--SCAN TABLE t1
 }
 
 do_eqp_test 1.3 {
   SELECT * FROM f1 WHERE f1 MATCH ? ORDER BY ff
 } {
-  0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 65537:}
+  QUERY PLAN
-  0 0 0 {USE TEMP B-TREE FOR ORDER BY}
+  |--SCAN TABLE f1 VIRTUAL TABLE INDEX 65537:
+  `--USE TEMP B-TREE FOR ORDER BY
 }
 
 do_eqp_test 1.4 {
   SELECT * FROM f1 ORDER BY rank
 } {
-  0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 0:}
+  QUERY PLAN
-  0 0 0 {USE TEMP B-TREE FOR ORDER BY}
+  |--SCAN TABLE f1 VIRTUAL TABLE INDEX 0:
+  `--USE TEMP B-TREE FOR ORDER BY
 }
 
 do_eqp_test 1.5 {
   SELECT * FROM f1 WHERE rank MATCH ?
-} {
+} {SCAN TABLE f1 VIRTUAL TABLE INDEX 2:}
-  0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 2:}
-}
-
-
-
 
 finish_test

ext/fts5/test/fts5query.test

@@ -64,7 +64,7 @@ for {set tn 1 ; set pgsz 64} {$tn<32} {incr tn; incr pgsz 16} {
     execsql COMMIT
   } {}
 
-  do_execsql_test 1.$tn.2 {
+  do_execsql_test 2.$tn.2 {
     INSERT INTO t1(t1) VALUES('integrity-check');
   }
 
@@ -77,5 +77,15 @@ for {set tn 1 ; set pgsz 64} {$tn<32} {incr tn; incr pgsz 16} {
   }
 }
 
+reset_db
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(a);
+  INSERT INTO x1(rowid, a) VALUES(-1000000000000, 'toyota');
+  INSERT INTO x1(rowid, a) VALUES(1, 'tarago');
+}
+do_execsql_test 3.1 {
+  SELECT rowid FROM x1('t*');
+} {-1000000000000 1}
+
 
 finish_test

ext/fts5/test/fts5rank.test

@@ -148,7 +148,18 @@ do_execsql_test 4.1 {
   VTest MATCH 'wrinkle in time OR a wrinkle in time' ORDER BY rank;
 } {{wrinkle in time} {Bill Smith}}
 
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 5.0 {
+  CREATE VIRTUAL TABLE ttt USING fts5(a);
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100
+  )
+  INSERT INTO ttt SELECT 'word ' || i FROM s;
+}
 
-
+do_execsql_test 5.1 {
+  SELECT rowid FROM ttt('word') WHERE rowid BETWEEN 30 AND 40 ORDER BY rank;
+} {30 31 32 33 34 35 36 37 38 39 40}
 
 finish_test

ext/fts5/test/fts5rebuild.test

@@ -39,6 +39,7 @@ do_execsql_test 1.4 {
   INSERT INTO f1(f1) VALUES('integrity-check');
 } {}
 
+sqlite3_db_config db DEFENSIVE 0
 do_execsql_test 1.5 {
   DELETE FROM f1_data;
 } {}

ext/fts5/test/fts5rowid.test

@@ -70,6 +70,7 @@ set res [db one {SELECT count(*) FROM x1_data}]
 do_execsql_test 2.3 {
   SELECT count(fts5_decode(rowid, block)) FROM x1_data;
 } $res
+sqlite3_db_config db DEFENSIVE 0
 do_execsql_test 2.4 {
   UPDATE x1_data SET block = X'';
   SELECT count(fts5_decode(rowid, block)) FROM x1_data;

ext/fts5/test/fts5unicode.test

@@ -41,7 +41,6 @@ foreach {tn t} {1 ascii 2 unicode61} {
 #-------------------------------------------------------------------------
 # Check that "unicode61" really is the default tokenizer.
 #
-
 do_execsql_test 2.0 "
   CREATE VIRTUAL TABLE t1 USING fts5(x);
   CREATE VIRTUAL TABLE t2 USING fts5(x, tokenize = unicode61);
@@ -56,5 +55,31 @@ do_execsql_test 2.1 "
   SELECT 't3' FROM t3 WHERE t3 MATCH '\xE0\xE8\xEC';
 " {t1 t2}
 
+#-------------------------------------------------------------------------
+# Check that codepoints that require 4 bytes to store in utf-8 (those that
+# require 17 or more bits to store).
+#
+
+set A [db one {SELECT char(0x1F75E)}]    ;# Type So
+set B [db one {SELECT char(0x1F5FD)}]    ;# Type So
+set C [db one {SELECT char(0x2F802)}]    ;# Type Lo
+set D [db one {SELECT char(0x2F808)}]    ;# Type Lo
+
+do_execsql_test 3.0 "
+  CREATE VIRTUAL TABLE xyz USING fts5(x,
+    tokenize = \"unicode61 separators '$C' tokenchars '$A'\"
+  );
+  CREATE VIRTUAL TABLE xyz_v USING fts5vocab(xyz, row);
+
+  INSERT INTO xyz VALUES('$A$B$C$D');
+"
+
+do_execsql_test 3.1 {
+  SELECT * FROM xyz_v;
+} [list $A 1 1 $D 1 1]
+  
+
+
+
 
 finish_test

ext/fts5/test/fts5unicode4.test

@@ -0,0 +1,31 @@
+# 2018 July 25
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5unicode4
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE sss USING fts5(a, prefix=3); 
+}
+
+do_execsql_test 1.1 {
+  INSERT INTO sss VALUES('まりや');
+}
+
+finish_test

ext/fts5/test/fts5version.test

@@ -36,9 +36,10 @@ do_execsql_test 1.3 {
   SELECT rowid FROM t1 WHERE t1 MATCH 'a';
 } {1}
 
+sqlite3_db_config db DEFENSIVE 0
 do_execsql_test 1.4 {
   UPDATE t1_config set v=5 WHERE k='version';
-} 
+}
 
 do_test 1.5 {
   db close
@@ -53,6 +54,7 @@ do_test 1.6 {
 } {1 {invalid fts5 file format (found 5, expected 4) - run 'rebuild'}}
 
 do_test 1.7 {
+  sqlite3_db_config db DEFENSIVE 0
   execsql { DELETE FROM t1_config WHERE k='version' }
   db close
   sqlite3 db test.db

ext/fts5/test/fts5vocab.test

@@ -420,6 +420,7 @@ if {[detail_is_none]} { set resc [row_to_col $resr] }
 do_execsql_test 8.1.1 { SELECT * FROM x1_r; } $resr
 do_execsql_test 8.1.2 { SELECT * FROM x1_c } $resc
 
+sqlite3_db_config db DEFENSIVE 0
 do_execsql_test 8.2 {
   PRAGMA writable_schema = 1;
   UPDATE sqlite_master 
@@ -481,4 +482,3 @@ do_test 9.6 {
 
 
 finish_test
-

ext/fts5/test/fts5vocab2.test

@@ -13,7 +13,7 @@
 #
 
 source [file join [file dirname [info script]] fts5_common.tcl]
-set testprefix fts5vocab
+set testprefix fts5vocab2
 
 # If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
@@ -206,4 +206,3 @@ do_execsql_test 3.5 {
 }
 
 finish_test
-

ext/icu/README.txt

@@ -39,8 +39,8 @@ SQLite. Documentation follows.
     To utilise "general" case mapping, the upper() or lower() scalar 
     functions are invoked with one argument:
 
-        upper('ABC') -> 'abc'
+        upper('abc') -> 'ABC'
-        lower('abc') -> 'ABC'
+        lower('ABC') -> 'abc'
 
     To access ICU "language specific" case mapping, upper() or lower()
     should be invoked with two arguments. The second argument is the name

ext/icu/icu.c

@@ -28,7 +28,9 @@
 **     provide case-independent matching.
 */
 
-#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU)
+#if !defined(SQLITE_CORE)                  \
+ || defined(SQLITE_ENABLE_ICU)             \
+ || defined(SQLITE_ENABLE_ICU_COLLATIONS)
 
 /* Include ICU headers */
 #include <unicode/utypes.h>
@@ -45,6 +47,26 @@
   #include "sqlite3.h"
 #endif
 
+/*
+** This function is called when an ICU function called from within
+** the implementation of an SQL scalar function returns an error.
+**
+** The scalar function context passed as the first argument is 
+** loaded with an error message based on the following two args.
+*/
+static void icuFunctionError(
+  sqlite3_context *pCtx,       /* SQLite scalar function context */
+  const char *zName,           /* Name of ICU function that failed */
+  UErrorCode e                 /* Error code returned by ICU function */
+){
+  char zBuf[128];
+  sqlite3_snprintf(128, zBuf, "ICU error: %s(): %s", zName, u_errorName(e));
+  zBuf[127] = '\0';
+  sqlite3_result_error(pCtx, zBuf, -1);
+}
+
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU)
+
 /*
 ** Maximum length (in bytes) of the pattern in a LIKE or GLOB
 ** operator.
@@ -224,24 +246,6 @@ static void icuLikeFunc(
   }
 }
 
-/*
-** This function is called when an ICU function called from within
-** the implementation of an SQL scalar function returns an error.
-**
-** The scalar function context passed as the first argument is 
-** loaded with an error message based on the following two args.
-*/
-static void icuFunctionError(
-  sqlite3_context *pCtx,       /* SQLite scalar function context */
-  const char *zName,           /* Name of ICU function that failed */
-  UErrorCode e                 /* Error code returned by ICU function */
-){
-  char zBuf[128];
-  sqlite3_snprintf(128, zBuf, "ICU error: %s(): %s", zName, u_errorName(e));
-  zBuf[127] = '\0';
-  sqlite3_result_error(pCtx, zBuf, -1);
-}
-
 /*
 ** Function to delete compiled regexp objects. Registered as
 ** a destructor function with sqlite3_set_auxdata().
@@ -407,6 +411,8 @@ static void icuCaseFunc16(sqlite3_context *p, int nArg, sqlite3_value **apArg){
   assert( 0 );     /* Unreachable */
 }
 
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) */
+
 /*
 ** Collation sequence destructor function. The pCtx argument points to
 ** a UCollator structure previously allocated using ucol_open().
@@ -501,6 +507,7 @@ int sqlite3IcuInit(sqlite3 *db){
     void (*xFunc)(sqlite3_context*,int,sqlite3_value**);
   } scalars[] = {
     {"icu_load_collation",  2, SQLITE_UTF8,                1, icuLoadCollation},
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU)
     {"regexp", 2, SQLITE_ANY|SQLITE_DETERMINISTIC,         0, icuRegexpFunc},
     {"lower",  1, SQLITE_UTF16|SQLITE_DETERMINISTIC,       0, icuCaseFunc16},
     {"lower",  2, SQLITE_UTF16|SQLITE_DETERMINISTIC,       0, icuCaseFunc16},
@@ -512,10 +519,10 @@ int sqlite3IcuInit(sqlite3 *db){
     {"upper",  2, SQLITE_UTF8|SQLITE_DETERMINISTIC,        1, icuCaseFunc16},
     {"like",   2, SQLITE_UTF8|SQLITE_DETERMINISTIC,        0, icuLikeFunc},
     {"like",   3, SQLITE_UTF8|SQLITE_DETERMINISTIC,        0, icuLikeFunc},
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) */
   };
   int rc = SQLITE_OK;
   int i;
-
   
   for(i=0; rc==SQLITE_OK && i<(int)(sizeof(scalars)/sizeof(scalars[0])); i++){
     const struct IcuScalar *p = &scalars[i];

ext/lsm1/Makefile

@@ -43,7 +43,7 @@ LSMTESTSRC = $(LSMDIR)/lsm-test/lsmtest1.c $(LSMDIR)/lsm-test/lsmtest2.c     \
 
 # all: lsm.so
 
-LSMOPTS += -DLSM_MUTEX_PTHREADS=1 -I$(LSMDIR)
+LSMOPTS += -DLSM_MUTEX_PTHREADS=1 -I$(LSMDIR) -DHAVE_ZLIB
 
 lsm.so:	$(LSMOBJ)
 	$(TCCX) -shared -o lsm.so $(LSMOBJ)
@@ -53,4 +53,4 @@ lsm.so:	$(LSMOBJ)
 	
 lsmtest$(EXE): $(LSMOBJ) $(LSMTESTSRC) $(LSMTESTHDR) sqlite3.o
 	# $(TCPPX) -c $(TOP)/lsm-test/lsmtest_tdb2.cc
-	$(TCCX) $(LSMOPTS) $(LSMTESTSRC) $(LSMOBJ) sqlite3.o -o lsmtest$(EXE) $(THREADLIB)
+	$(TCCX) $(LSMOPTS) $(LSMTESTSRC) $(LSMOBJ) sqlite3.o -o lsmtest$(EXE) $(THREADLIB) -lz

ext/lsm1/lsm-test/lsmtest.h

@@ -121,6 +121,7 @@ int test_mdb_scan(TestDb *, void *, int, void *, int, void *, int,
 */
 int test_lsm_open(const char*, const char *zFile, int bClear, TestDb **ppDb);
 int test_lsm_lomem_open(const char*, const char*, int bClear, TestDb **ppDb);
+int test_lsm_lomem2_open(const char*, const char*, int bClear, TestDb **ppDb);
 int test_lsm_zip_open(const char*, const char*, int bClear, TestDb **ppDb);
 int test_lsm_small_open(const char*, const char*, int bClear, TestDb **ppDb);
 int test_lsm_mt2(const char*, const char *zFile, int bClear, TestDb **ppDb);

ext/lsm1/lsm-test/lsmtest1.c

@@ -274,6 +274,7 @@ static void doDataTest1(
   int rc = LSM_OK;
   Datasource *pData;
   TestDb *pDb;
+  int iToggle = 0;
 
   /* Start the test case, open a database and allocate the datasource. */
   pDb = testOpen(zSystem, 1, &rc);
@@ -287,8 +288,11 @@ static void doDataTest1(
     testWriteDatasourceRange(pDb, pData, i, p->nVerify, &rc);
     i += p->nVerify;
 
+    if( iToggle ) testBegin(pDb, 1, &rc);
     /* Check that the db content is correct. */
     testDbContents(pDb, pData, p->nRow, 0, i-1, p->nTest, p->bTestScan, &rc);
+    if( iToggle ) testCommit(pDb, 0, &rc);
+    iToggle = (iToggle+1)%2;
 
     if( bRecover ){
       testReopenRecover(&pDb, &rc);

ext/lsm1/lsm-test/lsmtest_tdb.c

@@ -721,6 +721,7 @@ static struct Lib {
   { "sqlite3",      "testdb.sqlite",    sql_open },
   { "lsm_small",    "testdb.lsm_small", test_lsm_small_open },
   { "lsm_lomem",    "testdb.lsm_lomem", test_lsm_lomem_open },
+  { "lsm_lomem2",   "testdb.lsm_lomem2", test_lsm_lomem2_open },
 #ifdef HAVE_ZLIB
   { "lsm_zip",      "testdb.lsm_zip",   test_lsm_zip_open },
 #endif

ext/lsm1/lsm-test/lsmtest_tdb3.c

@@ -617,8 +617,12 @@ static int test_lsm_fetch(
 
   if( pKey==0 ) return LSM_OK;
 
-  rc = lsm_csr_open(pDb->db, &csr);
+  if( pDb->pCsr==0 ){
-  if( rc!=LSM_OK ) return rc;
+    rc = lsm_csr_open(pDb->db, &csr);
+    if( rc!=LSM_OK ) return rc;
+  }else{
+    csr = pDb->pCsr;
+  }
 
   rc = lsm_csr_seek(csr, pKey, nKey, LSM_SEEK_EQ);
   if( rc==LSM_OK ){
@@ -638,7 +642,9 @@ static int test_lsm_fetch(
       *pnVal = -1;
     }
   }
-  lsm_csr_close(csr);
+  if( pDb->pCsr==0 ){
+    lsm_csr_close(csr);
+  }
   return rc;
 }
 
@@ -652,10 +658,28 @@ static int test_lsm_scan(
 ){
   LsmDb *pDb = (LsmDb *)pTestDb;
   lsm_cursor *csr;
+  lsm_cursor *csr2 = 0;
   int rc;
 
-  rc = lsm_csr_open(pDb->db, &csr);
+  if( pDb->pCsr==0 ){
-  if( rc!=LSM_OK ) return rc;
+    rc = lsm_csr_open(pDb->db, &csr);
+    if( rc!=LSM_OK ) return rc;
+  }else{
+    rc = LSM_OK;
+    csr = pDb->pCsr;
+  }
+
+  /* To enhance testing, if both pLast and pFirst are defined, seek the
+  ** cursor to the "end" boundary here. Then the next block seeks it to
+  ** the "start" ready for the scan. The point is to test that cursors
+  ** can be reused.  */
+  if( pLast && pFirst ){
+    if( bReverse ){
+      rc = lsm_csr_seek(csr, pFirst, nFirst, LSM_SEEK_LE);
+    }else{
+      rc = lsm_csr_seek(csr, pLast, nLast, LSM_SEEK_GE);
+    }
+  }
 
   if( bReverse ){
     if( pLast ){
@@ -696,7 +720,9 @@ static int test_lsm_scan(
     }
   }
 
-  lsm_csr_close(csr);
+  if( pDb->pCsr==0 ){
+    lsm_csr_close(csr);
+  }
   return rc;
 }
 
@@ -762,6 +788,7 @@ static void xWorkHook(lsm_db *db, void *pArg){
 #define TEST_MT_MIN_CKPT -4
 #define TEST_MT_MAX_CKPT -5
 
+
 int test_lsm_config_str(
   LsmDb *pLsm,
   lsm_db *db, 
@@ -1033,6 +1060,19 @@ int test_lsm_lomem_open(
   return testLsmOpen(zCfg, zFilename, bClear, ppDb);
 }
 
+int test_lsm_lomem2_open(
+  const char *zSpec, 
+  const char *zFilename, 
+  int bClear, 
+  TestDb **ppDb
+){
+    /* "max_freelist=4 autocheckpoint=32" */
+  const char *zCfg = 
+    "page_size=512 block_size=64 autoflush=0 mmap=0 "
+  ;
+  return testLsmOpen(zCfg, zFilename, bClear, ppDb);
+}
+
 int test_lsm_zip_open(
   const char *zSpec, 
   const char *zFilename, 

ext/lsm1/lsmInt.h

@@ -110,7 +110,7 @@ typedef unsigned long long int u64;
 #endif
 
 /* A page number is a 64-bit integer. */
-typedef i64 Pgno;
+typedef i64 LsmPgno;
 
 #ifdef LSM_DEBUG
 int lsmErrorBkpt(int);
@@ -402,9 +402,9 @@ struct lsm_db {
 };
 
 struct Segment {
-  Pgno iFirst;                     /* First page of this run */
+  LsmPgno iFirst;                  /* First page of this run */
-  Pgno iLastPg;                    /* Last page of this run */
+  LsmPgno iLastPg;                 /* Last page of this run */
-  Pgno iRoot;                      /* Root page number (if any) */
+  LsmPgno iRoot;                   /* Root page number (if any) */
   int nSize;                       /* Size of this run in pages */
 
   Redirect *pRedirect;             /* Block redirects (or NULL) */
@@ -456,7 +456,7 @@ struct Level {
 **   output segment.
 */
 struct MergeInput {
-  Pgno iPg;                       /* Page on which next input is stored */
+  LsmPgno iPg;                    /* Page on which next input is stored */
   int iCell;                      /* Cell containing next input to merge */
 };
 struct Merge {
@@ -465,7 +465,7 @@ struct Merge {
   MergeInput splitkey;            /* Location in file of current splitkey */
   int nSkip;                      /* Number of separators entries to skip */
   int iOutputOff;                 /* Write offset on output page */
-  Pgno iCurrentPtr;               /* Current pointer value */
+  LsmPgno iCurrentPtr;            /* Current pointer value */
 };
 
 /* 
@@ -579,10 +579,10 @@ struct Snapshot {
   Redirect redirect;              /* Block redirection array */
 
   /* Used by worker snapshots only */
-  int nBlock;                     /* Number of blocks in database file */
+  int nBlock;                        /* Number of blocks in database file */
-  Pgno aiAppend[LSM_APPLIST_SZ];  /* Append point list */
+  LsmPgno aiAppend[LSM_APPLIST_SZ];  /* Append point list */
-  Freelist freelist;              /* Free block list */
+  Freelist freelist;                 /* Free block list */
-  u32 nWrite;                     /* Total number of pages written to disk */
+  u32 nWrite;                        /* Total number of pages written to disk */
 };
 #define LSM_INITIAL_SNAPSHOT_ID 11
 
@@ -710,7 +710,7 @@ void lsmFsSetPageSize(FileSystem *, int);
 int lsmFsFileid(lsm_db *pDb, void **ppId, int *pnId);
 
 /* Creating, populating, gobbling and deleting sorted runs. */
-void lsmFsGobble(lsm_db *, Segment *, Pgno *, int);
+void lsmFsGobble(lsm_db *, Segment *, LsmPgno *, int);
 int lsmFsSortedDelete(FileSystem *, Snapshot *, int, Segment *);
 int lsmFsSortedFinish(FileSystem *, Segment *);
 int lsmFsSortedAppend(FileSystem *, Snapshot *, Level *, int, Page **);
@@ -727,14 +727,14 @@ void lsmSortedSplitkey(lsm_db *, Level *, int *);
 
 /* Reading sorted run content. */
 int lsmFsDbPageLast(FileSystem *pFS, Segment *pSeg, Page **ppPg);
-int lsmFsDbPageGet(FileSystem *, Segment *, Pgno, Page **);
+int lsmFsDbPageGet(FileSystem *, Segment *, LsmPgno, Page **);
 int lsmFsDbPageNext(Segment *, Page *, int eDir, Page **);
 
 u8 *lsmFsPageData(Page *, int *);
 int lsmFsPageRelease(Page *);
 int lsmFsPagePersist(Page *);
 void lsmFsPageRef(Page *);
-Pgno lsmFsPageNumber(Page *);
+LsmPgno lsmFsPageNumber(Page *);
 
 int lsmFsNRead(FileSystem *);
 int lsmFsNWrite(FileSystem *);
@@ -748,7 +748,7 @@ int lsmFsDbPageIsLast(Segment *pSeg, Page *pPg);
 int lsmFsIntegrityCheck(lsm_db *);
 #endif
 
-Pgno lsmFsRedirectPage(FileSystem *, Redirect *, Pgno);
+LsmPgno lsmFsRedirectPage(FileSystem *, Redirect *, LsmPgno);
 
 int lsmFsPageWritable(Page *);
 
@@ -768,8 +768,8 @@ int lsmFsSyncDb(FileSystem *, int);
 void lsmFsFlushWaiting(FileSystem *, int *);
 
 /* Used by lsm_info(ARRAY_STRUCTURE) and lsm_config(MMAP) */
-int lsmInfoArrayStructure(lsm_db *pDb, int bBlock, Pgno iFirst, char **pzOut);
+int lsmInfoArrayStructure(lsm_db *pDb, int bBlock, LsmPgno iFirst, char **pz);
-int lsmInfoArrayPages(lsm_db *pDb, Pgno iFirst, char **pzOut);
+int lsmInfoArrayPages(lsm_db *pDb, LsmPgno iFirst, char **pzOut);
 int lsmConfigMmap(lsm_db *pDb, int *piParam);
 
 int lsmEnvOpen(lsm_env *, const char *, int, lsm_file **);
@@ -785,7 +785,7 @@ void lsmEnvSleep(lsm_env *, int);
 
 int lsmFsReadSyncedId(lsm_db *db, int, i64 *piVal);
 
-int lsmFsSegmentContainsPg(FileSystem *pFS, Segment *, Pgno, int *);
+int lsmFsSegmentContainsPg(FileSystem *pFS, Segment *, LsmPgno, int *);
 
 void lsmFsPurgeCache(FileSystem *);
 
@@ -796,7 +796,7 @@ void lsmFsPurgeCache(FileSystem *);
 /* 
 ** Functions from file "lsm_sorted.c".
 */
-int lsmInfoPageDump(lsm_db *, Pgno, int, char **);
+int lsmInfoPageDump(lsm_db *, LsmPgno, int, char **);
 void lsmSortedCleanup(lsm_db *);
 int lsmSortedAutoWork(lsm_db *, int nUnit);
 

ext/lsm1/lsm_ckpt.c

@@ -389,7 +389,7 @@ static void ckptExportAppendlist(
   int *pRc                        /* IN/OUT: Error code */
 ){
   int i;
-  Pgno *aiAppend = db->pWorker->aiAppend;
+  LsmPgno *aiAppend = db->pWorker->aiAppend;
 
   for(i=0; i<LSM_APPLIST_SZ; i++){
     ckptAppend64(p, piOut, aiAppend[i], pRc);

ext/lsm1/lsm_file.c

@@ -269,7 +269,7 @@ struct FileSystem {
 struct Page {
   u8 *aData;                      /* Buffer containing page data */
   int nData;                      /* Bytes of usable data at aData[] */
-  Pgno iPg;                       /* Page number */
+  LsmPgno iPg;                    /* Page number */
   int nRef;                       /* Number of outstanding references */
   int flags;                      /* Combination of PAGE_XXX flags */
   Page *pHashNext;                /* Next page in hash table slot */
@@ -332,7 +332,7 @@ static int IOERR_WRAPPER(int rc){
 #ifdef NDEBUG
 # define assert_lists_are_ok(x)
 #else
-static Page *fsPageFindInHash(FileSystem *pFS, Pgno iPg, int *piHash);
+static Page *fsPageFindInHash(FileSystem *pFS, LsmPgno iPg, int *piHash);
 
 static void assert_lists_are_ok(FileSystem *pFS){
 #if 0
@@ -532,7 +532,7 @@ int lsmFsCloseAndDeleteLog(FileSystem *pFS){
 ** Return true if page iReal of the database should be accessed using mmap.
 ** False otherwise.
 */
-static int fsMmapPage(FileSystem *pFS, Pgno iReal){
+static int fsMmapPage(FileSystem *pFS, LsmPgno iReal){
   return ((i64)iReal*pFS->nPagesize <= pFS->nMapLimit);
 }
 
@@ -540,7 +540,7 @@ static int fsMmapPage(FileSystem *pFS, Pgno iReal){
 ** Given that there are currently nHash slots in the hash table, return 
 ** the hash key for file iFile, page iPg.
 */
-static int fsHashKey(int nHash, Pgno iPg){
+static int fsHashKey(int nHash, LsmPgno iPg){
   return (iPg % nHash);
 }
 
@@ -880,13 +880,13 @@ void lsmFsSetBlockSize(FileSystem *pFS, int nBlocksize){
 ** page on each block is the byte offset immediately following the 4-byte
 ** "previous block" pointer at the start of each block.
 */
-static Pgno fsFirstPageOnBlock(FileSystem *pFS, int iBlock){
+static LsmPgno fsFirstPageOnBlock(FileSystem *pFS, int iBlock){
-  Pgno iPg;
+  LsmPgno iPg;
   if( pFS->pCompress ){
     if( iBlock==1 ){
       iPg = pFS->nMetasize * 2 + 4;
     }else{
-      iPg = pFS->nBlocksize * (Pgno)(iBlock-1) + 4;
+      iPg = pFS->nBlocksize * (LsmPgno)(iBlock-1) + 4;
     }
   }else{
     const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize);
@@ -907,9 +907,9 @@ static Pgno fsFirstPageOnBlock(FileSystem *pFS, int iBlock){
 ** page on each block is the byte offset of the byte immediately before 
 ** the 4-byte "next block" pointer at the end of each block.
 */
-static Pgno fsLastPageOnBlock(FileSystem *pFS, int iBlock){
+static LsmPgno fsLastPageOnBlock(FileSystem *pFS, int iBlock){
   if( pFS->pCompress ){
-    return pFS->nBlocksize * (Pgno)iBlock - 1 - 4;
+    return pFS->nBlocksize * (LsmPgno)iBlock - 1 - 4;
   }else{
     const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize);
     return iBlock * nPagePerBlock;
@@ -920,7 +920,7 @@ static Pgno fsLastPageOnBlock(FileSystem *pFS, int iBlock){
 ** Return the block number of the block that page iPg is located on. 
 ** Blocks are numbered starting from 1.
 */
-static int fsPageToBlock(FileSystem *pFS, Pgno iPg){
+static int fsPageToBlock(FileSystem *pFS, LsmPgno iPg){
   if( pFS->pCompress ){
     return (int)((iPg / pFS->nBlocksize) + 1);
   }else{
@@ -933,7 +933,7 @@ static int fsPageToBlock(FileSystem *pFS, Pgno iPg){
 **
 ** This function is only called in non-compressed database mode.
 */
-static int fsIsLast(FileSystem *pFS, Pgno iPg){
+static int fsIsLast(FileSystem *pFS, LsmPgno iPg){
   const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize);
   assert( !pFS->pCompress );
   return ( iPg && (iPg % nPagePerBlock)==0 );
@@ -944,7 +944,7 @@ static int fsIsLast(FileSystem *pFS, Pgno iPg){
 **
 ** This function is only called in non-compressed database mode.
 */
-static int fsIsFirst(FileSystem *pFS, Pgno iPg){
+static int fsIsFirst(FileSystem *pFS, LsmPgno iPg){
   const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize);
   assert( !pFS->pCompress );
   return ( (iPg % nPagePerBlock)==1
@@ -967,7 +967,7 @@ u8 *lsmFsPageData(Page *pPage, int *pnData){
 /*
 ** Return the page number of a page.
 */
-Pgno lsmFsPageNumber(Page *pPage){
+LsmPgno lsmFsPageNumber(Page *pPage){
   /* assert( (pPage->flags & PAGE_DIRTY)==0 ); */
   return pPage ? pPage->iPg : 0;
 }
@@ -1058,7 +1058,7 @@ void lsmFsPurgeCache(FileSystem *pFS){
 ** Either way, if argument piHash is not NULL set *piHash to the hash slot
 ** number that page iPg would be stored in before returning.
 */
-static Page *fsPageFindInHash(FileSystem *pFS, Pgno iPg, int *piHash){
+static Page *fsPageFindInHash(FileSystem *pFS, LsmPgno iPg, int *piHash){
   Page *p;                        /* Return value */
   int iHash = fsHashKey(pFS->nHash, iPg);
 
@@ -1189,8 +1189,8 @@ static int fsRedirectBlock(Redirect *p, int iBlk){
 ** object passed as the second argument, return the destination page to
 ** which it is redirected. Otherwise, return a copy of iPg.
 */
-Pgno lsmFsRedirectPage(FileSystem *pFS, Redirect *pRedir, Pgno iPg){
+LsmPgno lsmFsRedirectPage(FileSystem *pFS, Redirect *pRedir, LsmPgno iPg){
-  Pgno iReal = iPg;
+  LsmPgno iReal = iPg;
 
   if( pRedir ){
     const int nPagePerBlock = (
@@ -1203,7 +1203,7 @@ Pgno lsmFsRedirectPage(FileSystem *pFS, Redirect *pRedir, Pgno iPg){
       if( iFrom>iBlk ) break;
       if( iFrom==iBlk ){
         int iTo = pRedir->a[i].iTo;
-        iReal = iPg - (Pgno)(iFrom - iTo) * nPagePerBlock;
+        iReal = iPg - (LsmPgno)(iFrom - iTo) * nPagePerBlock;
         if( iTo==1 ){
           iReal += (fsFirstPageOnBlock(pFS, 1)-1);
         }
@@ -1217,7 +1217,7 @@ Pgno lsmFsRedirectPage(FileSystem *pFS, Redirect *pRedir, Pgno iPg){
 }
 
 /* Required by the circular fsBlockNext<->fsPageGet dependency. */
-static int fsPageGet(FileSystem *, Segment *, Pgno, int, Page **, int *);
+static int fsPageGet(FileSystem *, Segment *, LsmPgno, int, Page **, int *);
 
 /*
 ** Parameter iBlock is a database file block. This function reads the value 
@@ -1269,7 +1269,7 @@ static int fsBlockNext(
 /*
 ** Return the page number of the last page on the same block as page iPg.
 */
-Pgno fsLastPageOnPagesBlock(FileSystem *pFS, Pgno iPg){
+LsmPgno fsLastPageOnPagesBlock(FileSystem *pFS, LsmPgno iPg){
   return fsLastPageOnBlock(pFS, fsPageToBlock(pFS, iPg));
 }
 
@@ -1537,7 +1537,7 @@ static int fsReadPagedata(
 static int fsPageGet(
   FileSystem *pFS,                /* File-system handle */
   Segment *pSeg,                  /* Block redirection to use (or NULL) */
-  Pgno iPg,                       /* Page id */
+  LsmPgno iPg,                    /* Page id */
   int noContent,                  /* True to not load content from disk */
   Page **ppPg,                    /* OUT: New page handle */
   int *pnSpace                    /* OUT: Bytes of free space */
@@ -1549,7 +1549,7 @@ static int fsPageGet(
   /* In most cases iReal is the same as iPg. Except, if pSeg->pRedirect is 
   ** not NULL, and the block containing iPg has been redirected, then iReal
   ** is the page number after redirection.  */
-  Pgno iReal = lsmFsRedirectPage(pFS, (pSeg ? pSeg->pRedirect : 0), iPg);
+  LsmPgno iReal = lsmFsRedirectPage(pFS, (pSeg ? pSeg->pRedirect : 0), iPg);
 
   assert_lists_are_ok(pFS);
   assert( iPg>=fsFirstPageOnBlock(pFS, 1) );
@@ -1689,8 +1689,8 @@ int lsmFsReadSyncedId(lsm_db *db, int iMeta, i64 *piVal){
 static int fsRunEndsBetween(
   Segment *pRun, 
   Segment *pIgnore, 
-  Pgno iFirst, 
+  LsmPgno iFirst, 
-  Pgno iLast
+  LsmPgno iLast
 ){
   return (pRun!=pIgnore && (
         (pRun->iFirst>=iFirst && pRun->iFirst<=iLast)
@@ -1705,8 +1705,8 @@ static int fsRunEndsBetween(
 static int fsLevelEndsBetween(
   Level *pLevel, 
   Segment *pIgnore, 
-  Pgno iFirst, 
+  LsmPgno iFirst, 
-  Pgno iLast
+  LsmPgno iLast
 ){
   int i;
 
@@ -1733,13 +1733,13 @@ static int fsFreeBlock(
   int iBlk                        /* Block number of block to free */
 ){
   int rc = LSM_OK;                /* Return code */
-  Pgno iFirst;                    /* First page on block iBlk */
+  LsmPgno iFirst;                 /* First page on block iBlk */
-  Pgno iLast;                     /* Last page on block iBlk */
+  LsmPgno iLast;                  /* Last page on block iBlk */
   Level *pLevel;                  /* Used to iterate through levels */
 
   int iIn;                        /* Used to iterate through append points */
   int iOut = 0;                   /* Used to output append points */
-  Pgno *aApp = pSnapshot->aiAppend;
+  LsmPgno *aApp = pSnapshot->aiAppend;
 
   iFirst = fsFirstPageOnBlock(pFS, iBlk);
   iLast = fsLastPageOnBlock(pFS, iBlk);
@@ -1811,11 +1811,16 @@ int lsmFsSortedDelete(
 ** number from the array that falls on block iBlk. Or, if none of the pages
 ** in aPgno[] fall on block iBlk, return 0.
 */
-static Pgno firstOnBlock(FileSystem *pFS, int iBlk, Pgno *aPgno, int nPgno){
+static LsmPgno firstOnBlock(
-  Pgno iRet = 0;
+  FileSystem *pFS, 
+  int iBlk, 
+  LsmPgno *aPgno, 
+  int nPgno
+){
+  LsmPgno iRet = 0;
   int i;
   for(i=0; i<nPgno; i++){
-    Pgno iPg = aPgno[i];
+    LsmPgno iPg = aPgno[i];
     if( fsPageToBlock(pFS, iPg)==iBlk && (iRet==0 || iPg<iRet) ){
       iRet = iPg;
     }
@@ -1828,7 +1833,7 @@ static Pgno firstOnBlock(FileSystem *pFS, int iBlk, Pgno *aPgno, int nPgno){
 ** Return true if page iPg, which is a part of segment p, lies on
 ** a redirected block. 
 */
-static int fsPageRedirects(FileSystem *pFS, Segment *p, Pgno iPg){
+static int fsPageRedirects(FileSystem *pFS, Segment *p, LsmPgno iPg){
   return (iPg!=0 && iPg!=lsmFsRedirectPage(pFS, p->pRedirect, iPg));
 }
 
@@ -1854,7 +1859,7 @@ static int fsSegmentRedirects(FileSystem *pFS, Segment *p){
 void lsmFsGobble(
   lsm_db *pDb,
   Segment *pRun, 
-  Pgno *aPgno,
+  LsmPgno *aPgno,
   int nPgno
 ){
   int rc = LSM_OK;
@@ -1871,7 +1876,7 @@ void lsmFsGobble(
 
   while( rc==LSM_OK ){
     int iNext = 0;
-    Pgno iFirst = firstOnBlock(pFS, iBlk, aPgno, nPgno);
+    LsmPgno iFirst = firstOnBlock(pFS, iBlk, aPgno, nPgno);
     if( iFirst ){
       pRun->iFirst = iFirst;
       break;
@@ -1905,11 +1910,11 @@ void lsmFsGobble(
 static int fsNextPageOffset(
   FileSystem *pFS,                /* File system object */
   Segment *pSeg,                  /* Segment to move within */
-  Pgno iPg,                       /* Offset of current page */
+  LsmPgno iPg,                    /* Offset of current page */
   int nByte,                      /* Size of current page including headers */
-  Pgno *piNext                    /* OUT: Offset of next page. Or zero (EOF) */
+  LsmPgno *piNext                 /* OUT: Offset of next page. Or zero (EOF) */
 ){
-  Pgno iNext;
+  LsmPgno iNext;
   int rc;
 
   assert( pFS->pCompress );
@@ -1939,8 +1944,8 @@ static int fsNextPageOffset(
 static int fsGetPageBefore(
   FileSystem *pFS, 
   Segment *pSeg, 
-  Pgno iPg, 
+  LsmPgno iPg, 
-  Pgno *piPrev
+  LsmPgno *piPrev
 ){
   u8 aSz[3];
   int rc;
@@ -1990,7 +1995,7 @@ static int fsGetPageBefore(
 int lsmFsDbPageNext(Segment *pRun, Page *pPg, int eDir, Page **ppNext){
   int rc = LSM_OK;
   FileSystem *pFS = pPg->pFS;
-  Pgno iPg = pPg->iPg;
+  LsmPgno iPg = pPg->iPg;
 
   assert( 0==fsSegmentRedirects(pFS, pRun) );
   if( pFS->pCompress ){
@@ -2062,10 +2067,10 @@ int lsmFsDbPageNext(Segment *pRun, Page *pPg, int eDir, Page **ppNext){
 ** start the new segment immediately following any segment that is part
 ** of the right-hand-side of pLvl.
 */
-static Pgno findAppendPoint(FileSystem *pFS, Level *pLvl){
+static LsmPgno findAppendPoint(FileSystem *pFS, Level *pLvl){
   int i;
-  Pgno *aiAppend = pFS->pDb->pWorker->aiAppend;
+  LsmPgno *aiAppend = pFS->pDb->pWorker->aiAppend;
-  Pgno iRet = 0;
+  LsmPgno iRet = 0;
 
   for(i=LSM_APPLIST_SZ-1; iRet==0 && i>=0; i--){
     if( (iRet = aiAppend[i]) ){
@@ -2098,10 +2103,10 @@ int lsmFsSortedAppend(
 ){
   int rc = LSM_OK;
   Page *pPg = 0;
-  Pgno iApp = 0;
+  LsmPgno iApp = 0;
-  Pgno iNext = 0;
+  LsmPgno iNext = 0;
   Segment *p = &pLvl->lhs;
-  Pgno iPrev = p->iLastPg;
+  LsmPgno iPrev = p->iLastPg;
 
   *ppOut = 0;
   assert( p->pRedirect==0 );
@@ -2195,7 +2200,7 @@ int lsmFsSortedFinish(FileSystem *pFS, Segment *p){
     */
     if( fsLastPageOnPagesBlock(pFS, p->iLastPg)!=p->iLastPg ){
       int i;
-      Pgno *aiAppend = pFS->pDb->pWorker->aiAppend;
+      LsmPgno *aiAppend = pFS->pDb->pWorker->aiAppend;
       for(i=0; i<LSM_APPLIST_SZ; i++){
         if( aiAppend[i]==0 ){
           aiAppend[i] = p->iLastPg+1;
@@ -2226,7 +2231,7 @@ int lsmFsSortedFinish(FileSystem *pFS, Segment *p){
 **
 ** Return LSM_OK if successful, or an lsm error code if an error occurs.
 */
-int lsmFsDbPageGet(FileSystem *pFS, Segment *pSeg, Pgno iPg, Page **ppPg){
+int lsmFsDbPageGet(FileSystem *pFS, Segment *pSeg, LsmPgno iPg, Page **ppPg){
   return fsPageGet(pFS, pSeg, iPg, 0, ppPg, 0);
 }
 
@@ -2238,7 +2243,7 @@ int lsmFsDbPageGet(FileSystem *pFS, Segment *pSeg, Pgno iPg, Page **ppPg){
 */
 int lsmFsDbPageLast(FileSystem *pFS, Segment *pSeg, Page **ppPg){
   int rc;
-  Pgno iPg = pSeg->iLastPg;
+  LsmPgno iPg = pSeg->iLastPg;
   if( pFS->pCompress ){
     int nSpace;
     iPg++;
@@ -2366,14 +2371,14 @@ static void fsMovePage(
   FileSystem *pFS,                /* File system object */
   int iTo,                        /* Destination block */
   int iFrom,                      /* Source block */
-  Pgno *piPg                      /* IN/OUT: Page number */
+  LsmPgno *piPg                   /* IN/OUT: Page number */
 ){
-  Pgno iPg = *piPg;
+  LsmPgno iPg = *piPg;
   if( iFrom==fsPageToBlock(pFS, iPg) ){
     const int nPagePerBlock = (
         pFS->pCompress ? pFS ->nBlocksize : (pFS->nBlocksize / pFS->nPagesize)
     );
-    *piPg = iPg - (Pgno)(iFrom - iTo) * nPagePerBlock;
+    *piPg = iPg - (LsmPgno)(iFrom - iTo) * nPagePerBlock;
   }
 }
 
@@ -2457,21 +2462,21 @@ int lsmFsMoveBlock(FileSystem *pFS, Segment *pSeg, int iTo, int iFrom){
 **
 ** This function is only used in compressed database mode.
 */
-static Pgno fsAppendData(
+static LsmPgno fsAppendData(
   FileSystem *pFS,                /* File-system handle */
   Segment *pSeg,                  /* Segment to append to */
   const u8 *aData,                /* Buffer containing data to write */
   int nData,                      /* Size of buffer aData[] in bytes */
   int *pRc                        /* IN/OUT: Error code */
 ){
-  Pgno iRet = 0;
+  LsmPgno iRet = 0;
   int rc = *pRc;
   assert( pFS->pCompress );
   if( rc==LSM_OK ){
     int nRem = 0;
     int nWrite = 0;
-    Pgno iLastOnBlock;
+    LsmPgno iLastOnBlock;
-    Pgno iApp = pSeg->iLastPg+1;
+    LsmPgno iApp = pSeg->iLastPg+1;
 
     /* If this is the first data written into the segment, find an append-point
     ** or allocate a new block.  */
@@ -2519,7 +2524,7 @@ static Pgno fsAppendData(
 
         /* Set the "prev" pointer on the new block */
         if( rc==LSM_OK ){
-          Pgno iWrite;
+          LsmPgno iWrite;
           lsmPutU32(aPtr, fsPageToBlock(pFS, iApp));
           iWrite = fsFirstPageOnBlock(pFS, iBlk);
           rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iWrite-4, aPtr, sizeof(aPtr));
@@ -2588,11 +2593,11 @@ static int fsCompressIntoBuffer(FileSystem *pFS, Page *pPg){
 static int fsAppendPage(
   FileSystem *pFS, 
   Segment *pSeg,
-  Pgno *piNew,
+  LsmPgno *piNew,
   int *piPrev,
   int *piNext
 ){
-  Pgno iPrev = pSeg->iLastPg;
+  LsmPgno iPrev = pSeg->iLastPg;
   int rc;
   assert( iPrev!=0 );
 
@@ -2650,7 +2655,7 @@ void lsmFsFlushWaiting(FileSystem *pFS, int *pRc){
 /*
 ** If there exists a hash-table entry associated with page iPg, remove it.
 */
-static void fsRemoveHashEntry(FileSystem *pFS, Pgno iPg){
+static void fsRemoveHashEntry(FileSystem *pFS, LsmPgno iPg){
   Page *p;
   int iHash = fsHashKey(pFS->nHash, iPg);
 
@@ -2803,9 +2808,9 @@ int lsmFsSortedPadding(
   Segment *pSeg
 ){
   int rc = LSM_OK;
-  if( pFS->pCompress ){
+  if( pFS->pCompress && pSeg->iFirst ){
-    Pgno iLast2;
+    LsmPgno iLast2;
-    Pgno iLast = pSeg->iLastPg;     /* Current last page of segment */
+    LsmPgno iLast = pSeg->iLastPg;  /* Current last page of segment */
     int nPad;                       /* Bytes of padding required */
     u8 aSz[3];
 
@@ -2935,7 +2940,7 @@ int lsmFsSectorSize(FileSystem *pFS){
 /*
 ** Helper function for lsmInfoArrayStructure().
 */
-static Segment *startsWith(Segment *pRun, Pgno iFirst){
+static Segment *startsWith(Segment *pRun, LsmPgno iFirst){
   return (iFirst==pRun->iFirst) ? pRun : 0;
 }
 
@@ -2943,7 +2948,7 @@ static Segment *startsWith(Segment *pRun, Pgno iFirst){
 ** Return the segment that starts with page iFirst, if any. If no such segment
 ** can be found, return NULL.
 */
-static Segment *findSegment(Snapshot *pWorker, Pgno iFirst){
+static Segment *findSegment(Snapshot *pWorker, LsmPgno iFirst){
   Level *pLvl;                    /* Used to iterate through db levels */
   Segment *pSeg = 0;              /* Pointer to segment to return */
 
@@ -2970,7 +2975,7 @@ static Segment *findSegment(Snapshot *pWorker, Pgno iFirst){
 int lsmInfoArrayStructure(
   lsm_db *pDb, 
   int bBlock,                     /* True for block numbers only */
-  Pgno iFirst,
+  LsmPgno iFirst,
   char **pzOut
 ){
   int rc = LSM_OK;
@@ -3035,7 +3040,7 @@ int lsmInfoArrayStructure(
 int lsmFsSegmentContainsPg(
   FileSystem *pFS, 
   Segment *pSeg, 
-  Pgno iPg, 
+  LsmPgno iPg, 
   int *pbRes
 ){
   Redirect *pRedir = pSeg->pRedirect;
@@ -3064,7 +3069,7 @@ int lsmFsSegmentContainsPg(
 **
 ** If an error occurs, *pzOut is set to NULL and an LSM error code returned.
 */
-int lsmInfoArrayPages(lsm_db *pDb, Pgno iFirst, char **pzOut){
+int lsmInfoArrayPages(lsm_db *pDb, LsmPgno iFirst, char **pzOut){
   int rc = LSM_OK;
   Snapshot *pWorker;              /* Worker snapshot */
   Segment *pSeg = 0;              /* Array to report on */
@@ -3297,7 +3302,7 @@ int lsmFsIntegrityCheck(lsm_db *pDb){
 */
 int lsmFsDbPageIsLast(Segment *pSeg, Page *pPg){
   if( pPg->pFS->pCompress ){
-    Pgno iNext = 0;
+    LsmPgno iNext = 0;
     int rc;
     rc = fsNextPageOffset(pPg->pFS, pSeg, pPg->iPg, pPg->nCompress+6, &iNext);
     return (rc!=LSM_OK || iNext==0);

ext/lsm1/lsm_main.c

@@ -583,14 +583,14 @@ int lsm_info(lsm_db *pDb, int eParam, ...){
     }
 
     case LSM_INFO_ARRAY_STRUCTURE: {
-      Pgno pgno = va_arg(ap, Pgno);
+      LsmPgno pgno = va_arg(ap, LsmPgno);
       char **pzVal = va_arg(ap, char **);
       rc = lsmInfoArrayStructure(pDb, 0, pgno, pzVal);
       break;
     }
 
     case LSM_INFO_ARRAY_PAGES: {
-      Pgno pgno = va_arg(ap, Pgno);
+      LsmPgno pgno = va_arg(ap, LsmPgno);
       char **pzVal = va_arg(ap, char **);
       rc = lsmInfoArrayPages(pDb, pgno, pzVal);
       break;
@@ -598,7 +598,7 @@ int lsm_info(lsm_db *pDb, int eParam, ...){
 
     case LSM_INFO_PAGE_HEX_DUMP:
     case LSM_INFO_PAGE_ASCII_DUMP: {
-      Pgno pgno = va_arg(ap, Pgno);
+      LsmPgno pgno = va_arg(ap, LsmPgno);
       char **pzVal = va_arg(ap, char **);
       int bUnlock = 0;
       rc = infoGetWorker(pDb, 0, &bUnlock);
@@ -683,7 +683,7 @@ static int doWriteOp(
     int nDiff;
 
     if( nQuant>pDb->nTreeLimit ){
-      nQuant = pDb->nTreeLimit;
+      nQuant = LSM_MAX(pDb->nTreeLimit, pgsz);
     }
 
     nBefore = lsmTreeSize(pDb);

ext/lsm1/lsm_shared.c

@@ -340,9 +340,6 @@ static int doDbConnect(lsm_db *pDb){
   /* Obtain a pointer to the shared-memory header */
   assert( pDb->pShmhdr==0 );
   assert( pDb->bReadonly==0 );
-  rc = lsmShmCacheChunks(pDb, 1);
-  if( rc!=LSM_OK ) return rc;
-  pDb->pShmhdr = (ShmHeader *)pDb->apShm[0];
 
   /* Block for an exclusive lock on DMS1. This lock serializes all calls
   ** to doDbConnect() and doDbDisconnect() across all processes.  */
@@ -353,10 +350,11 @@ static int doDbConnect(lsm_db *pDb){
     nUs = nUs * 2;
     if( nUs>nUsMax ) nUs = nUsMax;
   }
-  if( rc!=LSM_OK ){
+  if( rc==LSM_OK ){
-    pDb->pShmhdr = 0;
+    rc = lsmShmCacheChunks(pDb, 1);
-    return rc;
   }
+  if( rc!=LSM_OK ) return rc;
+  pDb->pShmhdr = (ShmHeader *)pDb->apShm[0];
 
   /* Try an exclusive lock on DMS2/DMS3. If successful, this is the first 
   ** and only connection to the database. In this case initialize the 
@@ -522,13 +520,11 @@ int lsmDbDatabaseConnect(
   ** recovery as necessary. Or, if this is a read-only database handle,
   ** defer attempting to connect to the system until a read-transaction
   ** is opened.  */
-  if( pDb->bReadonly==0 ){
+  if( rc==LSM_OK ){
-    if( rc==LSM_OK ){
+    rc = lsmFsConfigure(pDb);
-      rc = lsmFsConfigure(pDb);
+  }
-    }
+  if( rc==LSM_OK && pDb->bReadonly==0 ){
-    if( rc==LSM_OK ){
+    rc = doDbConnect(pDb);
-      rc = doDbConnect(pDb);
-    }
   }
 
   return rc;

ext/lsm1/lsm_sorted.c

@@ -92,7 +92,7 @@
 #define SEGMENT_POINTER_OFFSET(pgsz)        ((pgsz) - 2 - 2 - 8)
 #define SEGMENT_CELLPTR_OFFSET(pgsz, iCell) ((pgsz) - 2 - 2 - 8 - 2 - (iCell)*2)
 
-#define SEGMENT_EOF(pgsz, nEntry) SEGMENT_CELLPTR_OFFSET(pgsz, nEntry)
+#define SEGMENT_EOF(pgsz, nEntry) SEGMENT_CELLPTR_OFFSET(pgsz, nEntry-1)
 
 #define SEGMENT_BTREE_FLAG     0x0001
 #define PGFTR_SKIP_NEXT_FLAG   0x0002
@@ -104,9 +104,9 @@
 #endif
 
 typedef struct SegmentPtr SegmentPtr;
-typedef struct Blob Blob;
+typedef struct LsmBlob LsmBlob;
 
-struct Blob {
+struct LsmBlob {
   lsm_env *pEnv;
   void *pData;
   int nData;
@@ -129,18 +129,18 @@ struct SegmentPtr {
   Page *pPg;                    /* Current page */
   u16 flags;                    /* Copy of page flags field */
   int nCell;                    /* Number of cells on pPg */
-  Pgno iPtr;                    /* Base cascade pointer */
+  LsmPgno iPtr;                 /* Base cascade pointer */
 
   /* Current cell. See segmentPtrLoadCell() */
   int iCell;                    /* Current record within page pPg */
   int eType;                    /* Type of current record */
-  Pgno iPgPtr;                  /* Cascade pointer offset */
+  LsmPgno iPgPtr;               /* Cascade pointer offset */
   void *pKey; int nKey;         /* Key associated with current record */
   void *pVal; int nVal;         /* Current record value (eType==WRITE only) */
 
   /* Blobs used to allocate buffers for pKey and pVal as required */
-  Blob blob1;
+  LsmBlob blob1;
-  Blob blob2;
+  LsmBlob blob2;
 };
 
 /*
@@ -171,10 +171,10 @@ struct BtreeCursor {
   void *pKey;
   int nKey;
   int eType;
-  Pgno iPtr;
+  LsmPgno iPtr;
 
   /* Storage for key, if not local */
-  Blob blob;
+  LsmBlob blob;
 };
 
 
@@ -203,8 +203,8 @@ struct MultiCursor {
   int flags;                      /* Mask of CURSOR_XXX flags */
 
   int eType;                      /* Cache of current key type */
-  Blob key;                       /* Cache of current key (or NULL) */
+  LsmBlob key;                    /* Cache of current key (or NULL) */
-  Blob val;                       /* Cache of current value */
+  LsmBlob val;                    /* Cache of current value */
 
   /* All the component cursors: */
   TreeCursor *apTreeCsr[2];       /* Up to two tree cursors */
@@ -221,7 +221,7 @@ struct MultiCursor {
   void *pSystemVal;               /* Pointer to buffer to free */
 
   /* Used by worker cursors only */
-  Pgno *pPrevMergePtr;
+  LsmPgno *pPrevMergePtr;
 };
 
 /*
@@ -295,11 +295,11 @@ struct MergeWorker {
   Hierarchy hier;                 /* B-tree hierarchy under construction */
   Page *pPage;                    /* Current output page */
   int nWork;                      /* Number of calls to mergeWorkerNextPage() */
-  Pgno *aGobble;                  /* Gobble point for each input segment */
+  LsmPgno *aGobble;               /* Gobble point for each input segment */
 
-  Pgno iIndirect;
+  LsmPgno iIndirect;
   struct SavedPgno {
-    Pgno iPgno;
+    LsmPgno iPgno;
     int bStore;
   } aSave[2];
 };
@@ -371,7 +371,7 @@ void lsmPutU64(u8 *aOut, u64 nVal){
   aOut[7] = (u8)((nVal    ) & 0xFF);
 }
 
-static int sortedBlobGrow(lsm_env *pEnv, Blob *pBlob, int nData){
+static int sortedBlobGrow(lsm_env *pEnv, LsmBlob *pBlob, int nData){
   assert( pBlob->pEnv==pEnv || (pBlob->pEnv==0 && pBlob->pData==0) );
   if( pBlob->nAlloc<nData ){
     pBlob->pData = lsmReallocOrFree(pEnv, pBlob->pData, nData);
@@ -382,7 +382,7 @@ static int sortedBlobGrow(lsm_env *pEnv, Blob *pBlob, int nData){
   return LSM_OK;
 }
 
-static int sortedBlobSet(lsm_env *pEnv, Blob *pBlob, void *pData, int nData){
+static int sortedBlobSet(lsm_env *pEnv, LsmBlob *pBlob, void *pData, int nData){
   if( sortedBlobGrow(pEnv, pBlob, nData) ) return LSM_NOMEM;
   memcpy(pBlob->pData, pData, nData);
   pBlob->nData = nData;
@@ -390,15 +390,15 @@ static int sortedBlobSet(lsm_env *pEnv, Blob *pBlob, void *pData, int nData){
 }
 
 #if 0
-static int sortedBlobCopy(Blob *pDest, Blob *pSrc){
+static int sortedBlobCopy(LsmBlob *pDest, LsmBlob *pSrc){
   return sortedBlobSet(pDest, pSrc->pData, pSrc->nData);
 }
 #endif
 
-static void sortedBlobFree(Blob *pBlob){
+static void sortedBlobFree(LsmBlob *pBlob){
   assert( pBlob->pEnv || pBlob->pData==0 );
   if( pBlob->pData ) lsmFree(pBlob->pEnv, pBlob->pData);
-  memset(pBlob, 0, sizeof(Blob));
+  memset(pBlob, 0, sizeof(LsmBlob));
 }
 
 static int sortedReadData(
@@ -407,7 +407,7 @@ static int sortedReadData(
   int iOff,
   int nByte,
   void **ppData,
-  Blob *pBlob
+  LsmBlob *pBlob
 ){
   int rc = LSM_OK;
   int iEnd;
@@ -481,8 +481,8 @@ static int pageGetNRec(u8 *aData, int nData){
   return (int)lsmGetU16(&aData[SEGMENT_NRECORD_OFFSET(nData)]);
 }
 
-static Pgno pageGetPtr(u8 *aData, int nData){
+static LsmPgno pageGetPtr(u8 *aData, int nData){
-  return (Pgno)lsmGetU64(&aData[SEGMENT_POINTER_OFFSET(nData)]);
+  return (LsmPgno)lsmGetU64(&aData[SEGMENT_POINTER_OFFSET(nData)]);
 }
 
 static int pageGetFlags(u8 *aData, int nData){
@@ -506,8 +506,8 @@ static int pageObjGetNRec(Page *pPg){
 ** Return the decoded (possibly relative) pointer value stored in cell 
 ** iCell from page aData/nData.
 */
-static Pgno pageGetRecordPtr(u8 *aData, int nData, int iCell){
+static LsmPgno pageGetRecordPtr(u8 *aData, int nData, int iCell){
-  Pgno iRet;                      /* Return value */
+  LsmPgno iRet;                   /* Return value */
   u8 *aCell;                      /* Pointer to cell iCell */
 
   assert( iCell<pageGetNRec(aData, nData) && iCell>=0 );
@@ -522,7 +522,7 @@ static u8 *pageGetKey(
   int iCell,                      /* Index of cell on page to read */
   int *piTopic,                   /* OUT: Topic associated with this key */
   int *pnKey,                     /* OUT: Size of key in bytes */
-  Blob *pBlob                     /* If required, use this for dynamic memory */
+  LsmBlob *pBlob                  /* If required, use this for dynamic memory */
 ){
   u8 *pKey;
   int nDummy;
@@ -554,7 +554,7 @@ static int pageGetKeyCopy(
   Page *pPg,                      /* Page to read from */
   int iCell,                      /* Index of cell on page to read */
   int *piTopic,                   /* OUT: Topic associated with this key */
-  Blob *pBlob                     /* If required, use this for dynamic memory */
+  LsmBlob *pBlob                  /* If required, use this for dynamic memory */
 ){
   int rc = LSM_OK;
   int nKey;
@@ -569,8 +569,8 @@ static int pageGetKeyCopy(
   return rc;
 }
 
-static Pgno pageGetBtreeRef(Page *pPg, int iKey){
+static LsmPgno pageGetBtreeRef(Page *pPg, int iKey){
-  Pgno iRef;
+  LsmPgno iRef;
   u8 *aData;
   int nData;
   u8 *aCell;
@@ -592,11 +592,11 @@ static int pageGetBtreeKey(
   Segment *pSeg,                  /* Segment page pPg belongs to */
   Page *pPg,
   int iKey, 
-  Pgno *piPtr, 
+  LsmPgno *piPtr, 
   int *piTopic, 
   void **ppKey,
   int *pnKey,
-  Blob *pBlob
+  LsmBlob *pBlob
 ){
   u8 *aData;
   int nData;
@@ -613,7 +613,7 @@ static int pageGetBtreeKey(
 
   if( eType==0 ){
     int rc;
-    Pgno iRef;                  /* Page number of referenced page */
+    LsmPgno iRef;               /* Page number of referenced page */
     Page *pRef;
     aCell += GETVARINT64(aCell, iRef);
     rc = lsmFsDbPageGet(lsmPageFS(pPg), pSeg, iRef, &pRef);
@@ -638,7 +638,7 @@ static int btreeCursorLoadKey(BtreeCursor *pCsr){
     pCsr->nKey = 0;
     pCsr->eType = 0;
   }else{
-    Pgno dummy;
+    LsmPgno dummy;
     int iPg = pCsr->iPg;
     int iCell = pCsr->aPg[iPg].iCell;
     while( iCell<0 && (--iPg)>=0 ){
@@ -683,7 +683,7 @@ static int btreeCursorNext(BtreeCursor *pCsr){
   assert( pPg->iCell<=nCell );
   pPg->iCell++;
   if( pPg->iCell==nCell ){
-    Pgno iLoad;
+    LsmPgno iLoad;
 
     /* Up to parent. */
     lsmFsPageRelease(pPg->pPage);
@@ -842,7 +842,7 @@ static int btreeCursorRestore(
   if( p->iPg ){
     lsm_env *pEnv = lsmFsEnv(pCsr->pFS);
     int iCell;                    /* Current cell number on leaf page */
-    Pgno iLeaf;                   /* Page number of current leaf page */
+    LsmPgno iLeaf;                /* Page number of current leaf page */
     int nDepth;                   /* Depth of b-tree structure */
     Segment *pSeg = pCsr->pSeg;
 
@@ -866,7 +866,7 @@ static int btreeCursorRestore(
 
     /* Populate any other aPg[] array entries */
     if( rc==LSM_OK && nDepth>1 ){
-      Blob blob = {0,0,0};
+      LsmBlob blob = {0,0,0};
       void *pSeek;
       int nSeek;
       int iTopicSeek;
@@ -883,7 +883,7 @@ static int btreeCursorRestore(
         pSeek = 0;
         nSeek = 0;
       }else{
-        Pgno dummy;
+        LsmPgno dummy;
         rc = pageGetBtreeKey(pSeg, pPg,
             0, &dummy, &iTopicSeek, &pSeek, &nSeek, &pCsr->blob
         );
@@ -912,7 +912,7 @@ static int btreeCursorRestore(
             int iTry = (iMin+iMax)/2;
             void *pKey; int nKey;         /* Key for cell iTry */
             int iTopic;                   /* Topic for key pKeyT/nKeyT */
-            Pgno iPtr;                    /* Pointer for cell iTry */
+            LsmPgno iPtr;                 /* Pointer for cell iTry */
             int res;                      /* (pSeek - pKeyT) */
 
             rc = pageGetBtreeKey(
@@ -955,7 +955,7 @@ static int btreeCursorRestore(
       aData = fsPageData(pBtreePg->pPage, &nData);
       pCsr->iPtr = btreeCursorPtr(aData, nData, pBtreePg->iCell+1);
       if( pBtreePg->iCell<0 ){
-        Pgno dummy;
+        LsmPgno dummy;
         int i;
         for(i=pCsr->iPg-1; i>=0; i--){
           if( pCsr->aPg[i].iCell>0 ) break;
@@ -1030,7 +1030,7 @@ static int segmentPtrReadData(
   int iOff,
   int nByte,
   void **ppData,
-  Blob *pBlob
+  LsmBlob *pBlob
 ){
   return sortedReadData(pPtr->pSeg, pPtr->pPg, iOff, nByte, ppData, pBlob);
 }
@@ -1123,7 +1123,7 @@ static void sortedSplitkey(lsm_db *pDb, Level *pLevel, int *pRc){
   }
   if( rc==LSM_OK ){
     int iTopic;
-    Blob blob = {0, 0, 0, 0};
+    LsmBlob blob = {0, 0, 0, 0};
     u8 *aData;
     int nData;
   
@@ -1131,7 +1131,7 @@ static void sortedSplitkey(lsm_db *pDb, Level *pLevel, int *pRc){
     if( pageGetFlags(aData, nData) & SEGMENT_BTREE_FLAG ){
       void *pKey;
       int nKey;
-      Pgno dummy;
+      LsmPgno dummy;
       rc = pageGetBtreeKey(pSeg,
           pPg, pMerge->splitkey.iCell, &dummy, &iTopic, &pKey, &nKey, &blob
       );
@@ -1342,7 +1342,7 @@ static int assertKeyLocation(
   void *pKey, int nKey
 ){
   lsm_env *pEnv = lsmFsEnv(pCsr->pDb->pFS);
-  Blob blob = {0, 0, 0};
+  LsmBlob blob = {0, 0, 0};
   int eDir;
   int iTopic = 0;                 /* TODO: Fix me */
 
@@ -1488,7 +1488,7 @@ static int ptrFwdPointer(
   Page *pPage,
   int iCell,
   Segment *pSeg,
-  Pgno *piPtr,
+  LsmPgno *piPtr,
   int *pbFound
 ){
   Page *pPg = pPage;
@@ -1573,14 +1573,14 @@ static int sortedRhsFirst(MultiCursor *pCsr, Level *pLvl, SegmentPtr *pPtr){
 static int segmentPtrFwdPointer(
   MultiCursor *pCsr,              /* Multi-cursor pPtr belongs to */
   SegmentPtr *pPtr,               /* Segment-pointer to extract FC ptr from */
-  Pgno *piPtr                     /* OUT: FC pointer value */
+  LsmPgno *piPtr                  /* OUT: FC pointer value */
 ){
   Level *pLvl = pPtr->pLevel;
   Level *pNext = pLvl->pNext;
   Page *pPg = pPtr->pPg;
   int rc;
   int bFound;
-  Pgno iOut = 0;
+  LsmPgno iOut = 0;
 
   if( pPtr->pSeg==&pLvl->lhs || pPtr->pSeg==&pLvl->aRhs[pLvl->nRight-1] ){
     if( pNext==0 
@@ -1641,7 +1641,7 @@ static int segmentPtrSeek(
   int rc = LSM_OK;
   int iMin;
   int iMax;
-  Pgno iPtrOut = 0;
+  LsmPgno iPtrOut = 0;
 
   /* If the current page contains an oversized entry, then there are no
   ** pointers to one or more of the subsequent pages in the sorted run.
@@ -1768,18 +1768,18 @@ static int seekInBtree(
   Segment *pSeg,                  /* Seek within this segment */
   int iTopic,
   void *pKey, int nKey,           /* Key to seek to */
-  Pgno *aPg,                      /* OUT: Page numbers */
+  LsmPgno *aPg,                   /* OUT: Page numbers */
   Page **ppPg                     /* OUT: Leaf (sorted-run) page reference */
 ){
   int i = 0;
   int rc;
   int iPg;
   Page *pPg = 0;
-  Blob blob = {0, 0, 0};
+  LsmBlob blob = {0, 0, 0};
 
   iPg = (int)pSeg->iRoot;
   do {
-    Pgno *piFirst = 0;
+    LsmPgno *piFirst = 0;
     if( aPg ){
       aPg[i++] = iPg;
       piFirst = &aPg[i];
@@ -1808,7 +1808,7 @@ static int seekInBtree(
         int iTry = (iMin+iMax)/2;
         void *pKeyT; int nKeyT;       /* Key for cell iTry */
         int iTopicT;                  /* Topic for key pKeyT/nKeyT */
-        Pgno iPtr;                    /* Pointer associated with cell iTry */
+        LsmPgno iPtr;                 /* Pointer associated with cell iTry */
         int res;                      /* (pKey - pKeyT) */
 
         rc = pageGetBtreeKey(
@@ -1899,7 +1899,7 @@ static int seekInLevel(
   int eSeek,                      /* Search bias - see above */
   int iTopic,                     /* Key topic to search for */
   void *pKey, int nKey,           /* Key to search for */
-  Pgno *piPgno,                   /* IN/OUT: fraction cascade pointer (or 0) */
+  LsmPgno *piPgno,                /* IN/OUT: fraction cascade pointer (or 0) */
   int *pbStop                     /* OUT: See above */
 ){
   Level *pLvl = aPtr[0].pLevel;   /* Level to seek within */
@@ -1922,6 +1922,7 @@ static int seekInLevel(
   ** is not a composite level and there is no split-key). Search the 
   ** left-hand-side of the level in this case.  */
   if( res<0 ){
+    int i;
     int iPtr = 0;
     if( nRhs==0 ) iPtr = (int)*piPgno;
 
@@ -1931,12 +1932,16 @@ static int seekInLevel(
     if( rc==LSM_OK && nRhs>0 && eSeek==LSM_SEEK_GE && aPtr[0].pPg==0 ){
       res = 0;
     }
+    for(i=1; i<=nRhs; i++){
+      segmentPtrReset(&aPtr[i], LSM_SEGMENTPTR_FREE_THRESHOLD);
+    }
   }
   
   if( res>=0 ){
     int bHit = 0;                 /* True if at least one rhs is not EOF */
     int iPtr = (int)*piPgno;
     int i;
+    segmentPtrReset(&aPtr[0], LSM_SEGMENTPTR_FREE_THRESHOLD);
     for(i=1; rc==LSM_OK && i<=nRhs && bStop==0; i++){
       SegmentPtr *pPtr = &aPtr[i];
       iOut = 0;
@@ -2868,7 +2873,7 @@ static int multiCursorEnd(MultiCursor *pCsr, int bLast){
   int rc = LSM_OK;
   int i;
 
-  pCsr->flags &= ~(CURSOR_NEXT_OK | CURSOR_PREV_OK);
+  pCsr->flags &= ~(CURSOR_NEXT_OK | CURSOR_PREV_OK | CURSOR_SEEK_EQ);
   pCsr->flags |= (bLast ? CURSOR_PREV_OK : CURSOR_NEXT_OK);
   pCsr->iFree = 0;
 
@@ -3055,7 +3060,7 @@ int lsmMCursorSeek(
   int bStop = 0;                  /* Set to true to halt search operation */
   int rc = LSM_OK;                /* Return code */
   int iPtr = 0;                   /* Used to iterate through pCsr->aPtr[] */
-  Pgno iPgno = 0;                 /* FC pointer value */
+  LsmPgno iPgno = 0;              /* FC pointer value */
 
   assert( pCsr->apTreeCsr[0]==0 || iTopic==0 );
   assert( pCsr->apTreeCsr[1]==0 || iTopic==0 );
@@ -3537,7 +3542,7 @@ static int mergeWorkerLoadHierarchy(MergeWorker *pMW){
 **         + Type byte (always SORTED_SEPARATOR or SORTED_SYSTEM_SEPARATOR),
 **         + Absolute pointer value (varint),
 **         + Number of bytes in key (varint),
-**         + Blob containing key data.
+**         + LsmBlob containing key data.
 **
 **   2. All pointer values are stored as absolute values (not offsets 
 **      relative to the footer pointer value).
@@ -3571,8 +3576,8 @@ static int mergeWorkerLoadHierarchy(MergeWorker *pMW){
 static int mergeWorkerBtreeWrite(
   MergeWorker *pMW,
   u8 eType,
-  Pgno iPtr,
+  LsmPgno iPtr,
-  Pgno iKeyPg,
+  LsmPgno iKeyPg,
   void *pKey,
   int nKey
 ){
@@ -3682,7 +3687,7 @@ static int mergeWorkerBtreeWrite(
 static int mergeWorkerBtreeIndirect(MergeWorker *pMW){
   int rc = LSM_OK;
   if( pMW->iIndirect ){
-    Pgno iKeyPg = pMW->aSave[1].iPgno;
+    LsmPgno iKeyPg = pMW->aSave[1].iPgno;
     rc = mergeWorkerBtreeWrite(pMW, 0, pMW->iIndirect, iKeyPg, 0, 0);
     pMW->iIndirect = 0;
   }
@@ -3703,7 +3708,7 @@ static int mergeWorkerPushHierarchy(
   int nKey                        /* Size of pKey buffer in bytes */
 ){
   int rc = LSM_OK;                /* Return Code */
-  Pgno iPtr;                      /* Pointer value to accompany pKey/nKey */
+  LsmPgno iPtr;                   /* Pointer value to accompany pKey/nKey */
 
   assert( pMW->aSave[0].bStore==0 );
   assert( pMW->aSave[1].bStore==0 );
@@ -3734,7 +3739,7 @@ static int mergeWorkerFinishHierarchy(
 ){
   int i;                          /* Used to loop through apHier[] */
   int rc = LSM_OK;                /* Return code */
-  Pgno iPtr;                      /* New right-hand-child pointer value */
+  LsmPgno iPtr;                   /* New right-hand-child pointer value */
 
   iPtr = pMW->aSave[0].iPgno;
   for(i=0; i<pMW->hier.nHier && rc==LSM_OK; i++){
@@ -3830,7 +3835,7 @@ static int mergeWorkerPersistAndRelease(MergeWorker *pMW){
 */
 static int mergeWorkerNextPage(
   MergeWorker *pMW,               /* Merge worker object to append page to */
-  Pgno iFPtr                      /* Pointer value for footer of new page */
+  LsmPgno iFPtr                   /* Pointer value for footer of new page */
 ){
   int rc = LSM_OK;                /* Return code */
   Page *pNext = 0;                /* New page appended to run */
@@ -3999,6 +4004,11 @@ static int mergeWorkerWrite(
     ** marked read-only, advance to the next page of the output run. */
     iOff = pMerge->iOutputOff;
     if( iOff<0 || pPg==0 || iOff+nHdr > SEGMENT_EOF(nData, nRec+1) ){
+      if( iOff>=0 && pPg ){
+        /* Zero any free space on the page */
+        assert( aData );
+        memset(&aData[iOff], 0, SEGMENT_EOF(nData, nRec)-iOff);
+      }
       iFPtr = (int)*pMW->pCsr->pPrevMergePtr;
       iRPtr = iPtr - iFPtr;
       iOff = 0;
@@ -4069,36 +4079,49 @@ static void mergeWorkerShutdown(MergeWorker *pMW, int *pRc){
   /* Unless the merge has finished, save the cursor position in the
   ** Merge.aInput[] array. See function mergeWorkerInit() for the 
   ** code to restore a cursor position based on aInput[].  */
-  if( rc==LSM_OK && pCsr && lsmMCursorValid(pCsr) ){
+  if( rc==LSM_OK && pCsr ){
     Merge *pMerge = pMW->pLevel->pMerge;
-    int bBtree = (pCsr->pBtCsr!=0);
+    if( lsmMCursorValid(pCsr) ){
-    int iPtr;
+      int bBtree = (pCsr->pBtCsr!=0);
+      int iPtr;
 
-    /* pMerge->nInput==0 indicates that this is a FlushTree() operation. */
+      /* pMerge->nInput==0 indicates that this is a FlushTree() operation. */
-    assert( pMerge->nInput==0 || pMW->pLevel->nRight>0 );
+      assert( pMerge->nInput==0 || pMW->pLevel->nRight>0 );
-    assert( pMerge->nInput==0 || pMerge->nInput==(pCsr->nPtr+bBtree) );
+      assert( pMerge->nInput==0 || pMerge->nInput==(pCsr->nPtr+bBtree) );
 
-    for(i=0; i<(pMerge->nInput-bBtree); i++){
+      for(i=0; i<(pMerge->nInput-bBtree); i++){
-      SegmentPtr *pPtr = &pCsr->aPtr[i];
+        SegmentPtr *pPtr = &pCsr->aPtr[i];
-      if( pPtr->pPg ){
+        if( pPtr->pPg ){
-        pMerge->aInput[i].iPg = lsmFsPageNumber(pPtr->pPg);
+          pMerge->aInput[i].iPg = lsmFsPageNumber(pPtr->pPg);
-        pMerge->aInput[i].iCell = pPtr->iCell;
+          pMerge->aInput[i].iCell = pPtr->iCell;
+        }else{
+          pMerge->aInput[i].iPg = 0;
+          pMerge->aInput[i].iCell = 0;
+        }
+      }
+      if( bBtree && pMerge->nInput ){
+        assert( i==pCsr->nPtr );
+        btreeCursorPosition(pCsr->pBtCsr, &pMerge->aInput[i]);
+      }
+
+      /* Store the location of the split-key */
+      iPtr = pCsr->aTree[1] - CURSOR_DATA_SEGMENT;
+      if( iPtr<pCsr->nPtr ){
+        pMerge->splitkey = pMerge->aInput[iPtr];
       }else{
-        pMerge->aInput[i].iPg = 0;
+        btreeCursorSplitkey(pCsr->pBtCsr, &pMerge->splitkey);
-        pMerge->aInput[i].iCell = 0;
       }
     }
-    if( bBtree && pMerge->nInput ){
-      assert( i==pCsr->nPtr );
-      btreeCursorPosition(pCsr->pBtCsr, &pMerge->aInput[i]);
-    }
 
-    /* Store the location of the split-key */
+    /* Zero any free space left on the final page. This helps with
-    iPtr = pCsr->aTree[1] - CURSOR_DATA_SEGMENT;
+    ** compression if using a compression hook. And prevents valgrind
-    if( iPtr<pCsr->nPtr ){
+    ** from complaining about uninitialized byte passed to write(). */
-      pMerge->splitkey = pMerge->aInput[iPtr];
+    if( pMW->pPage ){
-    }else{
+      int nData;
-      btreeCursorSplitkey(pCsr->pBtCsr, &pMerge->splitkey);
+      u8 *aData = fsPageData(pMW->pPage, &nData);
+      int iOff = pMerge->iOutputOff;
+      int iEof = SEGMENT_EOF(nData, pageGetNRec(aData, nData));
+      memset(&aData[iOff], 0, iEof - iOff);
     }
     
     pMerge->iOutputOff = -1;
@@ -4200,7 +4223,7 @@ static int mergeWorkerStep(MergeWorker *pMW){
   int rc = LSM_OK;              /* Return code */
   int eType;                    /* SORTED_SEPARATOR, WRITE or DELETE */
   void *pKey; int nKey;         /* Key */
-  Pgno iPtr;
+  LsmPgno iPtr;
   int iVal;
 
   pCsr = pMW->pCsr;
@@ -4353,7 +4376,7 @@ static int sortedNewToplevel(
   if( rc!=LSM_OK ){
     lsmMCursorClose(pCsr, 0);
   }else{
-    Pgno iLeftPtr = 0;
+    LsmPgno iLeftPtr = 0;
     Merge merge;                  /* Merge object used to create new level */
     MergeWorker mergeworker;      /* MergeWorker object for the same purpose */
 
@@ -4530,7 +4553,7 @@ static int mergeWorkerInit(
   memset(pMW, 0, sizeof(MergeWorker));
   pMW->pDb = pDb;
   pMW->pLevel = pLevel;
-  pMW->aGobble = lsmMallocZeroRc(pDb->pEnv, sizeof(Pgno) * pLevel->nRight, &rc);
+  pMW->aGobble = lsmMallocZeroRc(pDb->pEnv, sizeof(LsmPgno)*pLevel->nRight,&rc);
 
   /* Create a multi-cursor to read the data to write to the new
   ** segment. The new segment contains:
@@ -4612,7 +4635,7 @@ static int sortedBtreeGobble(
   int rc = LSM_OK;
   if( rtTopic(pCsr->eType)==0 ){
     Segment *pSeg = pCsr->aPtr[iGobble].pSeg;
-    Pgno *aPg;
+    LsmPgno *aPg;
     int nPg;
 
     /* Seek from the root of the b-tree to the segment leaf that may contain
@@ -4621,7 +4644,7 @@ static int sortedBtreeGobble(
     ** gobbled up to (but not including) the first of these page numbers.
     */
     assert( pSeg->iRoot>0 );
-    aPg = lsmMallocZeroRc(pDb->pEnv, sizeof(Pgno)*32, &rc);
+    aPg = lsmMallocZeroRc(pDb->pEnv, sizeof(LsmPgno)*32, &rc);
     if( rc==LSM_OK ){
       rc = seekInBtree(pCsr, pSeg, 
           rtTopic(pCsr->eType), pCsr->key.pData, pCsr->key.nData, aPg, 0
@@ -5234,16 +5257,15 @@ static int doLsmSingleWork(
   /* If the in-memory part of the free-list is too large, write a new 
   ** top-level containing just the in-memory free-list entries to disk. */
   if( rc==LSM_OK && pDb->pWorker->freelist.nEntry > pDb->nMaxFreelist ){
-    int nPg = 0;
     while( rc==LSM_OK && lsmDatabaseFull(pDb) ){
+      int nPg = 0;
       rc = sortedWork(pDb, 16, nMerge, 1, &nPg);
       nRem -= nPg;
     }
     if( rc==LSM_OK ){
       rc = sortedNewFreelistOnly(pDb);
     }
-    nRem -= nPg;
+    bDirty = 1;
-    if( nPg ) bDirty = 1;
   }
 
   if( rc==LSM_OK ){
@@ -5448,9 +5470,9 @@ int lsmFlushTreeToDisk(lsm_db *pDb){
 */
 static char *segToString(lsm_env *pEnv, Segment *pSeg, int nMin){
   int nSize = pSeg->nSize;
-  Pgno iRoot = pSeg->iRoot;
+  LsmPgno iRoot = pSeg->iRoot;
-  Pgno iFirst = pSeg->iFirst;
+  LsmPgno iFirst = pSeg->iFirst;
-  Pgno iLast = pSeg->iLastPg;
+  LsmPgno iLast = pSeg->iLastPg;
   char *z;
 
   char *z1;
@@ -5509,7 +5531,7 @@ static int fileToString(
 }
 
 void sortedDumpPage(lsm_db *pDb, Segment *pRun, Page *pPg, int bVals){
-  Blob blob = {0, 0, 0};         /* Blob used for keys */
+  LsmBlob blob = {0, 0, 0};       /* LsmBlob used for keys */
   LsmString s;
   int i;
 
@@ -5545,7 +5567,7 @@ void sortedDumpPage(lsm_db *pDb, Segment *pRun, Page *pPg, int bVals){
     aCell += lsmVarintGet32(aCell, &iPgPtr);
 
     if( eType==0 ){
-      Pgno iRef;                  /* Page number of referenced page */
+      LsmPgno iRef;               /* Page number of referenced page */
       aCell += lsmVarintGet64(aCell, &iRef);
       lsmFsDbPageGet(pDb->pFS, pRun, iRef, &pRef);
       aKey = pageGetKey(pRun, pRef, 0, &iTopic, &nKey, &blob);
@@ -5589,7 +5611,7 @@ static void infoCellDump(
   int *piPgPtr,
   u8 **paKey, int *pnKey,
   u8 **paVal, int *pnVal,
-  Blob *pBlob
+  LsmBlob *pBlob
 ){
   u8 *aData; int nData;           /* Page data */
   u8 *aKey; int nKey = 0;         /* Key */
@@ -5607,7 +5629,7 @@ static void infoCellDump(
 
   if( eType==0 ){
     int dummy;
-    Pgno iRef;                  /* Page number of referenced page */
+    LsmPgno iRef;                 /* Page number of referenced page */
     aCell += lsmVarintGet64(aCell, &iRef);
     if( bIndirect ){
       lsmFsDbPageGet(pDb->pFS, pSeg, iRef, &pRef);
@@ -5653,7 +5675,7 @@ static int infoAppendBlob(LsmString *pStr, int bHex, u8 *z, int n){
 
 static int infoPageDump(
   lsm_db *pDb,                    /* Database handle */
-  Pgno iPg,                       /* Page number of page to dump */
+  LsmPgno iPg,                    /* Page number of page to dump */
   int flags,
   char **pzOut                    /* OUT: lsmMalloc'd string */
 ){
@@ -5694,7 +5716,7 @@ static int infoPageDump(
   }
 
   if( rc==LSM_OK ){
-    Blob blob = {0, 0, 0, 0};
+    LsmBlob blob = {0, 0, 0, 0};
     int nKeyWidth = 0;
     LsmString str;
     int nRec;
@@ -5729,7 +5751,7 @@ static int infoPageDump(
       u8 *aVal; int nVal = 0;       /* Value */
       int iPgPtr;
       int eType;
-      Pgno iAbsPtr;
+      LsmPgno iAbsPtr;
       char zFlags[8];
 
       infoCellDump(pDb, pSeg, bIndirect, pPg, iCell, &eType, &iPgPtr,
@@ -5795,7 +5817,7 @@ static int infoPageDump(
 
 int lsmInfoPageDump(
   lsm_db *pDb,                    /* Database handle */
-  Pgno iPg,                       /* Page number of page to dump */
+  LsmPgno iPg,                    /* Page number of page to dump */
   int bHex,                       /* True to output key/value in hex form */
   char **pzOut                    /* OUT: lsmMalloc'd string */
 ){
@@ -5971,8 +5993,8 @@ void lsmSortedExpandBtreePage(Page *pPg, int nOrig){
 #ifdef LSM_DEBUG_EXPENSIVE
 static void assertRunInOrder(lsm_db *pDb, Segment *pSeg){
   Page *pPg = 0;
-  Blob blob1 = {0, 0, 0, 0};
+  LsmBlob blob1 = {0, 0, 0, 0};
-  Blob blob2 = {0, 0, 0, 0};
+  LsmBlob blob2 = {0, 0, 0, 0};
 
   lsmFsDbPageGet(pDb->pFS, pSeg, pSeg->iFirst, &pPg);
   while( pPg ){
@@ -6034,7 +6056,7 @@ static int assertPointersOk(
   int rc = LSM_OK;                /* Error code */
   SegmentPtr ptr1;                /* Iterates through pOne */
   SegmentPtr ptr2;                /* Iterates through pTwo */
-  Pgno iPrev;
+  LsmPgno iPrev;
 
   assert( pOne && pTwo );
 
@@ -6057,7 +6079,7 @@ static int assertPointersOk(
   }
       
   while( rc==LSM_OK && ptr2.pPg ){
-    Pgno iThis;
+    LsmPgno iThis;
 
     /* Advance to the next page of segment pTwo that contains at least
     ** one cell. Break out of the loop if the iterator reaches EOF.  */
@@ -6119,7 +6141,7 @@ static int assertBtreeOk(
 ){
   int rc = LSM_OK;                /* Return code */
   if( pSeg->iRoot ){
-    Blob blob = {0, 0, 0};        /* Buffer used to cache overflow keys */
+    LsmBlob blob = {0, 0, 0};     /* Buffer used to cache overflow keys */
     FileSystem *pFS = pDb->pFS;   /* File system to read from */
     Page *pPg = 0;                /* Main run page */
     BtreeCursor *pCsr = 0;        /* Btree cursor */

ext/lsm1/tool/mklsm1c.tcl

@@ -0,0 +1,88 @@
+#!/bin/sh
+# restart with tclsh \
+exec tclsh "$0" "$@"
+
+set srcdir [file dirname [file dirname [info script]]]
+set G(src) [string map [list %dir% $srcdir] {
+  %dir%/lsm.h
+  %dir%/lsmInt.h
+  %dir%/lsm_vtab.c
+  %dir%/lsm_ckpt.c
+  %dir%/lsm_file.c
+  %dir%/lsm_log.c
+  %dir%/lsm_main.c
+  %dir%/lsm_mem.c
+  %dir%/lsm_mutex.c
+  %dir%/lsm_shared.c
+  %dir%/lsm_sorted.c
+  %dir%/lsm_str.c
+  %dir%/lsm_tree.c
+  %dir%/lsm_unix.c
+  %dir%/lsm_varint.c
+  %dir%/lsm_win32.c
+}]
+
+set G(hdr) {
+
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_LSM1) 
+
+#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) 
+# define NDEBUG 1
+#endif
+#if defined(NDEBUG) && defined(SQLITE_DEBUG)
+# undef NDEBUG
+#endif
+
+}
+
+set G(footer) {
+    
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_LSM1) */
+}
+
+#-------------------------------------------------------------------------
+# Read and return the entire contents of text file $zFile from disk.
+#
+proc readfile {zFile} {
+  set fd [open $zFile]
+  set data [read $fd]
+  close $fd
+  return $data
+}
+
+proc lsm1c_init {zOut} {
+  global G
+  set G(fd) stdout
+  set G(fd) [open $zOut w]
+
+  puts -nonewline $G(fd) $G(hdr)
+}
+
+proc lsm1c_printfile {zIn} {
+  global G
+  set data [readfile $zIn]
+  set zTail [file tail $zIn]
+  puts $G(fd) "#line 1 \"$zTail\""
+
+  foreach line [split $data "\n"] {
+    if {[regexp {^# *include.*lsm} $line]} {
+      set line "/* $line */"
+    } elseif { [regexp {^(const )?[a-zA-Z][a-zA-Z0-9]* [*]?lsm[^_]} $line] } {
+      set line "static $line"
+    }
+    puts $G(fd) $line
+  }
+}
+
+proc lsm1c_close {} {
+  global G
+  puts -nonewline $G(fd) $G(footer)
+  if {$G(fd)!="stdout"} {
+    close $G(fd)
+  }
+}
+
+
+lsm1c_init lsm1.c
+foreach f $G(src) { lsm1c_printfile $f }
+lsm1c_close

ext/misc/README.md

@@ -14,11 +14,20 @@ as follows:
      It is a good example of how to go about implementing a custom
      [table-valued function](https://www.sqlite.org/vtab.html#tabfunc2).
 
+  *  **csv.c** —  A [virtual table](https://sqlite.org/vtab.html)
+     for reading 
+     [Comma-Separated-Value (CSV) files](https://en.wikipedia.org/wiki/Comma-separated_values).
+
   *  **dbdump.c** —  This is not actually a loadable extension, but
      rather a library that implements an approximate equivalent to the
      ".dump" command of the
      [command-line shell](https://www.sqlite.org/cli.html).
 
+  *  **json1.c** —  Various SQL functions and table-valued functions
+     for processing JSON.  This extension is already built into the
+     [SQLite amalgamation](https://sqlite.org/amalgamation.html).  See
+     <https://sqlite.org/json1.html> for additional information.
+
   *  **memvfs.c** &mdash;  This file implements a custom
      [VFS](https://www.sqlite.org/vfs.html) that stores an entire database
      file in a single block of RAM.  It serves as a good example of how
@@ -38,3 +47,14 @@ as follows:
      on the source filename with digits removed, so if we used the name
      "sha3.c" then the entry point would conflict with the prior "sha1.c"
      extension.
+
+  *  **unionvtab.c** &mdash; Implementation of the unionvtab and
+     [swarmvtab](https://sqlite.org/swarmvtab.html) virtual tables.
+     These virtual tables allow a single
+     large table to be spread out across multiple database files.  In the
+     case of swarmvtab, the individual database files can be attached on
+     demand.
+
+  *  **zipfile.c** &mdash;  A [virtual table](https://sqlite.org/vtab.html)
+     that can read and write a 
+     [ZIP archive](https://en.wikipedia.org/wiki/Zip_%28file_format%29).

ext/misc/amatch.c

@@ -1473,7 +1473,8 @@ static sqlite3_module amatchModule = {
   0,                      /* xRename */
   0,                      /* xSavepoint */
   0,                      /* xRelease */
-  0                       /* xRollbackTo */
+  0,                      /* xRollbackTo */
+  0                       /* xShadowName */
 };
 
 #endif /* SQLITE_OMIT_VIRTUALTABLE */

ext/misc/appendvfs.c

@@ -0,0 +1,565 @@
+/*
+** 2017-10-20
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This file implements a VFS shim that allows an SQLite database to be
+** appended onto the end of some other file, such as an executable.
+**
+** A special record must appear at the end of the file that identifies the
+** file as an appended database and provides an offset to page 1.  For
+** best performance page 1 should be located at a disk page boundary, though
+** that is not required.
+**
+** When opening a database using this VFS, the connection might treat
+** the file as an ordinary SQLite database, or it might treat is as a
+** database appended onto some other file.  Here are the rules:
+**
+**  (1)  When opening a new empty file, that file is treated as an ordinary
+**       database.
+**
+**  (2)  When opening a file that begins with the standard SQLite prefix
+**       string "SQLite format 3", that file is treated as an ordinary
+**       database.
+**
+**  (3)  When opening a file that ends with the appendvfs trailer string
+**       "Start-Of-SQLite3-NNNNNNNN" that file is treated as an appended
+**       database.
+**
+**  (4)  If none of the above apply and the SQLITE_OPEN_CREATE flag is
+**       set, then a new database is appended to the already existing file.
+**
+**  (5)  Otherwise, SQLITE_CANTOPEN is returned.
+**
+** To avoid unnecessary complications with the PENDING_BYTE, the size of
+** the file containing the database is limited to 1GB.  This VFS will refuse
+** to read or write past the 1GB mark.  This restriction might be lifted in
+** future versions.  For now, if you need a large database, then keep the
+** database in a separate file.
+**
+** If the file being opened is not an appended database, then this shim is
+** a pass-through into the default underlying VFS.
+**/
+#include "sqlite3ext.h"
+SQLITE_EXTENSION_INIT1
+#include <string.h>
+#include <assert.h>
+
+/* The append mark at the end of the database is:
+**
+**     Start-Of-SQLite3-NNNNNNNN
+**     123456789 123456789 12345
+**
+** The NNNNNNNN represents a 64-bit big-endian unsigned integer which is
+** the offset to page 1.
+*/
+#define APND_MARK_PREFIX     "Start-Of-SQLite3-"
+#define APND_MARK_PREFIX_SZ  17
+#define APND_MARK_SIZE       25
+
+/*
+** Maximum size of the combined prefix + database + append-mark.  This
+** must be less than 0x40000000 to avoid locking issues on Windows.
+*/
+#define APND_MAX_SIZE  (65536*15259)
+
+/*
+** Forward declaration of objects used by this utility
+*/
+typedef struct sqlite3_vfs ApndVfs;
+typedef struct ApndFile ApndFile;
+
+/* Access to a lower-level VFS that (might) implement dynamic loading,
+** access to randomness, etc.
+*/
+#define ORIGVFS(p)  ((sqlite3_vfs*)((p)->pAppData))
+#define ORIGFILE(p) ((sqlite3_file*)(((ApndFile*)(p))+1))
+
+/* An open file */
+struct ApndFile {
+  sqlite3_file base;              /* IO methods */
+  sqlite3_int64 iPgOne;           /* File offset to page 1 */
+  sqlite3_int64 iMark;            /* Start of the append-mark */
+};
+
+/*
+** Methods for ApndFile
+*/
+static int apndClose(sqlite3_file*);
+static int apndRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst);
+static int apndWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst);
+static int apndTruncate(sqlite3_file*, sqlite3_int64 size);
+static int apndSync(sqlite3_file*, int flags);
+static int apndFileSize(sqlite3_file*, sqlite3_int64 *pSize);
+static int apndLock(sqlite3_file*, int);
+static int apndUnlock(sqlite3_file*, int);
+static int apndCheckReservedLock(sqlite3_file*, int *pResOut);
+static int apndFileControl(sqlite3_file*, int op, void *pArg);
+static int apndSectorSize(sqlite3_file*);
+static int apndDeviceCharacteristics(sqlite3_file*);
+static int apndShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**);
+static int apndShmLock(sqlite3_file*, int offset, int n, int flags);
+static void apndShmBarrier(sqlite3_file*);
+static int apndShmUnmap(sqlite3_file*, int deleteFlag);
+static int apndFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp);
+static int apndUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p);
+
+/*
+** Methods for ApndVfs
+*/
+static int apndOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *);
+static int apndDelete(sqlite3_vfs*, const char *zName, int syncDir);
+static int apndAccess(sqlite3_vfs*, const char *zName, int flags, int *);
+static int apndFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut);
+static void *apndDlOpen(sqlite3_vfs*, const char *zFilename);
+static void apndDlError(sqlite3_vfs*, int nByte, char *zErrMsg);
+static void (*apndDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void);
+static void apndDlClose(sqlite3_vfs*, void*);
+static int apndRandomness(sqlite3_vfs*, int nByte, char *zOut);
+static int apndSleep(sqlite3_vfs*, int microseconds);
+static int apndCurrentTime(sqlite3_vfs*, double*);
+static int apndGetLastError(sqlite3_vfs*, int, char *);
+static int apndCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*);
+static int apndSetSystemCall(sqlite3_vfs*, const char*,sqlite3_syscall_ptr);
+static sqlite3_syscall_ptr apndGetSystemCall(sqlite3_vfs*, const char *z);
+static const char *apndNextSystemCall(sqlite3_vfs*, const char *zName);
+
+static sqlite3_vfs apnd_vfs = {
+  3,                            /* iVersion (set when registered) */
+  0,                            /* szOsFile (set when registered) */
+  1024,                         /* mxPathname */
+  0,                            /* pNext */
+  "apndvfs",                    /* zName */
+  0,                            /* pAppData (set when registered) */ 
+  apndOpen,                     /* xOpen */
+  apndDelete,                   /* xDelete */
+  apndAccess,                   /* xAccess */
+  apndFullPathname,             /* xFullPathname */
+  apndDlOpen,                   /* xDlOpen */
+  apndDlError,                  /* xDlError */
+  apndDlSym,                    /* xDlSym */
+  apndDlClose,                  /* xDlClose */
+  apndRandomness,               /* xRandomness */
+  apndSleep,                    /* xSleep */
+  apndCurrentTime,              /* xCurrentTime */
+  apndGetLastError,             /* xGetLastError */
+  apndCurrentTimeInt64,         /* xCurrentTimeInt64 */
+  apndSetSystemCall,            /* xSetSystemCall */
+  apndGetSystemCall,            /* xGetSystemCall */
+  apndNextSystemCall            /* xNextSystemCall */
+};
+
+static const sqlite3_io_methods apnd_io_methods = {
+  3,                              /* iVersion */
+  apndClose,                      /* xClose */
+  apndRead,                       /* xRead */
+  apndWrite,                      /* xWrite */
+  apndTruncate,                   /* xTruncate */
+  apndSync,                       /* xSync */
+  apndFileSize,                   /* xFileSize */
+  apndLock,                       /* xLock */
+  apndUnlock,                     /* xUnlock */
+  apndCheckReservedLock,          /* xCheckReservedLock */
+  apndFileControl,                /* xFileControl */
+  apndSectorSize,                 /* xSectorSize */
+  apndDeviceCharacteristics,      /* xDeviceCharacteristics */
+  apndShmMap,                     /* xShmMap */
+  apndShmLock,                    /* xShmLock */
+  apndShmBarrier,                 /* xShmBarrier */
+  apndShmUnmap,                   /* xShmUnmap */
+  apndFetch,                      /* xFetch */
+  apndUnfetch                     /* xUnfetch */
+};
+
+
+
+/*
+** Close an apnd-file.
+*/
+static int apndClose(sqlite3_file *pFile){
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xClose(pFile);
+}
+
+/*
+** Read data from an apnd-file.
+*/
+static int apndRead(
+  sqlite3_file *pFile, 
+  void *zBuf, 
+  int iAmt, 
+  sqlite_int64 iOfst
+){
+  ApndFile *p = (ApndFile *)pFile;
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xRead(pFile, zBuf, iAmt, iOfst+p->iPgOne);
+}
+
+/*
+** Add the append-mark onto the end of the file.
+*/
+static int apndWriteMark(ApndFile *p, sqlite3_file *pFile){
+  int i;
+  unsigned char a[APND_MARK_SIZE];
+  memcpy(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ);
+  for(i=0; i<8; i++){
+    a[APND_MARK_PREFIX_SZ+i] = (p->iPgOne >> (56 - i*8)) & 0xff;
+  }
+  return pFile->pMethods->xWrite(pFile, a, APND_MARK_SIZE, p->iMark);
+}
+
+/*
+** Write data to an apnd-file.
+*/
+static int apndWrite(
+  sqlite3_file *pFile,
+  const void *zBuf,
+  int iAmt,
+  sqlite_int64 iOfst
+){
+  int rc;
+  ApndFile *p = (ApndFile *)pFile;
+  pFile = ORIGFILE(pFile);
+  if( iOfst+iAmt>=APND_MAX_SIZE ) return SQLITE_FULL;
+  rc = pFile->pMethods->xWrite(pFile, zBuf, iAmt, iOfst+p->iPgOne);
+  if( rc==SQLITE_OK &&  iOfst + iAmt + p->iPgOne > p->iMark ){
+    sqlite3_int64 sz = 0;
+    rc = pFile->pMethods->xFileSize(pFile, &sz);
+    if( rc==SQLITE_OK ){
+      p->iMark = sz - APND_MARK_SIZE;
+      if( iOfst + iAmt + p->iPgOne > p->iMark ){
+        p->iMark = p->iPgOne + iOfst + iAmt;
+        rc = apndWriteMark(p, pFile);
+      }
+    }
+  }
+  return rc;
+}
+
+/*
+** Truncate an apnd-file.
+*/
+static int apndTruncate(sqlite3_file *pFile, sqlite_int64 size){
+  int rc;
+  ApndFile *p = (ApndFile *)pFile;
+  pFile = ORIGFILE(pFile);
+  rc = pFile->pMethods->xTruncate(pFile, size+p->iPgOne+APND_MARK_SIZE);
+  if( rc==SQLITE_OK ){
+    p->iMark = p->iPgOne+size;
+    rc = apndWriteMark(p, pFile);
+  }
+  return rc;
+}
+
+/*
+** Sync an apnd-file.
+*/
+static int apndSync(sqlite3_file *pFile, int flags){
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xSync(pFile, flags);
+}
+
+/*
+** Return the current file-size of an apnd-file.
+*/
+static int apndFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){
+  ApndFile *p = (ApndFile *)pFile;
+  int rc;
+  pFile = ORIGFILE(p);
+  rc = pFile->pMethods->xFileSize(pFile, pSize);
+  if( rc==SQLITE_OK && p->iPgOne ){
+    *pSize -= p->iPgOne + APND_MARK_SIZE;
+  }
+  return rc;
+}
+
+/*
+** Lock an apnd-file.
+*/
+static int apndLock(sqlite3_file *pFile, int eLock){
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xLock(pFile, eLock);
+}
+
+/*
+** Unlock an apnd-file.
+*/
+static int apndUnlock(sqlite3_file *pFile, int eLock){
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xUnlock(pFile, eLock);
+}
+
+/*
+** Check if another file-handle holds a RESERVED lock on an apnd-file.
+*/
+static int apndCheckReservedLock(sqlite3_file *pFile, int *pResOut){
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xCheckReservedLock(pFile, pResOut);
+}
+
+/*
+** File control method. For custom operations on an apnd-file.
+*/
+static int apndFileControl(sqlite3_file *pFile, int op, void *pArg){
+  ApndFile *p = (ApndFile *)pFile;
+  int rc;
+  pFile = ORIGFILE(pFile);
+  rc = pFile->pMethods->xFileControl(pFile, op, pArg);
+  if( rc==SQLITE_OK && op==SQLITE_FCNTL_VFSNAME ){
+    *(char**)pArg = sqlite3_mprintf("apnd(%lld)/%z", p->iPgOne, *(char**)pArg);
+  }
+  return rc;
+}
+
+/*
+** Return the sector-size in bytes for an apnd-file.
+*/
+static int apndSectorSize(sqlite3_file *pFile){
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xSectorSize(pFile);
+}
+
+/*
+** Return the device characteristic flags supported by an apnd-file.
+*/
+static int apndDeviceCharacteristics(sqlite3_file *pFile){
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xDeviceCharacteristics(pFile);
+}
+
+/* Create a shared memory file mapping */
+static int apndShmMap(
+  sqlite3_file *pFile,
+  int iPg,
+  int pgsz,
+  int bExtend,
+  void volatile **pp
+){
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xShmMap(pFile,iPg,pgsz,bExtend,pp);
+}
+
+/* Perform locking on a shared-memory segment */
+static int apndShmLock(sqlite3_file *pFile, int offset, int n, int flags){
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xShmLock(pFile,offset,n,flags);
+}
+
+/* Memory barrier operation on shared memory */
+static void apndShmBarrier(sqlite3_file *pFile){
+  pFile = ORIGFILE(pFile);
+  pFile->pMethods->xShmBarrier(pFile);
+}
+
+/* Unmap a shared memory segment */
+static int apndShmUnmap(sqlite3_file *pFile, int deleteFlag){
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xShmUnmap(pFile,deleteFlag);
+}
+
+/* Fetch a page of a memory-mapped file */
+static int apndFetch(
+  sqlite3_file *pFile,
+  sqlite3_int64 iOfst,
+  int iAmt,
+  void **pp
+){
+  ApndFile *p = (ApndFile *)pFile;
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xFetch(pFile, iOfst+p->iPgOne, iAmt, pp);
+}
+
+/* Release a memory-mapped page */
+static int apndUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){
+  ApndFile *p = (ApndFile *)pFile;
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xUnfetch(pFile, iOfst+p->iPgOne, pPage);
+}
+
+/*
+** Check to see if the file is an ordinary SQLite database file.
+*/
+static int apndIsOrdinaryDatabaseFile(sqlite3_int64 sz, sqlite3_file *pFile){
+  int rc;
+  char zHdr[16];
+  static const char aSqliteHdr[] = "SQLite format 3";
+  if( sz<512 ) return 0;
+  rc = pFile->pMethods->xRead(pFile, zHdr, sizeof(zHdr), 0);
+  if( rc ) return 0;
+  return memcmp(zHdr, aSqliteHdr, sizeof(zHdr))==0;
+}
+
+/*
+** Try to read the append-mark off the end of a file.  Return the
+** start of the appended database if the append-mark is present.  If
+** there is no append-mark, return -1;
+*/
+static sqlite3_int64 apndReadMark(sqlite3_int64 sz, sqlite3_file *pFile){
+  int rc, i;
+  sqlite3_int64 iMark;
+  unsigned char a[APND_MARK_SIZE];
+
+  if( sz<=APND_MARK_SIZE ) return -1;
+  rc = pFile->pMethods->xRead(pFile, a, APND_MARK_SIZE, sz-APND_MARK_SIZE);
+  if( rc ) return -1;
+  if( memcmp(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ)!=0 ) return -1;
+  iMark = ((sqlite3_int64)(a[APND_MARK_PREFIX_SZ]&0x7f))<<56;
+  for(i=1; i<8; i++){    
+    iMark += (sqlite3_int64)a[APND_MARK_PREFIX_SZ+i]<<(56-8*i);
+  }
+  return iMark;
+}
+
+/*
+** Open an apnd file handle.
+*/
+static int apndOpen(
+  sqlite3_vfs *pVfs,
+  const char *zName,
+  sqlite3_file *pFile,
+  int flags,
+  int *pOutFlags
+){
+  ApndFile *p;
+  sqlite3_file *pSubFile;
+  sqlite3_vfs *pSubVfs;
+  int rc;
+  sqlite3_int64 sz;
+  pSubVfs = ORIGVFS(pVfs);
+  if( (flags & SQLITE_OPEN_MAIN_DB)==0 ){
+    return pSubVfs->xOpen(pSubVfs, zName, pFile, flags, pOutFlags);
+  }
+  p = (ApndFile*)pFile;
+  memset(p, 0, sizeof(*p));
+  pSubFile = ORIGFILE(pFile);
+  p->base.pMethods = &apnd_io_methods;
+  rc = pSubVfs->xOpen(pSubVfs, zName, pSubFile, flags, pOutFlags);
+  if( rc ) goto apnd_open_done;
+  rc = pSubFile->pMethods->xFileSize(pSubFile, &sz);
+  if( rc ){
+    pSubFile->pMethods->xClose(pSubFile);
+    goto apnd_open_done;
+  }
+  if( apndIsOrdinaryDatabaseFile(sz, pSubFile) ){
+    memmove(pFile, pSubFile, pSubVfs->szOsFile);
+    return SQLITE_OK;
+  }
+  p->iMark = 0;
+  p->iPgOne = apndReadMark(sz, pFile);
+  if( p->iPgOne>0 ){
+    return SQLITE_OK;
+  }
+  if( (flags & SQLITE_OPEN_CREATE)==0 ){
+    pSubFile->pMethods->xClose(pSubFile);
+    rc = SQLITE_CANTOPEN;
+  }
+  p->iPgOne = (sz+0xfff) & ~(sqlite3_int64)0xfff;
+apnd_open_done:
+  if( rc ) pFile->pMethods = 0;
+  return rc;
+}
+
+/*
+** All other VFS methods are pass-thrus.
+*/
+static int apndDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){
+  return ORIGVFS(pVfs)->xDelete(ORIGVFS(pVfs), zPath, dirSync);
+}
+static int apndAccess(
+  sqlite3_vfs *pVfs, 
+  const char *zPath, 
+  int flags, 
+  int *pResOut
+){
+  return ORIGVFS(pVfs)->xAccess(ORIGVFS(pVfs), zPath, flags, pResOut);
+}
+static int apndFullPathname(
+  sqlite3_vfs *pVfs, 
+  const char *zPath, 
+  int nOut, 
+  char *zOut
+){
+  return ORIGVFS(pVfs)->xFullPathname(ORIGVFS(pVfs),zPath,nOut,zOut);
+}
+static void *apndDlOpen(sqlite3_vfs *pVfs, const char *zPath){
+  return ORIGVFS(pVfs)->xDlOpen(ORIGVFS(pVfs), zPath);
+}
+static void apndDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){
+  ORIGVFS(pVfs)->xDlError(ORIGVFS(pVfs), nByte, zErrMsg);
+}
+static void (*apndDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){
+  return ORIGVFS(pVfs)->xDlSym(ORIGVFS(pVfs), p, zSym);
+}
+static void apndDlClose(sqlite3_vfs *pVfs, void *pHandle){
+  ORIGVFS(pVfs)->xDlClose(ORIGVFS(pVfs), pHandle);
+}
+static int apndRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){
+  return ORIGVFS(pVfs)->xRandomness(ORIGVFS(pVfs), nByte, zBufOut);
+}
+static int apndSleep(sqlite3_vfs *pVfs, int nMicro){
+  return ORIGVFS(pVfs)->xSleep(ORIGVFS(pVfs), nMicro);
+}
+static int apndCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){
+  return ORIGVFS(pVfs)->xCurrentTime(ORIGVFS(pVfs), pTimeOut);
+}
+static int apndGetLastError(sqlite3_vfs *pVfs, int a, char *b){
+  return ORIGVFS(pVfs)->xGetLastError(ORIGVFS(pVfs), a, b);
+}
+static int apndCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){
+  return ORIGVFS(pVfs)->xCurrentTimeInt64(ORIGVFS(pVfs), p);
+}
+static int apndSetSystemCall(
+  sqlite3_vfs *pVfs,
+  const char *zName,
+  sqlite3_syscall_ptr pCall
+){
+  return ORIGVFS(pVfs)->xSetSystemCall(ORIGVFS(pVfs),zName,pCall);
+}
+static sqlite3_syscall_ptr apndGetSystemCall(
+  sqlite3_vfs *pVfs,
+  const char *zName
+){
+  return ORIGVFS(pVfs)->xGetSystemCall(ORIGVFS(pVfs),zName);
+}
+static const char *apndNextSystemCall(sqlite3_vfs *pVfs, const char *zName){
+  return ORIGVFS(pVfs)->xNextSystemCall(ORIGVFS(pVfs), zName);
+}
+
+  
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+/* 
+** This routine is called when the extension is loaded.
+** Register the new VFS.
+*/
+int sqlite3_appendvfs_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  int rc = SQLITE_OK;
+  sqlite3_vfs *pOrig;
+  SQLITE_EXTENSION_INIT2(pApi);
+  (void)pzErrMsg;
+  (void)db;
+  pOrig = sqlite3_vfs_find(0);
+  apnd_vfs.iVersion = pOrig->iVersion;
+  apnd_vfs.pAppData = pOrig;
+  apnd_vfs.szOsFile = pOrig->szOsFile + sizeof(ApndFile);
+  rc = sqlite3_vfs_register(&apnd_vfs, 0);
+#ifdef APPENDVFS_TEST
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_auto_extension((void(*)(void))apndvfsRegister);
+  }
+#endif
+  if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY;
+  return rc;
+}

ext/misc/bgckpt.c

@@ -231,8 +231,14 @@ int Bgckpt_Init(Tcl_Interp *interp){
 #endif   /* SQLITE_TEST */
 
 #else
-int Bgckpt_Init(Tcl_Interp *interp){
+#if defined(INCLUDE_SQLITE_TCL_H)
-  return TCL_OK;
+#  include "sqlite_tcl.h"
-}
+#else
+#  include "tcl.h"
+#  ifndef SQLITE_TCLAPI
+#    define SQLITE_TCLAPI
+#  endif
+#endif
+int Bgckpt_Init(Tcl_Interp *interp){ return TCL_OK; }
 #endif
 

ext/misc/btreeinfo.c

@@ -0,0 +1,429 @@
+/*
+** 2017-10-24
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This file contains an implementation of the "sqlite_btreeinfo" virtual table.
+**
+** The sqlite_btreeinfo virtual table is a read-only eponymous-only virtual
+** table that shows information about all btrees in an SQLite database file.
+** The schema is like this:
+**
+** CREATE TABLE sqlite_btreeinfo(
+**    type TEXT,                   -- "table" or "index"
+**    name TEXT,                   -- Name of table or index for this btree.
+**    tbl_name TEXT,               -- Associated table
+**    rootpage INT,                -- The root page of the btree
+**    sql TEXT,                    -- SQL for this btree - from sqlite_master
+**    hasRowid BOOLEAN,            -- True if the btree has a rowid
+**    nEntry INT,                  -- Estimated number of enteries
+**    nPage INT,                   -- Estimated number of pages
+**    depth INT,                   -- Depth of the btree
+**    szPage INT,                  -- Size of each page in bytes
+**    zSchema TEXT HIDDEN          -- The schema to which this btree belongs
+** );
+**
+** The first 5 fields are taken directly from the sqlite_master table.
+** Considering only the first 5 fields, the only difference between 
+** this virtual table and the sqlite_master table is that this virtual
+** table omits all entries that have a 0 or NULL rowid - in other words
+** it omits triggers and views.
+**
+** The value added by this table comes in the next 5 fields.
+**
+** Note that nEntry and nPage are *estimated*.  They are computed doing
+** a single search from the root to a leaf, counting the number of cells
+** at each level, and assuming that unvisited pages have a similar number
+** of cells.
+**
+** The sqlite_dbpage virtual table must be available for this virtual table
+** to operate.
+**
+** USAGE EXAMPLES:
+**
+** Show the table btrees in a schema order with the tables with the most
+** rows occuring first:
+**
+**      SELECT name, nEntry
+**        FROM sqlite_btreeinfo
+**       WHERE type='table'
+**       ORDER BY nEntry DESC, name;
+**
+** Show the names of all WITHOUT ROWID tables: 
+**
+**      SELECT name FROM sqlite_btreeinfo
+**       WHERE type='table' AND NOT hasRowid;
+*/
+#if !defined(SQLITEINT_H)
+#include "sqlite3ext.h"
+#endif
+SQLITE_EXTENSION_INIT1
+#include <string.h>
+#include <assert.h>
+
+/* Columns available in this virtual table */
+#define BINFO_COLUMN_TYPE         0
+#define BINFO_COLUMN_NAME         1
+#define BINFO_COLUMN_TBL_NAME     2
+#define BINFO_COLUMN_ROOTPAGE     3
+#define BINFO_COLUMN_SQL          4
+#define BINFO_COLUMN_HASROWID     5
+#define BINFO_COLUMN_NENTRY       6
+#define BINFO_COLUMN_NPAGE        7
+#define BINFO_COLUMN_DEPTH        8
+#define BINFO_COLUMN_SZPAGE       9
+#define BINFO_COLUMN_SCHEMA      10
+
+/* Forward declarations */
+typedef struct BinfoTable BinfoTable;
+typedef struct BinfoCursor BinfoCursor;
+
+/* A cursor for the sqlite_btreeinfo table */
+struct BinfoCursor {
+  sqlite3_vtab_cursor base;       /* Base class.  Must be first */
+  sqlite3_stmt *pStmt;            /* Query against sqlite_master */
+  int rc;                         /* Result of previous sqlite_step() call */
+  int hasRowid;                   /* hasRowid value.  Negative if unknown. */
+  sqlite3_int64 nEntry;           /* nEntry value */
+  int nPage;                      /* nPage value */
+  int depth;                      /* depth value */
+  int szPage;                     /* size of a btree page.  0 if unknown */
+  char *zSchema;                  /* Schema being interrogated */
+};
+
+/* The sqlite_btreeinfo table */
+struct BinfoTable {
+  sqlite3_vtab base;              /* Base class.  Must be first */
+  sqlite3 *db;                    /* The databse connection */
+};
+
+/*
+** Connect to the sqlite_btreeinfo virtual table.
+*/
+static int binfoConnect(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
+){
+  BinfoTable *pTab = 0;
+  int rc = SQLITE_OK;
+  rc = sqlite3_declare_vtab(db, 
+      "CREATE TABLE x(\n"
+      " type TEXT,\n"
+      " name TEXT,\n"
+      " tbl_name TEXT,\n"
+      " rootpage INT,\n"
+      " sql TEXT,\n"
+      " hasRowid BOOLEAN,\n"
+      " nEntry INT,\n"
+      " nPage INT,\n"
+      " depth INT,\n"
+      " szPage INT,\n"
+      " zSchema TEXT HIDDEN\n"
+      ")");
+  if( rc==SQLITE_OK ){
+    pTab = (BinfoTable *)sqlite3_malloc64(sizeof(BinfoTable));
+    if( pTab==0 ) rc = SQLITE_NOMEM;
+  }
+  assert( rc==SQLITE_OK || pTab==0 );
+  if( pTab ){
+    pTab->db = db;
+  }
+  *ppVtab = (sqlite3_vtab*)pTab;
+  return rc;
+}
+
+/*
+** Disconnect from or destroy a btreeinfo virtual table.
+*/
+static int binfoDisconnect(sqlite3_vtab *pVtab){
+  sqlite3_free(pVtab);
+  return SQLITE_OK;
+}
+
+/*
+** idxNum:
+**
+**     0     Use "main" for the schema
+**     1     Schema identified by parameter ?1
+*/
+static int binfoBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
+  int i;
+  pIdxInfo->estimatedCost = 10000.0;  /* Cost estimate */
+  pIdxInfo->estimatedRows = 100;
+  for(i=0; i<pIdxInfo->nConstraint; i++){
+    struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[i];
+    if( p->usable
+     && p->iColumn==BINFO_COLUMN_SCHEMA
+     && p->op==SQLITE_INDEX_CONSTRAINT_EQ
+    ){
+      pIdxInfo->estimatedCost = 1000.0;
+      pIdxInfo->idxNum = 1;
+      pIdxInfo->aConstraintUsage[i].argvIndex = 1;
+      pIdxInfo->aConstraintUsage[i].omit = 1;
+      break;
+    }
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Open a new btreeinfo cursor.
+*/
+static int binfoOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
+  BinfoCursor *pCsr;
+
+  pCsr = (BinfoCursor *)sqlite3_malloc64(sizeof(BinfoCursor));
+  if( pCsr==0 ){
+    return SQLITE_NOMEM;
+  }else{
+    memset(pCsr, 0, sizeof(BinfoCursor));
+    pCsr->base.pVtab = pVTab;
+  }
+
+  *ppCursor = (sqlite3_vtab_cursor *)pCsr;
+  return SQLITE_OK;
+}
+
+/*
+** Close a btreeinfo cursor.
+*/
+static int binfoClose(sqlite3_vtab_cursor *pCursor){
+  BinfoCursor *pCsr = (BinfoCursor *)pCursor;
+  sqlite3_finalize(pCsr->pStmt);
+  sqlite3_free(pCsr->zSchema);
+  sqlite3_free(pCsr);
+  return SQLITE_OK;
+}
+
+/*
+** Move a btreeinfo cursor to the next entry in the file.
+*/
+static int binfoNext(sqlite3_vtab_cursor *pCursor){
+  BinfoCursor *pCsr = (BinfoCursor *)pCursor;
+  pCsr->rc = sqlite3_step(pCsr->pStmt);
+  pCsr->hasRowid = -1;
+  return pCsr->rc==SQLITE_ERROR ? SQLITE_ERROR : SQLITE_OK;
+}
+
+/* We have reached EOF if previous sqlite3_step() returned
+** anything other than SQLITE_ROW;
+*/
+static int binfoEof(sqlite3_vtab_cursor *pCursor){
+  BinfoCursor *pCsr = (BinfoCursor *)pCursor;
+  return pCsr->rc!=SQLITE_ROW;
+}
+
+/* Position a cursor back to the beginning.
+*/
+static int binfoFilter(
+  sqlite3_vtab_cursor *pCursor, 
+  int idxNum, const char *idxStr,
+  int argc, sqlite3_value **argv
+){
+  BinfoCursor *pCsr = (BinfoCursor *)pCursor;
+  BinfoTable *pTab = (BinfoTable *)pCursor->pVtab;
+  char *zSql;
+  int rc;
+
+  sqlite3_free(pCsr->zSchema);
+  if( idxNum==1 && sqlite3_value_type(argv[0])!=SQLITE_NULL ){
+    pCsr->zSchema = sqlite3_mprintf("%s", sqlite3_value_text(argv[0]));
+  }else{
+    pCsr->zSchema = sqlite3_mprintf("main");
+  }
+  zSql = sqlite3_mprintf(
+      "SELECT 0, 'table','sqlite_master','sqlite_master',1,NULL "
+      "UNION ALL "
+      "SELECT rowid, type, name, tbl_name, rootpage, sql"
+      " FROM \"%w\".sqlite_master WHERE rootpage>=1",
+       pCsr->zSchema);
+  sqlite3_finalize(pCsr->pStmt);
+  pCsr->pStmt = 0;
+  pCsr->hasRowid = -1;
+  rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0);
+  sqlite3_free(zSql);
+  if( rc==SQLITE_OK ){
+    rc = binfoNext(pCursor);
+  }
+  return rc;
+}
+
+/* Decode big-endian integers */
+static unsigned int get_uint16(unsigned char *a){
+  return (a[0]<<8)|a[1];
+}
+static unsigned int get_uint32(unsigned char *a){
+  return (a[0]<<24)|(a[1]<<16)|(a[2]<<8)|a[3];
+}
+
+/* Examine the b-tree rooted at pgno and estimate its size.
+** Return non-zero if anything goes wrong.
+*/
+static int binfoCompute(sqlite3 *db, int pgno, BinfoCursor *pCsr){
+  sqlite3_int64 nEntry = 1;
+  int nPage = 1;
+  unsigned char *aData;
+  sqlite3_stmt *pStmt = 0;
+  int rc = SQLITE_OK;
+  int pgsz = 0;
+  int nCell;
+  int iCell;
+
+  rc = sqlite3_prepare_v2(db, 
+           "SELECT data FROM sqlite_dbpage('main') WHERE pgno=?1", -1,
+           &pStmt, 0);
+  if( rc ) return rc;
+  pCsr->depth = 1;
+  while(1){
+    sqlite3_bind_int(pStmt, 1, pgno);
+    rc = sqlite3_step(pStmt);
+    if( rc!=SQLITE_ROW ){
+      rc = SQLITE_ERROR;
+      break;
+    }
+    pCsr->szPage = pgsz = sqlite3_column_bytes(pStmt, 0);
+    aData = (unsigned char*)sqlite3_column_blob(pStmt, 0);
+    if( aData==0 ){    
+      rc = SQLITE_NOMEM;
+      break;
+    }
+    if( pgno==1 ){
+      aData += 100;
+      pgsz -= 100;
+    }
+    pCsr->hasRowid = aData[0]!=2 && aData[0]!=10;
+    nCell = get_uint16(aData+3);
+    nEntry *= (nCell+1);
+    if( aData[0]==10 || aData[0]==13 ) break;
+    nPage *= (nCell+1);
+    if( nCell<=1 ){
+      pgno = get_uint32(aData+8);
+    }else{
+      iCell = get_uint16(aData+12+2*(nCell/2));
+      if( pgno==1 ) iCell -= 100;
+      if( iCell<=12 || iCell>=pgsz-4 ){
+        rc = SQLITE_CORRUPT;
+        break;
+      }
+      pgno = get_uint32(aData+iCell);
+    }
+    pCsr->depth++;
+    sqlite3_reset(pStmt);
+  }
+  sqlite3_finalize(pStmt);
+  pCsr->nPage = nPage;
+  pCsr->nEntry = nEntry;
+  if( rc==SQLITE_ROW ) rc = SQLITE_OK;
+  return rc;
+}
+
+/* Return a column for the sqlite_btreeinfo table */
+static int binfoColumn(
+  sqlite3_vtab_cursor *pCursor, 
+  sqlite3_context *ctx, 
+  int i
+){
+  BinfoCursor *pCsr = (BinfoCursor *)pCursor;
+  if( i>=BINFO_COLUMN_HASROWID && i<=BINFO_COLUMN_SZPAGE && pCsr->hasRowid<0 ){
+    int pgno = sqlite3_column_int(pCsr->pStmt, BINFO_COLUMN_ROOTPAGE+1);
+    sqlite3 *db = sqlite3_context_db_handle(ctx);
+    int rc = binfoCompute(db, pgno, pCsr);
+    if( rc ){
+      pCursor->pVtab->zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
+      return SQLITE_ERROR;
+    }
+  }
+  switch( i ){
+    case BINFO_COLUMN_NAME:
+    case BINFO_COLUMN_TYPE:
+    case BINFO_COLUMN_TBL_NAME:
+    case BINFO_COLUMN_ROOTPAGE:
+    case BINFO_COLUMN_SQL: {
+      sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pStmt, i+1));
+      break;
+    }
+    case BINFO_COLUMN_HASROWID: {
+      sqlite3_result_int(ctx, pCsr->hasRowid);
+      break;
+    }
+    case BINFO_COLUMN_NENTRY: {
+      sqlite3_result_int64(ctx, pCsr->nEntry);
+      break;
+    }
+    case BINFO_COLUMN_NPAGE: {
+      sqlite3_result_int(ctx, pCsr->nPage);
+      break;
+    }
+    case BINFO_COLUMN_DEPTH: {
+      sqlite3_result_int(ctx, pCsr->depth);
+      break;
+    }
+    case BINFO_COLUMN_SCHEMA: {
+      sqlite3_result_text(ctx, pCsr->zSchema, -1, SQLITE_STATIC);
+      break;
+    }
+  }
+  return SQLITE_OK;
+}
+
+/* Return the ROWID for the sqlite_btreeinfo table */
+static int binfoRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
+  BinfoCursor *pCsr = (BinfoCursor *)pCursor;
+  *pRowid = sqlite3_column_int64(pCsr->pStmt, 0);
+  return SQLITE_OK;
+}
+
+/*
+** Invoke this routine to register the "sqlite_btreeinfo" virtual table module
+*/
+int sqlite3BinfoRegister(sqlite3 *db){
+  static sqlite3_module binfo_module = {
+    0,                           /* iVersion */
+    0,                           /* xCreate */
+    binfoConnect,                /* xConnect */
+    binfoBestIndex,              /* xBestIndex */
+    binfoDisconnect,             /* xDisconnect */
+    0,                           /* xDestroy */
+    binfoOpen,                   /* xOpen - open a cursor */
+    binfoClose,                  /* xClose - close a cursor */
+    binfoFilter,                 /* xFilter - configure scan constraints */
+    binfoNext,                   /* xNext - advance a cursor */
+    binfoEof,                    /* xEof - check for end of scan */
+    binfoColumn,                 /* xColumn - read data */
+    binfoRowid,                  /* xRowid - read data */
+    0,                           /* xUpdate */
+    0,                           /* xBegin */
+    0,                           /* xSync */
+    0,                           /* xCommit */
+    0,                           /* xRollback */
+    0,                           /* xFindMethod */
+    0,                           /* xRename */
+    0,                           /* xSavepoint */
+    0,                           /* xRelease */
+    0,                           /* xRollbackTo */
+    0                            /* xShadowName */
+  };
+  return sqlite3_create_module(db, "sqlite_btreeinfo", &binfo_module, 0);
+}
+
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_btreeinfo_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  SQLITE_EXTENSION_INIT2(pApi);
+  return sqlite3BinfoRegister(db);
+}

ext/misc/closure.c

@@ -826,17 +826,12 @@ static int closureBestIndex(
   int iPlan = 0;
   int i;
   int idx = 1;
-  int seenMatch = 0;
   const struct sqlite3_index_constraint *pConstraint;
   closure_vtab *pVtab = (closure_vtab*)pTab;
   double rCost = 10000000.0;
 
   pConstraint = pIdxInfo->aConstraint;
   for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){
-    if( pConstraint->iColumn==CLOSURE_COL_ROOT
-     && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){
-      seenMatch = 1;
-    }
     if( pConstraint->usable==0 ) continue;
     if( (iPlan & 1)==0 
      && pConstraint->iColumn==CLOSURE_COL_ROOT
@@ -893,6 +888,18 @@ static int closureBestIndex(
     ** or else the result is an empty set. */
     iPlan = 0;
   }
+  if( (iPlan&1)==0 ){
+    /* If there is no usable "root=?" term, then set the index-type to 0.
+    ** Also clear any argvIndex variables already set. This is necessary
+    ** to prevent the core from throwing an "xBestIndex malfunction error"
+    ** error (because the argvIndex values are not contiguously assigned
+    ** starting from 1).  */
+    rCost *= 1e30;
+    for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){
+      pIdxInfo->aConstraintUsage[i].argvIndex = 0;
+    }
+    iPlan = 0;
+  }
   pIdxInfo->idxNum = iPlan;
   if( pIdxInfo->nOrderBy==1
    && pIdxInfo->aOrderBy[0].iColumn==CLOSURE_COL_ID
@@ -900,7 +907,6 @@ static int closureBestIndex(
   ){
     pIdxInfo->orderByConsumed = 1;
   }
-  if( seenMatch && (iPlan&1)==0 ) rCost *= 1e30;
   pIdxInfo->estimatedCost = rCost;
    
   return SQLITE_OK;
@@ -932,7 +938,8 @@ static sqlite3_module closureModule = {
   0,                      /* xRename */
   0,                      /* xSavepoint */
   0,                      /* xRelease */
-  0                       /* xRollbackTo */
+  0,                      /* xRollbackTo */
+  0                       /* xShadowName */
 };
 
 #endif /* SQLITE_OMIT_VIRTUALTABLE */

ext/misc/completion.c

@@ -62,6 +62,7 @@ struct completion_cursor {
   char *zPrefix;             /* The prefix for the word we want to complete */
   char *zLine;               /* The whole that we want to complete */
   const char *zCurrentRow;   /* Current output row */
+  int szRow;                 /* Length of the zCurrentRow string */
   sqlite3_stmt *pStmt;       /* Current statement */
   sqlite3_int64 iRowid;      /* The rowid */
   int ePhase;                /* Current phase */
@@ -78,7 +79,7 @@ struct completion_cursor {
 #define COMPLETION_INDEXES       5
 #define COMPLETION_TRIGGERS      6
 #define COMPLETION_DATABASES     7
-#define COMPLETION_TABLES        8
+#define COMPLETION_TABLES        8    /* Also VIEWs and TRIGGERs */
 #define COMPLETION_COLUMNS       9
 #define COMPLETION_MODULES       10
 #define COMPLETION_EOF           11
@@ -174,32 +175,6 @@ static int completionClose(sqlite3_vtab_cursor *cur){
   return SQLITE_OK;
 }
 
-/*
-** All SQL keywords understood by SQLite
-*/
-static const char *completionKwrds[] = {
-  "ABORT", "ACTION", "ADD", "AFTER", "ALL", "ALTER", "ANALYZE", "AND", "AS",
-  "ASC", "ATTACH", "AUTOINCREMENT", "BEFORE", "BEGIN", "BETWEEN", "BY",
-  "CASCADE", "CASE", "CAST", "CHECK", "COLLATE", "COLUMN", "COMMIT",
-  "CONFLICT", "CONSTRAINT", "CREATE", "CROSS", "CURRENT_DATE",
-  "CURRENT_TIME", "CURRENT_TIMESTAMP", "DATABASE", "DEFAULT", "DEFERRABLE",
-  "DEFERRED", "DELETE", "DESC", "DETACH", "DISTINCT", "DROP", "EACH",
-  "ELSE", "END", "ESCAPE", "EXCEPT", "EXCLUSIVE", "EXISTS", "EXPLAIN",
-  "FAIL", "FOR", "FOREIGN", "FROM", "FULL", "GLOB", "GROUP", "HAVING", "IF",
-  "IGNORE", "IMMEDIATE", "IN", "INDEX", "INDEXED", "INITIALLY", "INNER",
-  "INSERT", "INSTEAD", "INTERSECT", "INTO", "IS", "ISNULL", "JOIN", "KEY",
-  "LEFT", "LIKE", "LIMIT", "MATCH", "NATURAL", "NO", "NOT", "NOTNULL",
-  "NULL", "OF", "OFFSET", "ON", "OR", "ORDER", "OUTER", "PLAN", "PRAGMA",
-  "PRIMARY", "QUERY", "RAISE", "RECURSIVE", "REFERENCES", "REGEXP",
-  "REINDEX", "RELEASE", "RENAME", "REPLACE", "RESTRICT", "RIGHT",
-  "ROLLBACK", "ROW", "SAVEPOINT", "SELECT", "SET", "TABLE", "TEMP",
-  "TEMPORARY", "THEN", "TO", "TRANSACTION", "TRIGGER", "UNION", "UNIQUE",
-  "UPDATE", "USING", "VACUUM", "VALUES", "VIEW", "VIRTUAL", "WHEN", "WHERE",
-  "WITH", "WITHOUT",
-};
-#define completionKwCount \
-   (int)(sizeof(completionKwrds)/sizeof(completionKwrds[0]))
-
 /*
 ** Advance a completion_cursor to its next row of output.
 **
@@ -222,11 +197,11 @@ static int completionNext(sqlite3_vtab_cursor *cur){
   while( pCur->ePhase!=COMPLETION_EOF ){
     switch( pCur->ePhase ){
       case COMPLETION_KEYWORDS: {
-        if( pCur->j >= completionKwCount ){
+        if( pCur->j >= sqlite3_keyword_count() ){
           pCur->zCurrentRow = 0;
           pCur->ePhase = COMPLETION_DATABASES;
         }else{
-          pCur->zCurrentRow = completionKwrds[pCur->j++];
+          sqlite3_keyword_name(pCur->j++, &pCur->zCurrentRow, &pCur->szRow);
         }
         iCol = -1;
         break;
@@ -250,8 +225,7 @@ static int completionNext(sqlite3_vtab_cursor *cur){
             const char *zDb = (const char*)sqlite3_column_text(pS2, 1);
             zSql = sqlite3_mprintf(
                "%z%s"
-               "SELECT name FROM \"%w\".sqlite_master"
+               "SELECT name FROM \"%w\".sqlite_master",
-               " WHERE type='table'",
                zSql, zSep, zDb
             );
             if( zSql==0 ) return SQLITE_NOMEM;
@@ -299,6 +273,7 @@ static int completionNext(sqlite3_vtab_cursor *cur){
       if( sqlite3_step(pCur->pStmt)==SQLITE_ROW ){
         /* Extract the next row of content */
         pCur->zCurrentRow = (const char*)sqlite3_column_text(pCur->pStmt, iCol);
+        pCur->szRow = sqlite3_column_bytes(pCur->pStmt, iCol);
       }else{
         /* When all rows are finished, advance to the next phase */
         sqlite3_finalize(pCur->pStmt);
@@ -308,7 +283,9 @@ static int completionNext(sqlite3_vtab_cursor *cur){
       }
     }
     if( pCur->nPrefix==0 ) break;
-    if( sqlite3_strnicmp(pCur->zPrefix, pCur->zCurrentRow, pCur->nPrefix)==0 ){
+    if( pCur->nPrefix<=pCur->szRow
+     && sqlite3_strnicmp(pCur->zPrefix, pCur->zCurrentRow, pCur->nPrefix)==0
+    ){
       break;
     }
   }
@@ -328,7 +305,7 @@ static int completionColumn(
   completion_cursor *pCur = (completion_cursor*)cur;
   switch( i ){
     case COMPLETION_COLUMN_CANDIDATE: {
-      sqlite3_result_text(ctx, pCur->zCurrentRow, -1, SQLITE_TRANSIENT);
+      sqlite3_result_text(ctx, pCur->zCurrentRow, pCur->szRow,SQLITE_TRANSIENT);
       break;
     }
     case COMPLETION_COLUMN_PREFIX: {
@@ -388,7 +365,7 @@ static int completionFilter(
       pCur->zPrefix = sqlite3_mprintf("%s", sqlite3_value_text(argv[iArg]));
       if( pCur->zPrefix==0 ) return SQLITE_NOMEM;
     }
-    iArg++;
+    iArg = 1;
   }
   if( idxNum & 2 ){
     pCur->nLine = sqlite3_value_bytes(argv[iArg]);
@@ -396,7 +373,6 @@ static int completionFilter(
       pCur->zLine = sqlite3_mprintf("%s", sqlite3_value_text(argv[iArg]));
       if( pCur->zLine==0 ) return SQLITE_NOMEM;
     }
-    iArg++;
   }
   if( pCur->zLine!=0 && pCur->zPrefix==0 ){
     int i = pCur->nLine;
@@ -492,7 +468,8 @@ static sqlite3_module completionModule = {
   0,                         /* xRename */
   0,                         /* xSavepoint */
   0,                         /* xRelease */
-  0                          /* xRollbackTo */
+  0,                         /* xRollbackTo */
+  0                          /* xShadowName */
 };
 
 #endif /* SQLITE_OMIT_VIRTUALTABLE */

ext/misc/compress.c

@@ -27,6 +27,21 @@ SQLITE_EXTENSION_INIT1
 ** seven bits per integer stored in the lower seven bits of each byte.
 ** More significant bits occur first.  The most significant bit (0x80)
 ** is a flag to indicate the end of the integer.
+**
+** This function, SQLAR, and ZIP all use the same "deflate" compression
+** algorithm, but each is subtly different:
+**
+**   *  ZIP uses raw deflate.
+**
+**   *  SQLAR uses the "zlib format" which is raw deflate with a two-byte
+**      algorithm-identification header and a four-byte checksum at the end.
+**
+**   *  This utility uses the "zlib format" like SQLAR, but adds the variable-
+**      length integer uncompressed size value at the beginning.
+**
+** This function might be extended in the future to support compression
+** formats other than deflate, by providing a different algorithm-id
+** mark following the variable-length integer size parameter.
 */
 static void compressFunc(
   sqlite3_context *context,

ext/misc/csv.c

@@ -19,9 +19,9 @@
 **    CREATE VIRTUAL TABLE temp.csv USING csv(filename=FILENAME);
 **    SELECT * FROM csv;
 **
-** The columns are named "c1", "c2", "c3", ... by default.  But the
+** The columns are named "c1", "c2", "c3", ... by default.  Or the
-** application can define its own CREATE TABLE statement as an additional
+** application can define its own CREATE TABLE statement using the
-** parameter.  For example:
+** schema= parameter, like this:
 **
 **    CREATE VIRTUAL TABLE temp.csv2 USING csv(
 **       filename = "../http.log",
@@ -32,9 +32,9 @@
 ** the data= parameter.
 **
 ** If the columns=N parameter is supplied, then the CSV file is assumed to have
-** N columns.  If the columns parameter is omitted, the CSV file is opened
+** N columns.  If both the columns= and schema= parameters are omitted, then
-** as soon as the virtual table is constructed and the first row of the CSV
+** the number and names of the columns is determined by the first line of
-** is read in order to count the tables.
+** the CSV input.
 **
 ** Some extra debugging features (used for testing virtual tables) are available
 ** if this module is compiled with -DSQLITE_TEST.
@@ -132,6 +132,7 @@ static int csv_reader_open(
     }
     p->in = fopen(zFilename, "rb");
     if( p->in==0 ){
+      sqlite3_free(p->zIn);
       csv_reader_reset(p);
       csv_errmsg(p, "cannot open '%s' for reading", zFilename);
       return 1;
@@ -204,7 +205,8 @@ static int csv_append(CsvReader *p, char c){
 **   +  Store the character that terminates the field in p->cTerm.  Store
 **      EOF on end-of-file.
 **
-** Return "" at EOF.  Return 0 on an OOM error.
+** Return 0 at EOF or on OOM.  On EOF, the p->cTerm character will have
+** been set to EOF.
 */
 static char *csv_read_one_field(CsvReader *p){
   int c;
@@ -212,7 +214,7 @@ static char *csv_read_one_field(CsvReader *p){
   c = csv_getc(p);
   if( c==EOF ){
     p->cTerm = EOF;
-    return "";
+    return 0;
   }
   if( c=='"' ){
     int pc, ppc;
@@ -434,6 +436,34 @@ static int csv_boolean(const char *z){
   return -1;
 }
 
+/* Check to see if the string is of the form:  "TAG = BOOLEAN" or just "TAG".
+** If it is, set *pValue to be the value of the boolean ("true" if there is
+** not "= BOOLEAN" component) and return non-zero.  If the input string
+** does not begin with TAG, return zero.
+*/
+static int csv_boolean_parameter(
+  const char *zTag,       /* Tag we are looking for */
+  int nTag,               /* Size of the tag in bytes */
+  const char *z,          /* Input parameter */
+  int *pValue             /* Write boolean value here */
+){
+  int b;
+  z = csv_skip_whitespace(z);
+  if( strncmp(zTag, z, nTag)!=0 ) return 0;
+  z = csv_skip_whitespace(z + nTag);
+  if( z[0]==0 ){
+    *pValue = 1;
+    return 1;
+  }
+  if( z[0]!='=' ) return 0;
+  z = csv_skip_whitespace(z+1);
+  b = csv_boolean(z);
+  if( b>=0 ){
+    *pValue = b;
+    return 1;
+  }
+  return 0;
+}
 
 /*
 ** Parameters:
@@ -467,6 +497,7 @@ static int csvtabConnect(
 #ifdef SQLITE_TEST
   int tstFlags = 0;          /* Value for testflags=N parameter */
 #endif
+  int b;                     /* Value of a boolean parameter */
   int nCol = -99;            /* Value of the columns= parameter */
   CsvReader sRdr;            /* A CSV file reader used to store an error
                              ** message and/or to count the number of columns */
@@ -491,21 +522,12 @@ static int csvtabConnect(
     if( j<sizeof(azParam)/sizeof(azParam[0]) ){
       if( sRdr.zErr[0] ) goto csvtab_connect_error;
     }else
-    if( (zValue = csv_parameter("header",6,z))!=0 ){
+    if( csv_boolean_parameter("header",6,z,&b) ){
-      int x;
       if( bHeader>=0 ){
         csv_errmsg(&sRdr, "more than one 'header' parameter");
         goto csvtab_connect_error;
       }
-      x = csv_boolean(zValue);
+      bHeader = b;
-      if( x==1 ){
-        bHeader = 1;
-      }else if( x==0 ){
-        bHeader = 0;
-      }else{
-        csv_errmsg(&sRdr, "unrecognized argument to 'header': %s", zValue);
-        goto csvtab_connect_error;
-      }
     }else
 #ifdef SQLITE_TEST
     if( (zValue = csv_parameter("testflags",9,z))!=0 ){
@@ -519,54 +541,94 @@ static int csvtabConnect(
       }
       nCol = atoi(zValue);
       if( nCol<=0 ){
-        csv_errmsg(&sRdr, "must have at least one column");
+        csv_errmsg(&sRdr, "column= value must be positive");
         goto csvtab_connect_error;
       }
     }else
     {
-      csv_errmsg(&sRdr, "unrecognized parameter '%s'", z);
+      csv_errmsg(&sRdr, "bad parameter: '%s'", z);
       goto csvtab_connect_error;
     }
   }
   if( (CSV_FILENAME==0)==(CSV_DATA==0) ){
-    csv_errmsg(&sRdr, "must either filename= or data= but not both");
+    csv_errmsg(&sRdr, "must specify either filename= or data= but not both");
     goto csvtab_connect_error;
   }
-  if( nCol<=0 && csv_reader_open(&sRdr, CSV_FILENAME, CSV_DATA) ){
+
+  if( (nCol<=0 || bHeader==1)
+   && csv_reader_open(&sRdr, CSV_FILENAME, CSV_DATA)
+  ){
     goto csvtab_connect_error;
   }
   pNew = sqlite3_malloc( sizeof(*pNew) );
   *ppVtab = (sqlite3_vtab*)pNew;
   if( pNew==0 ) goto csvtab_connect_oom;
   memset(pNew, 0, sizeof(*pNew));
-  if( nCol>0 ){
+  if( CSV_SCHEMA==0 ){
+    sqlite3_str *pStr = sqlite3_str_new(0);
+    char *zSep = "";
+    int iCol = 0;
+    sqlite3_str_appendf(pStr, "CREATE TABLE x(");
+    if( nCol<0 && bHeader<1 ){
+      nCol = 0;
+      do{
+        csv_read_one_field(&sRdr);
+        nCol++;
+      }while( sRdr.cTerm==',' );
+    }
+    if( nCol>0 && bHeader<1 ){
+      for(iCol=0; iCol<nCol; iCol++){
+        sqlite3_str_appendf(pStr, "%sc%d TEXT", zSep, iCol);
+        zSep = ",";
+      }
+    }else{
+      do{
+        char *z = csv_read_one_field(&sRdr);
+        if( (nCol>0 && iCol<nCol) || (nCol<0 && bHeader) ){
+          sqlite3_str_appendf(pStr,"%s\"%w\" TEXT", zSep, z);
+          zSep = ",";
+          iCol++;
+        }
+      }while( sRdr.cTerm==',' );
+      if( nCol<0 ){
+        nCol = iCol;
+      }else{
+        while( iCol<nCol ){
+          sqlite3_str_appendf(pStr,"%sc%d TEXT", zSep, ++iCol);
+          zSep = ",";
+        }
+      }
+    }
     pNew->nCol = nCol;
-  }else{
+    sqlite3_str_appendf(pStr, ")");
+    CSV_SCHEMA = sqlite3_str_finish(pStr);
+    if( CSV_SCHEMA==0 ) goto csvtab_connect_oom;
+  }else if( nCol<0 ){
     do{
-      const char *z = csv_read_one_field(&sRdr);
+      csv_read_one_field(&sRdr);
-      if( z==0 ) goto csvtab_connect_oom;
       pNew->nCol++;
     }while( sRdr.cTerm==',' );
+  }else{
+    pNew->nCol = nCol;
   }
   pNew->zFilename = CSV_FILENAME;  CSV_FILENAME = 0;
   pNew->zData = CSV_DATA;          CSV_DATA = 0;
 #ifdef SQLITE_TEST
   pNew->tstFlags = tstFlags;
 #endif
-  pNew->iStart = bHeader==1 ? ftell(sRdr.in) : 0;
+  if( bHeader!=1 ){
-  csv_reader_reset(&sRdr);
+    pNew->iStart = 0;
-  if( CSV_SCHEMA==0 ){
+  }else if( pNew->zData ){
-    char *zSep = "";
+    pNew->iStart = (int)sRdr.iIn;
-    CSV_SCHEMA = sqlite3_mprintf("CREATE TABLE x(");
+  }else{
-    if( CSV_SCHEMA==0 ) goto csvtab_connect_oom;
+    pNew->iStart = ftell(sRdr.in);
-    for(i=0; i<pNew->nCol; i++){
-      CSV_SCHEMA = sqlite3_mprintf("%z%sc%d TEXT",CSV_SCHEMA, zSep, i);
-      zSep = ",";
-    }
-    CSV_SCHEMA = sqlite3_mprintf("%z);", CSV_SCHEMA);
   }
+  csv_reader_reset(&sRdr);
   rc = sqlite3_declare_vtab(db, CSV_SCHEMA);
-  if( rc ) goto csvtab_connect_error;
+  if( rc ){
+    csv_errmsg(&sRdr, "bad schema: '%s' - %s", CSV_SCHEMA, sqlite3_errmsg(db));
+    goto csvtab_connect_error;
+  }
   for(i=0; i<sizeof(azPValue)/sizeof(azPValue[0]); i++){
     sqlite3_free(azPValue[i]);
   }
@@ -662,7 +724,6 @@ static int csvtabNext(sqlite3_vtab_cursor *cur){
   do{
     z = csv_read_one_field(&pCur->rdr);
     if( z==0 ){
-      csv_xfer_error(pTab, &pCur->rdr);
       break;
     }
     if( i<pTab->nCol ){

ext/misc/dbdump.c

@@ -141,45 +141,12 @@ static void appendText(DText *p, char const *zAppend, char quote){
 ** Return '"' if quoting is required.  Return 0 if no quoting is required.
 */
 static char quoteChar(const char *zName){
-  /* All SQLite keywords, in alphabetical order */
+  int i;
-  static const char *azKeywords[] = {
-    "ABORT", "ACTION", "ADD", "AFTER", "ALL", "ALTER", "ANALYZE", "AND", "AS",
-    "ASC", "ATTACH", "AUTOINCREMENT", "BEFORE", "BEGIN", "BETWEEN", "BY",
-    "CASCADE", "CASE", "CAST", "CHECK", "COLLATE", "COLUMN", "COMMIT",
-    "CONFLICT", "CONSTRAINT", "CREATE", "CROSS", "CURRENT_DATE",
-    "CURRENT_TIME", "CURRENT_TIMESTAMP", "DATABASE", "DEFAULT", "DEFERRABLE",
-    "DEFERRED", "DELETE", "DESC", "DETACH", "DISTINCT", "DROP", "EACH",
-    "ELSE", "END", "ESCAPE", "EXCEPT", "EXCLUSIVE", "EXISTS", "EXPLAIN",
-    "FAIL", "FOR", "FOREIGN", "FROM", "FULL", "GLOB", "GROUP", "HAVING", "IF",
-    "IGNORE", "IMMEDIATE", "IN", "INDEX", "INDEXED", "INITIALLY", "INNER",
-    "INSERT", "INSTEAD", "INTERSECT", "INTO", "IS", "ISNULL", "JOIN", "KEY",
-    "LEFT", "LIKE", "LIMIT", "MATCH", "NATURAL", "NO", "NOT", "NOTNULL",
-    "NULL", "OF", "OFFSET", "ON", "OR", "ORDER", "OUTER", "PLAN", "PRAGMA",
-    "PRIMARY", "QUERY", "RAISE", "RECURSIVE", "REFERENCES", "REGEXP",
-    "REINDEX", "RELEASE", "RENAME", "REPLACE", "RESTRICT", "RIGHT",
-    "ROLLBACK", "ROW", "SAVEPOINT", "SELECT", "SET", "TABLE", "TEMP",
-    "TEMPORARY", "THEN", "TO", "TRANSACTION", "TRIGGER", "UNION", "UNIQUE",
-    "UPDATE", "USING", "VACUUM", "VALUES", "VIEW", "VIRTUAL", "WHEN", "WHERE",
-    "WITH", "WITHOUT",
-  };
-  int i, lwr, upr, mid, c;
   if( !isalpha((unsigned char)zName[0]) && zName[0]!='_' ) return '"';
   for(i=0; zName[i]; i++){
     if( !isalnum((unsigned char)zName[i]) && zName[i]!='_' ) return '"';
   }
-  lwr = 0;
+  return sqlite3_keyword_check(zName, i) ? '"' : 0;
-  upr = sizeof(azKeywords)/sizeof(azKeywords[0]) - 1;
-  while( lwr<=upr ){
-    mid = (lwr+upr)/2;
-    c = sqlite3_stricmp(azKeywords[mid], zName);
-    if( c==0 ) return '"';
-    if( c<0 ){
-      lwr = mid+1;
-    }else{
-      upr = mid-1;
-    }
-  }
-  return 0;
 }
 
 
@@ -293,7 +260,6 @@ static char **tableColumnList(DState *p, const char *zTab){
         ** ordinary column in the table.  Verify that azRowid[j] is a valid
         ** name for the rowid before adding it to azCol[0].  WITHOUT ROWID
         ** tables will fail this last check */
-        int rc;
         rc = sqlite3_table_column_metadata(p->db,0,zTab,azRowid[j],0,0,0,0,0);
         if( rc==SQLITE_OK ) azCol[0] = azRowid[j];
         break;
@@ -455,12 +421,12 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){
   if( strcmp(zType, "table")==0 ){
     DText sSelect;
     DText sTable;
-    char **azCol;
+    char **azTCol;
     int i;
     int nCol;
 
-    azCol = tableColumnList(p, zTable);
+    azTCol = tableColumnList(p, zTable);
-    if( azCol==0 ) return 0;
+    if( azTCol==0 ) return 0;
 
     initText(&sTable);
     appendText(&sTable, "INSERT INTO ", 0);
@@ -473,12 +439,12 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){
     ** In other words:  "INSERT INTO tab(rowid,a,b,c,...) VALUES(...)"
     ** instead of the usual "INSERT INTO tab VALUES(...)".
     */
-    if( azCol[0] ){
+    if( azTCol[0] ){
       appendText(&sTable, "(", 0);
-      appendText(&sTable, azCol[0], 0);
+      appendText(&sTable, azTCol[0], 0);
-      for(i=1; azCol[i]; i++){
+      for(i=1; azTCol[i]; i++){
         appendText(&sTable, ",", 0);
-        appendText(&sTable, azCol[i], quoteChar(azCol[i]));
+        appendText(&sTable, azTCol[i], quoteChar(azTCol[i]));
       }
       appendText(&sTable, ")", 0);
     }
@@ -487,19 +453,19 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){
     /* Build an appropriate SELECT statement */
     initText(&sSelect);
     appendText(&sSelect, "SELECT ", 0);
-    if( azCol[0] ){
+    if( azTCol[0] ){
-      appendText(&sSelect, azCol[0], 0);
+      appendText(&sSelect, azTCol[0], 0);
       appendText(&sSelect, ",", 0);
     }
-    for(i=1; azCol[i]; i++){
+    for(i=1; azTCol[i]; i++){
-      appendText(&sSelect, azCol[i], quoteChar(azCol[i]));
+      appendText(&sSelect, azTCol[i], quoteChar(azTCol[i]));
-      if( azCol[i+1] ){
+      if( azTCol[i+1] ){
         appendText(&sSelect, ",", 0);
       }
     }
     nCol = i;
-    if( azCol[0]==0 ) nCol--;
+    if( azTCol[0]==0 ) nCol--;
-    freeColumnList(azCol);
+    freeColumnList(azTCol);
     appendText(&sSelect, " FROM ", 0);
     appendText(&sSelect, zTable, quoteChar(zTable));
 
@@ -519,7 +485,15 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){
             }
             case SQLITE_FLOAT: {
               double r = sqlite3_column_double(pStmt,i);
-              output_formatted(p, "%!.20g", r);
+              sqlite3_uint64 ur;
+              memcpy(&ur,&r,sizeof(r));
+              if( ur==0x7ff0000000000000LL ){
+                p->xCallback("1e999", p->pArg);
+              }else if( ur==0xfff0000000000000LL ){
+                p->xCallback("-1e999", p->pArg);
+              }else{
+                output_formatted(p, "%!.20g", r);
+              }
               break;
             }
             case SQLITE_NULL: {

ext/misc/eval.c

@@ -34,6 +34,7 @@ struct EvalResult {
 static int callback(void *pCtx, int argc, char **argv, char **colnames){
   struct EvalResult *p = (struct EvalResult*)pCtx;
   int i; 
+  if( argv==0 ) return 0;
   for(i=0; i<argc; i++){
     const char *z = argv[i] ? argv[i] : "";
     size_t sz = strlen(z);

ext/misc/explain.c

@@ -0,0 +1,322 @@
+/*
+** 2018-09-16
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** This file demonstrates an eponymous virtual table that returns the
+** EXPLAIN output from an SQL statement.
+**
+** Usage example:
+**
+**     .load ./explain
+**     SELECT p2 FROM explain('SELECT * FROM sqlite_master')
+**      WHERE opcode='OpenRead';
+**
+** This module was originally written to help simplify SQLite testing,
+** by providing an easier means of verifying certain patterns in the
+** generated bytecode.
+*/
+#if !defined(SQLITEINT_H)
+#include "sqlite3ext.h"
+#endif
+SQLITE_EXTENSION_INIT1
+#include <assert.h>
+#include <string.h>
+
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+
+/* explain_vtab is a subclass of sqlite3_vtab which will
+** serve as the underlying representation of a explain virtual table
+*/
+typedef struct explain_vtab explain_vtab;
+struct explain_vtab {
+  sqlite3_vtab base;  /* Base class - must be first */
+  sqlite3 *db;        /* Database connection for this explain vtab */
+};
+
+/* explain_cursor is a subclass of sqlite3_vtab_cursor which will
+** serve as the underlying representation of a cursor that scans
+** over rows of the result from an EXPLAIN operation.
+*/
+typedef struct explain_cursor explain_cursor;
+struct explain_cursor {
+  sqlite3_vtab_cursor base;  /* Base class - must be first */
+  sqlite3 *db;               /* Database connection for this cursor */
+  char *zSql;                /* Value for the EXPLN_COLUMN_SQL column */
+  sqlite3_stmt *pExplain;    /* Statement being explained */
+  int rc;                    /* Result of last sqlite3_step() on pExplain */
+};
+
+/*
+** The explainConnect() method is invoked to create a new
+** explain_vtab that describes the explain virtual table.
+**
+** Think of this routine as the constructor for explain_vtab objects.
+**
+** All this routine needs to do is:
+**
+**    (1) Allocate the explain_vtab object and initialize all fields.
+**
+**    (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the
+**        result set of queries against explain will look like.
+*/
+static int explainConnect(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
+){
+  explain_vtab *pNew;
+  int rc;
+
+/* Column numbers */
+#define EXPLN_COLUMN_ADDR     0   /* Instruction address */
+#define EXPLN_COLUMN_OPCODE   1   /* Opcode */
+#define EXPLN_COLUMN_P1       2   /* Operand 1 */
+#define EXPLN_COLUMN_P2       3   /* Operand 2 */
+#define EXPLN_COLUMN_P3       4   /* Operand 3 */
+#define EXPLN_COLUMN_P4       5   /* Operand 4 */
+#define EXPLN_COLUMN_P5       6   /* Operand 5 */
+#define EXPLN_COLUMN_COMMENT  7   /* Comment */
+#define EXPLN_COLUMN_SQL      8   /* SQL that is being explained */
+
+
+  rc = sqlite3_declare_vtab(db,
+     "CREATE TABLE x(addr,opcode,p1,p2,p3,p4,p5,comment,sql HIDDEN)");
+  if( rc==SQLITE_OK ){
+    pNew = sqlite3_malloc( sizeof(*pNew) );
+    *ppVtab = (sqlite3_vtab*)pNew;
+    if( pNew==0 ) return SQLITE_NOMEM;
+    memset(pNew, 0, sizeof(*pNew));
+    pNew->db = db;
+  }
+  return rc;
+}
+
+/*
+** This method is the destructor for explain_cursor objects.
+*/
+static int explainDisconnect(sqlite3_vtab *pVtab){
+  sqlite3_free(pVtab);
+  return SQLITE_OK;
+}
+
+/*
+** Constructor for a new explain_cursor object.
+*/
+static int explainOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
+  explain_cursor *pCur;
+  pCur = sqlite3_malloc( sizeof(*pCur) );
+  if( pCur==0 ) return SQLITE_NOMEM;
+  memset(pCur, 0, sizeof(*pCur));
+  pCur->db = ((explain_vtab*)p)->db;
+  *ppCursor = &pCur->base;
+  return SQLITE_OK;
+}
+
+/*
+** Destructor for a explain_cursor.
+*/
+static int explainClose(sqlite3_vtab_cursor *cur){
+  explain_cursor *pCur = (explain_cursor*)cur;
+  sqlite3_finalize(pCur->pExplain);
+  sqlite3_free(pCur->zSql);
+  sqlite3_free(pCur);
+  return SQLITE_OK;
+}
+
+
+/*
+** Advance a explain_cursor to its next row of output.
+*/
+static int explainNext(sqlite3_vtab_cursor *cur){
+  explain_cursor *pCur = (explain_cursor*)cur;
+  pCur->rc = sqlite3_step(pCur->pExplain);
+  if( pCur->rc!=SQLITE_DONE && pCur->rc!=SQLITE_ROW ) return pCur->rc;
+  return SQLITE_OK;
+}
+
+/*
+** Return values of columns for the row at which the explain_cursor
+** is currently pointing.
+*/
+static int explainColumn(
+  sqlite3_vtab_cursor *cur,   /* The cursor */
+  sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
+  int i                       /* Which column to return */
+){
+  explain_cursor *pCur = (explain_cursor*)cur;
+  if( i==EXPLN_COLUMN_SQL ){
+    sqlite3_result_text(ctx, pCur->zSql, -1, SQLITE_TRANSIENT);
+  }else{
+    sqlite3_result_value(ctx, sqlite3_column_value(pCur->pExplain, i));
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Return the rowid for the current row.  In this implementation, the
+** rowid is the same as the output value.
+*/
+static int explainRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
+  explain_cursor *pCur = (explain_cursor*)cur;
+  *pRowid = sqlite3_column_int64(pCur->pExplain, 0);
+  return SQLITE_OK;
+}
+
+/*
+** Return TRUE if the cursor has been moved off of the last
+** row of output.
+*/
+static int explainEof(sqlite3_vtab_cursor *cur){
+  explain_cursor *pCur = (explain_cursor*)cur;
+  return pCur->rc!=SQLITE_ROW;
+}
+
+/*
+** This method is called to "rewind" the explain_cursor object back
+** to the first row of output.  This method is always called at least
+** once prior to any call to explainColumn() or explainRowid() or 
+** explainEof().
+**
+** The argv[0] is the SQL statement that is to be explained.
+*/
+static int explainFilter(
+  sqlite3_vtab_cursor *pVtabCursor, 
+  int idxNum, const char *idxStr,
+  int argc, sqlite3_value **argv
+){
+  explain_cursor *pCur = (explain_cursor *)pVtabCursor;
+  char *zSql = 0;
+  int rc;
+  sqlite3_finalize(pCur->pExplain);
+  pCur->pExplain = 0;
+  if( sqlite3_value_type(argv[0])!=SQLITE_TEXT ){
+    pCur->rc = SQLITE_DONE;
+    return SQLITE_OK;
+  }
+  sqlite3_free(pCur->zSql);
+  pCur->zSql = sqlite3_mprintf("%s", sqlite3_value_text(argv[0]));
+  if( pCur->zSql ){
+    zSql = sqlite3_mprintf("EXPLAIN %s", pCur->zSql);
+  }
+  if( zSql==0 ){
+    rc = SQLITE_NOMEM;
+  }else{
+    rc = sqlite3_prepare_v2(pCur->db, zSql, -1, &pCur->pExplain, 0);
+    sqlite3_free(zSql);
+  }
+  if( rc ){
+    sqlite3_finalize(pCur->pExplain);
+    pCur->pExplain = 0;
+    sqlite3_free(pCur->zSql);
+    pCur->zSql = 0;
+  }else{
+    pCur->rc = sqlite3_step(pCur->pExplain);
+    rc = (pCur->rc==SQLITE_DONE || pCur->rc==SQLITE_ROW) ? SQLITE_OK : pCur->rc;
+  }
+  return rc;
+}
+
+/*
+** SQLite will invoke this method one or more times while planning a query
+** that uses the explain virtual table.  This routine needs to create
+** a query plan for each invocation and compute an estimated cost for that
+** plan.
+*/
+static int explainBestIndex(
+  sqlite3_vtab *tab,
+  sqlite3_index_info *pIdxInfo
+){
+  int i;                  /* Loop counter */
+  int idx = -1;           /* Index of a usable == constraint against SQL */
+  int unusable = 0;       /* True if there are unusable constraints on SQL */
+
+  pIdxInfo->estimatedRows = 500;
+  for(i=0; i<pIdxInfo->nConstraint; i++){
+    struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[i];
+    if( p->iColumn!=EXPLN_COLUMN_SQL ) continue;
+    if( !p->usable ){
+      unusable = 1;
+    }else if( p->op==SQLITE_INDEX_CONSTRAINT_EQ ){
+      idx = i;
+    }
+  }
+  if( idx>=0 ){
+    /* There exists a usable == constraint against the SQL column */
+    pIdxInfo->estimatedCost = 10.0;
+    pIdxInfo->idxNum = 1;
+    pIdxInfo->aConstraintUsage[idx].argvIndex = 1;
+    pIdxInfo->aConstraintUsage[idx].omit = 1;
+  }else if( unusable ){
+    /* There are unusable constraints against the SQL column.  Do not allow
+    ** this plan to continue forward. */
+    return SQLITE_CONSTRAINT;
+  }
+  return SQLITE_OK;
+}
+
+/*
+** This following structure defines all the methods for the 
+** explain virtual table.
+*/
+static sqlite3_module explainModule = {
+  0,                         /* iVersion */
+  0,                         /* xCreate */
+  explainConnect,            /* xConnect */
+  explainBestIndex,          /* xBestIndex */
+  explainDisconnect,         /* xDisconnect */
+  0,                         /* xDestroy */
+  explainOpen,               /* xOpen - open a cursor */
+  explainClose,              /* xClose - close a cursor */
+  explainFilter,             /* xFilter - configure scan constraints */
+  explainNext,               /* xNext - advance a cursor */
+  explainEof,                /* xEof - check for end of scan */
+  explainColumn,             /* xColumn - read data */
+  explainRowid,              /* xRowid - read data */
+  0,                         /* xUpdate */
+  0,                         /* xBegin */
+  0,                         /* xSync */
+  0,                         /* xCommit */
+  0,                         /* xRollback */
+  0,                         /* xFindMethod */
+  0,                         /* xRename */
+  0,                         /* xSavepoint */
+  0,                         /* xRelease */
+  0,                         /* xRollbackTo */
+  0,                         /* xShadowName */
+};
+
+#endif /* SQLITE_OMIT_VIRTUALTABLE */
+
+int sqlite3ExplainVtabInit(sqlite3 *db){
+  int rc = SQLITE_OK;
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+  rc = sqlite3_create_module(db, "explain", &explainModule, 0);
+#endif
+  return rc;
+}
+
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_explain_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  int rc = SQLITE_OK;
+  SQLITE_EXTENSION_INIT2(pApi);
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+  rc = sqlite3ExplainVtabInit(db);
+#endif
+  return rc;
+}

ext/misc/fileio.c

@@ -11,11 +11,136 @@
 ******************************************************************************
 **
 ** This SQLite extension implements SQL functions readfile() and
-** writefile().
+** writefile(), and eponymous virtual type "fsdir".
+**
+** WRITEFILE(FILE, DATA [, MODE [, MTIME]]):
+**
+**   If neither of the optional arguments is present, then this UDF
+**   function writes blob DATA to file FILE. If successful, the number
+**   of bytes written is returned. If an error occurs, NULL is returned.
+**
+**   If the first option argument - MODE - is present, then it must
+**   be passed an integer value that corresponds to a POSIX mode
+**   value (file type + permissions, as returned in the stat.st_mode
+**   field by the stat() system call). Three types of files may
+**   be written/created:
+**
+**     regular files:  (mode & 0170000)==0100000
+**     symbolic links: (mode & 0170000)==0120000
+**     directories:    (mode & 0170000)==0040000
+**
+**   For a directory, the DATA is ignored. For a symbolic link, it is
+**   interpreted as text and used as the target of the link. For a
+**   regular file, it is interpreted as a blob and written into the
+**   named file. Regardless of the type of file, its permissions are
+**   set to (mode & 0777) before returning.
+**
+**   If the optional MTIME argument is present, then it is interpreted
+**   as an integer - the number of seconds since the unix epoch. The
+**   modification-time of the target file is set to this value before
+**   returning.
+**
+**   If three or more arguments are passed to this function and an
+**   error is encountered, an exception is raised.
+**
+** READFILE(FILE):
+**
+**   Read and return the contents of file FILE (type blob) from disk.
+**
+** FSDIR:
+**
+**   Used as follows:
+**
+**     SELECT * FROM fsdir($path [, $dir]);
+**
+**   Parameter $path is an absolute or relative pathname. If the file that it
+**   refers to does not exist, it is an error. If the path refers to a regular
+**   file or symbolic link, it returns a single row. Or, if the path refers
+**   to a directory, it returns one row for the directory, and one row for each
+**   file within the hierarchy rooted at $path.
+**
+**   Each row has the following columns:
+**
+**     name:  Path to file or directory (text value).
+**     mode:  Value of stat.st_mode for directory entry (an integer).
+**     mtime: Value of stat.st_mtime for directory entry (an integer).
+**     data:  For a regular file, a blob containing the file data. For a
+**            symlink, a text value containing the text of the link. For a
+**            directory, NULL.
+**
+**   If a non-NULL value is specified for the optional $dir parameter and
+**   $path is a relative path, then $path is interpreted relative to $dir. 
+**   And the paths returned in the "name" column of the table are also 
+**   relative to directory $dir.
 */
 #include "sqlite3ext.h"
 SQLITE_EXTENSION_INIT1
 #include <stdio.h>
+#include <string.h>
+#include <assert.h>
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#if !defined(_WIN32) && !defined(WIN32)
+#  include <unistd.h>
+#  include <dirent.h>
+#  include <utime.h>
+#  include <sys/time.h>
+#else
+#  include "windows.h"
+#  include <io.h>
+#  include <direct.h>
+#  include "test_windirent.h"
+#  define dirent DIRENT
+#  ifndef chmod
+#    define chmod _chmod
+#  endif
+#  ifndef stat
+#    define stat _stat
+#  endif
+#  define mkdir(path,mode) _mkdir(path)
+#  define lstat(path,buf) stat(path,buf)
+#endif
+#include <time.h>
+#include <errno.h>
+
+
+/*
+** Structure of the fsdir() table-valued function
+*/
+                 /*    0    1    2     3    4           5             */
+#define FSDIR_SCHEMA "(name,mode,mtime,data,path HIDDEN,dir HIDDEN)"
+#define FSDIR_COLUMN_NAME     0     /* Name of the file */
+#define FSDIR_COLUMN_MODE     1     /* Access mode */
+#define FSDIR_COLUMN_MTIME    2     /* Last modification time */
+#define FSDIR_COLUMN_DATA     3     /* File content */
+#define FSDIR_COLUMN_PATH     4     /* Path to top of search */
+#define FSDIR_COLUMN_DIR      5     /* Path is relative to this directory */
+
+
+/*
+** Set the result stored by context ctx to a blob containing the 
+** contents of file zName.
+*/
+static void readFileContents(sqlite3_context *ctx, const char *zName){
+  FILE *in;
+  long nIn;
+  void *pBuf;
+
+  in = fopen(zName, "rb");
+  if( in==0 ) return;
+  fseek(in, 0, SEEK_END);
+  nIn = ftell(in);
+  rewind(in);
+  pBuf = sqlite3_malloc( nIn );
+  if( pBuf && 1==fread(pBuf, nIn, 1, in) ){
+    sqlite3_result_blob(ctx, pBuf, nIn, sqlite3_free);
+  }else{
+    sqlite3_free(pBuf);
+  }
+  fclose(in);
+}
 
 /*
 ** Implementation of the "readfile(X)" SQL function.  The entire content
@@ -28,58 +153,794 @@ static void readfileFunc(
   sqlite3_value **argv
 ){
   const char *zName;
-  FILE *in;
-  long nIn;
-  void *pBuf;
-
   (void)(argc);  /* Unused parameter */
   zName = (const char*)sqlite3_value_text(argv[0]);
   if( zName==0 ) return;
-  in = fopen(zName, "rb");
+  readFileContents(context, zName);
-  if( in==0 ) return;
-  fseek(in, 0, SEEK_END);
-  nIn = ftell(in);
-  rewind(in);
-  pBuf = sqlite3_malloc( nIn );
-  if( pBuf && 1==fread(pBuf, nIn, 1, in) ){
-    sqlite3_result_blob(context, pBuf, nIn, sqlite3_free);
-  }else{
-    sqlite3_free(pBuf);
-  }
-  fclose(in);
 }
 
 /*
-** Implementation of the "writefile(X,Y)" SQL function.  The argument Y
+** Set the error message contained in context ctx to the results of
-** is written into file X.  The number of bytes written is returned.  Or
+** vprintf(zFmt, ...).
-** NULL is returned if something goes wrong, such as being unable to open
+*/
-** file X for writing.
+static void ctxErrorMsg(sqlite3_context *ctx, const char *zFmt, ...){
+  char *zMsg = 0;
+  va_list ap;
+  va_start(ap, zFmt);
+  zMsg = sqlite3_vmprintf(zFmt, ap);
+  sqlite3_result_error(ctx, zMsg, -1);
+  sqlite3_free(zMsg);
+  va_end(ap);
+}
+
+#if defined(_WIN32)
+/*
+** This function is designed to convert a Win32 FILETIME structure into the
+** number of seconds since the Unix Epoch (1970-01-01 00:00:00 UTC).
+*/
+static sqlite3_uint64 fileTimeToUnixTime(
+  LPFILETIME pFileTime
+){
+  SYSTEMTIME epochSystemTime;
+  ULARGE_INTEGER epochIntervals;
+  FILETIME epochFileTime;
+  ULARGE_INTEGER fileIntervals;
+
+  memset(&epochSystemTime, 0, sizeof(SYSTEMTIME));
+  epochSystemTime.wYear = 1970;
+  epochSystemTime.wMonth = 1;
+  epochSystemTime.wDay = 1;
+  SystemTimeToFileTime(&epochSystemTime, &epochFileTime);
+  epochIntervals.LowPart = epochFileTime.dwLowDateTime;
+  epochIntervals.HighPart = epochFileTime.dwHighDateTime;
+
+  fileIntervals.LowPart = pFileTime->dwLowDateTime;
+  fileIntervals.HighPart = pFileTime->dwHighDateTime;
+
+  return (fileIntervals.QuadPart - epochIntervals.QuadPart) / 10000000;
+}
+
+/*
+** This function attempts to normalize the time values found in the stat()
+** buffer to UTC.  This is necessary on Win32, where the runtime library
+** appears to return these values as local times.
+*/
+static void statTimesToUtc(
+  const char *zPath,
+  struct stat *pStatBuf
+){
+  HANDLE hFindFile;
+  WIN32_FIND_DATAW fd;
+  LPWSTR zUnicodeName;
+  extern LPWSTR sqlite3_win32_utf8_to_unicode(const char*);
+  zUnicodeName = sqlite3_win32_utf8_to_unicode(zPath);
+  if( zUnicodeName ){
+    memset(&fd, 0, sizeof(WIN32_FIND_DATAW));
+    hFindFile = FindFirstFileW(zUnicodeName, &fd);
+    if( hFindFile!=NULL ){
+      pStatBuf->st_ctime = (time_t)fileTimeToUnixTime(&fd.ftCreationTime);
+      pStatBuf->st_atime = (time_t)fileTimeToUnixTime(&fd.ftLastAccessTime);
+      pStatBuf->st_mtime = (time_t)fileTimeToUnixTime(&fd.ftLastWriteTime);
+      FindClose(hFindFile);
+    }
+    sqlite3_free(zUnicodeName);
+  }
+}
+#endif
+
+/*
+** This function is used in place of stat().  On Windows, special handling
+** is required in order for the included time to be returned as UTC.  On all
+** other systems, this function simply calls stat().
+*/
+static int fileStat(
+  const char *zPath,
+  struct stat *pStatBuf
+){
+#if defined(_WIN32)
+  int rc = stat(zPath, pStatBuf);
+  if( rc==0 ) statTimesToUtc(zPath, pStatBuf);
+  return rc;
+#else
+  return stat(zPath, pStatBuf);
+#endif
+}
+
+/*
+** This function is used in place of lstat().  On Windows, special handling
+** is required in order for the included time to be returned as UTC.  On all
+** other systems, this function simply calls lstat().
+*/
+static int fileLinkStat(
+  const char *zPath,
+  struct stat *pStatBuf
+){
+#if defined(_WIN32)
+  int rc = lstat(zPath, pStatBuf);
+  if( rc==0 ) statTimesToUtc(zPath, pStatBuf);
+  return rc;
+#else
+  return lstat(zPath, pStatBuf);
+#endif
+}
+
+/*
+** Argument zFile is the name of a file that will be created and/or written
+** by SQL function writefile(). This function ensures that the directory
+** zFile will be written to exists, creating it if required. The permissions
+** for any path components created by this function are set to (mode&0777).
+**
+** If an OOM condition is encountered, SQLITE_NOMEM is returned. Otherwise,
+** SQLITE_OK is returned if the directory is successfully created, or
+** SQLITE_ERROR otherwise.
+*/
+static int makeDirectory(
+  const char *zFile,
+  mode_t mode
+){
+  char *zCopy = sqlite3_mprintf("%s", zFile);
+  int rc = SQLITE_OK;
+
+  if( zCopy==0 ){
+    rc = SQLITE_NOMEM;
+  }else{
+    int nCopy = (int)strlen(zCopy);
+    int i = 1;
+
+    while( rc==SQLITE_OK ){
+      struct stat sStat;
+      int rc2;
+
+      for(; zCopy[i]!='/' && i<nCopy; i++);
+      if( i==nCopy ) break;
+      zCopy[i] = '\0';
+
+      rc2 = fileStat(zCopy, &sStat);
+      if( rc2!=0 ){
+        if( mkdir(zCopy, mode & 0777) ) rc = SQLITE_ERROR;
+      }else{
+        if( !S_ISDIR(sStat.st_mode) ) rc = SQLITE_ERROR;
+      }
+      zCopy[i] = '/';
+      i++;
+    }
+
+    sqlite3_free(zCopy);
+  }
+
+  return rc;
+}
+
+/*
+** This function does the work for the writefile() UDF. Refer to 
+** header comments at the top of this file for details.
+*/
+static int writeFile(
+  sqlite3_context *pCtx,          /* Context to return bytes written in */
+  const char *zFile,              /* File to write */
+  sqlite3_value *pData,           /* Data to write */
+  mode_t mode,                    /* MODE parameter passed to writefile() */
+  sqlite3_int64 mtime             /* MTIME parameter (or -1 to not set time) */
+){
+#if !defined(_WIN32) && !defined(WIN32)
+  if( S_ISLNK(mode) ){
+    const char *zTo = (const char*)sqlite3_value_text(pData);
+    if( symlink(zTo, zFile)<0 ) return 1;
+  }else
+#endif
+  {
+    if( S_ISDIR(mode) ){
+      if( mkdir(zFile, mode) ){
+        /* The mkdir() call to create the directory failed. This might not
+        ** be an error though - if there is already a directory at the same
+        ** path and either the permissions already match or can be changed
+        ** to do so using chmod(), it is not an error.  */
+        struct stat sStat;
+        if( errno!=EEXIST
+         || 0!=fileStat(zFile, &sStat)
+         || !S_ISDIR(sStat.st_mode)
+         || ((sStat.st_mode&0777)!=(mode&0777) && 0!=chmod(zFile, mode&0777))
+        ){
+          return 1;
+        }
+      }
+    }else{
+      sqlite3_int64 nWrite = 0;
+      const char *z;
+      int rc = 0;
+      FILE *out = fopen(zFile, "wb");
+      if( out==0 ) return 1;
+      z = (const char*)sqlite3_value_blob(pData);
+      if( z ){
+        sqlite3_int64 n = fwrite(z, 1, sqlite3_value_bytes(pData), out);
+        nWrite = sqlite3_value_bytes(pData);
+        if( nWrite!=n ){
+          rc = 1;
+        }
+      }
+      fclose(out);
+      if( rc==0 && mode && chmod(zFile, mode & 0777) ){
+        rc = 1;
+      }
+      if( rc ) return 2;
+      sqlite3_result_int64(pCtx, nWrite);
+    }
+  }
+
+  if( mtime>=0 ){
+#if defined(_WIN32)
+    /* Windows */
+    FILETIME lastAccess;
+    FILETIME lastWrite;
+    SYSTEMTIME currentTime;
+    LONGLONG intervals;
+    HANDLE hFile;
+    LPWSTR zUnicodeName;
+    extern LPWSTR sqlite3_win32_utf8_to_unicode(const char*);
+
+    GetSystemTime(&currentTime);
+    SystemTimeToFileTime(&currentTime, &lastAccess);
+    intervals = Int32x32To64(mtime, 10000000) + 116444736000000000;
+    lastWrite.dwLowDateTime = (DWORD)intervals;
+    lastWrite.dwHighDateTime = intervals >> 32;
+    zUnicodeName = sqlite3_win32_utf8_to_unicode(zFile);
+    if( zUnicodeName==0 ){
+      return 1;
+    }
+    hFile = CreateFileW(
+      zUnicodeName, FILE_WRITE_ATTRIBUTES, 0, NULL, OPEN_EXISTING,
+      FILE_FLAG_BACKUP_SEMANTICS, NULL
+    );
+    sqlite3_free(zUnicodeName);
+    if( hFile!=INVALID_HANDLE_VALUE ){
+      BOOL bResult = SetFileTime(hFile, NULL, &lastAccess, &lastWrite);
+      CloseHandle(hFile);
+      return !bResult;
+    }else{
+      return 1;
+    }
+#elif defined(AT_FDCWD) && 0 /* utimensat() is not universally available */
+    /* Recent unix */
+    struct timespec times[2];
+    times[0].tv_nsec = times[1].tv_nsec = 0;
+    times[0].tv_sec = time(0);
+    times[1].tv_sec = mtime;
+    if( utimensat(AT_FDCWD, zFile, times, AT_SYMLINK_NOFOLLOW) ){
+      return 1;
+    }
+#else
+    /* Legacy unix */
+    struct timeval times[2];
+    times[0].tv_usec = times[1].tv_usec = 0;
+    times[0].tv_sec = time(0);
+    times[1].tv_sec = mtime;
+    if( utimes(zFile, times) ){
+      return 1;
+    }
+#endif
+  }
+
+  return 0;
+}
+
+/*
+** Implementation of the "writefile(W,X[,Y[,Z]]])" SQL function.  
+** Refer to header comments at the top of this file for details.
 */
 static void writefileFunc(
   sqlite3_context *context,
   int argc,
   sqlite3_value **argv
 ){
-  FILE *out;
-  const char *z;
-  sqlite3_int64 rc;
   const char *zFile;
+  mode_t mode = 0;
+  int res;
+  sqlite3_int64 mtime = -1;
+
+  if( argc<2 || argc>4 ){
+    sqlite3_result_error(context, 
+        "wrong number of arguments to function writefile()", -1
+    );
+    return;
+  }
 
-  (void)(argc);  /* Unused parameter */
   zFile = (const char*)sqlite3_value_text(argv[0]);
   if( zFile==0 ) return;
-  out = fopen(zFile, "wb");
+  if( argc>=3 ){
-  if( out==0 ) return;
+    mode = (mode_t)sqlite3_value_int(argv[2]);
-  z = (const char*)sqlite3_value_blob(argv[1]);
+  }
-  if( z==0 ){
+  if( argc==4 ){
-    rc = 0;
+    mtime = sqlite3_value_int64(argv[3]);
-  }else{
+  }
-    rc = fwrite(z, 1, sqlite3_value_bytes(argv[1]), out);
+
+  res = writeFile(context, zFile, argv[1], mode, mtime);
+  if( res==1 && errno==ENOENT ){
+    if( makeDirectory(zFile, mode)==SQLITE_OK ){
+      res = writeFile(context, zFile, argv[1], mode, mtime);
+    }
+  }
+
+  if( argc>2 && res!=0 ){
+    if( S_ISLNK(mode) ){
+      ctxErrorMsg(context, "failed to create symlink: %s", zFile);
+    }else if( S_ISDIR(mode) ){
+      ctxErrorMsg(context, "failed to create directory: %s", zFile);
+    }else{
+      ctxErrorMsg(context, "failed to write file: %s", zFile);
+    }
   }
-  fclose(out);
-  sqlite3_result_int64(context, rc);
 }
 
+/*
+** SQL function:   lsmode(MODE)
+**
+** Given a numberic st_mode from stat(), convert it into a human-readable
+** text string in the style of "ls -l".
+*/
+static void lsModeFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  int i;
+  int iMode = sqlite3_value_int(argv[0]);
+  char z[16];
+  (void)argc;
+  if( S_ISLNK(iMode) ){
+    z[0] = 'l';
+  }else if( S_ISREG(iMode) ){
+    z[0] = '-';
+  }else if( S_ISDIR(iMode) ){
+    z[0] = 'd';
+  }else{
+    z[0] = '?';
+  }
+  for(i=0; i<3; i++){
+    int m = (iMode >> ((2-i)*3));
+    char *a = &z[1 + i*3];
+    a[0] = (m & 0x4) ? 'r' : '-';
+    a[1] = (m & 0x2) ? 'w' : '-';
+    a[2] = (m & 0x1) ? 'x' : '-';
+  }
+  z[10] = '\0';
+  sqlite3_result_text(context, z, -1, SQLITE_TRANSIENT);
+}
+
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+
+/* 
+** Cursor type for recursively iterating through a directory structure.
+*/
+typedef struct fsdir_cursor fsdir_cursor;
+typedef struct FsdirLevel FsdirLevel;
+
+struct FsdirLevel {
+  DIR *pDir;                 /* From opendir() */
+  char *zDir;                /* Name of directory (nul-terminated) */
+};
+
+struct fsdir_cursor {
+  sqlite3_vtab_cursor base;  /* Base class - must be first */
+
+  int nLvl;                  /* Number of entries in aLvl[] array */
+  int iLvl;                  /* Index of current entry */
+  FsdirLevel *aLvl;          /* Hierarchy of directories being traversed */
+
+  const char *zBase;
+  int nBase;
+
+  struct stat sStat;         /* Current lstat() results */
+  char *zPath;               /* Path to current entry */
+  sqlite3_int64 iRowid;      /* Current rowid */
+};
+
+typedef struct fsdir_tab fsdir_tab;
+struct fsdir_tab {
+  sqlite3_vtab base;         /* Base class - must be first */
+};
+
+/*
+** Construct a new fsdir virtual table object.
+*/
+static int fsdirConnect(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
+){
+  fsdir_tab *pNew = 0;
+  int rc;
+  (void)pAux;
+  (void)argc;
+  (void)argv;
+  (void)pzErr;
+  rc = sqlite3_declare_vtab(db, "CREATE TABLE x" FSDIR_SCHEMA);
+  if( rc==SQLITE_OK ){
+    pNew = (fsdir_tab*)sqlite3_malloc( sizeof(*pNew) );
+    if( pNew==0 ) return SQLITE_NOMEM;
+    memset(pNew, 0, sizeof(*pNew));
+  }
+  *ppVtab = (sqlite3_vtab*)pNew;
+  return rc;
+}
+
+/*
+** This method is the destructor for fsdir vtab objects.
+*/
+static int fsdirDisconnect(sqlite3_vtab *pVtab){
+  sqlite3_free(pVtab);
+  return SQLITE_OK;
+}
+
+/*
+** Constructor for a new fsdir_cursor object.
+*/
+static int fsdirOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
+  fsdir_cursor *pCur;
+  (void)p;
+  pCur = sqlite3_malloc( sizeof(*pCur) );
+  if( pCur==0 ) return SQLITE_NOMEM;
+  memset(pCur, 0, sizeof(*pCur));
+  pCur->iLvl = -1;
+  *ppCursor = &pCur->base;
+  return SQLITE_OK;
+}
+
+/*
+** Reset a cursor back to the state it was in when first returned
+** by fsdirOpen().
+*/
+static void fsdirResetCursor(fsdir_cursor *pCur){
+  int i;
+  for(i=0; i<=pCur->iLvl; i++){
+    FsdirLevel *pLvl = &pCur->aLvl[i];
+    if( pLvl->pDir ) closedir(pLvl->pDir);
+    sqlite3_free(pLvl->zDir);
+  }
+  sqlite3_free(pCur->zPath);
+  sqlite3_free(pCur->aLvl);
+  pCur->aLvl = 0;
+  pCur->zPath = 0;
+  pCur->zBase = 0;
+  pCur->nBase = 0;
+  pCur->nLvl = 0;
+  pCur->iLvl = -1;
+  pCur->iRowid = 1;
+}
+
+/*
+** Destructor for an fsdir_cursor.
+*/
+static int fsdirClose(sqlite3_vtab_cursor *cur){
+  fsdir_cursor *pCur = (fsdir_cursor*)cur;
+
+  fsdirResetCursor(pCur);
+  sqlite3_free(pCur);
+  return SQLITE_OK;
+}
+
+/*
+** Set the error message for the virtual table associated with cursor
+** pCur to the results of vprintf(zFmt, ...).
+*/
+static void fsdirSetErrmsg(fsdir_cursor *pCur, const char *zFmt, ...){
+  va_list ap;
+  va_start(ap, zFmt);
+  pCur->base.pVtab->zErrMsg = sqlite3_vmprintf(zFmt, ap);
+  va_end(ap);
+}
+
+
+/*
+** Advance an fsdir_cursor to its next row of output.
+*/
+static int fsdirNext(sqlite3_vtab_cursor *cur){
+  fsdir_cursor *pCur = (fsdir_cursor*)cur;
+  mode_t m = pCur->sStat.st_mode;
+
+  pCur->iRowid++;
+  if( S_ISDIR(m) ){
+    /* Descend into this directory */
+    int iNew = pCur->iLvl + 1;
+    FsdirLevel *pLvl;
+    if( iNew>=pCur->nLvl ){
+      int nNew = iNew+1;
+      int nByte = nNew*sizeof(FsdirLevel);
+      FsdirLevel *aNew = (FsdirLevel*)sqlite3_realloc(pCur->aLvl, nByte);
+      if( aNew==0 ) return SQLITE_NOMEM;
+      memset(&aNew[pCur->nLvl], 0, sizeof(FsdirLevel)*(nNew-pCur->nLvl));
+      pCur->aLvl = aNew;
+      pCur->nLvl = nNew;
+    }
+    pCur->iLvl = iNew;
+    pLvl = &pCur->aLvl[iNew];
+    
+    pLvl->zDir = pCur->zPath;
+    pCur->zPath = 0;
+    pLvl->pDir = opendir(pLvl->zDir);
+    if( pLvl->pDir==0 ){
+      fsdirSetErrmsg(pCur, "cannot read directory: %s", pCur->zPath);
+      return SQLITE_ERROR;
+    }
+  }
+
+  while( pCur->iLvl>=0 ){
+    FsdirLevel *pLvl = &pCur->aLvl[pCur->iLvl];
+    struct dirent *pEntry = readdir(pLvl->pDir);
+    if( pEntry ){
+      if( pEntry->d_name[0]=='.' ){
+       if( pEntry->d_name[1]=='.' && pEntry->d_name[2]=='\0' ) continue;
+       if( pEntry->d_name[1]=='\0' ) continue;
+      }
+      sqlite3_free(pCur->zPath);
+      pCur->zPath = sqlite3_mprintf("%s/%s", pLvl->zDir, pEntry->d_name);
+      if( pCur->zPath==0 ) return SQLITE_NOMEM;
+      if( fileLinkStat(pCur->zPath, &pCur->sStat) ){
+        fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zPath);
+        return SQLITE_ERROR;
+      }
+      return SQLITE_OK;
+    }
+    closedir(pLvl->pDir);
+    sqlite3_free(pLvl->zDir);
+    pLvl->pDir = 0;
+    pLvl->zDir = 0;
+    pCur->iLvl--;
+  }
+
+  /* EOF */
+  sqlite3_free(pCur->zPath);
+  pCur->zPath = 0;
+  return SQLITE_OK;
+}
+
+/*
+** Return values of columns for the row at which the series_cursor
+** is currently pointing.
+*/
+static int fsdirColumn(
+  sqlite3_vtab_cursor *cur,   /* The cursor */
+  sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
+  int i                       /* Which column to return */
+){
+  fsdir_cursor *pCur = (fsdir_cursor*)cur;
+  switch( i ){
+    case FSDIR_COLUMN_NAME: {
+      sqlite3_result_text(ctx, &pCur->zPath[pCur->nBase], -1, SQLITE_TRANSIENT);
+      break;
+    }
+
+    case FSDIR_COLUMN_MODE:
+      sqlite3_result_int64(ctx, pCur->sStat.st_mode);
+      break;
+
+    case FSDIR_COLUMN_MTIME:
+      sqlite3_result_int64(ctx, pCur->sStat.st_mtime);
+      break;
+
+    case FSDIR_COLUMN_DATA: {
+      mode_t m = pCur->sStat.st_mode;
+      if( S_ISDIR(m) ){
+        sqlite3_result_null(ctx);
+#if !defined(_WIN32) && !defined(WIN32)
+      }else if( S_ISLNK(m) ){
+        char aStatic[64];
+        char *aBuf = aStatic;
+        int nBuf = 64;
+        int n;
+
+        while( 1 ){
+          n = readlink(pCur->zPath, aBuf, nBuf);
+          if( n<nBuf ) break;
+          if( aBuf!=aStatic ) sqlite3_free(aBuf);
+          nBuf = nBuf*2;
+          aBuf = sqlite3_malloc(nBuf);
+          if( aBuf==0 ){
+            sqlite3_result_error_nomem(ctx);
+            return SQLITE_NOMEM;
+          }
+        }
+
+        sqlite3_result_text(ctx, aBuf, n, SQLITE_TRANSIENT);
+        if( aBuf!=aStatic ) sqlite3_free(aBuf);
+#endif
+      }else{
+        readFileContents(ctx, pCur->zPath);
+      }
+    }
+    case FSDIR_COLUMN_PATH:
+    default: {
+      /* The FSDIR_COLUMN_PATH and FSDIR_COLUMN_DIR are input parameters.
+      ** always return their values as NULL */
+      break;
+    }
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Return the rowid for the current row. In this implementation, the
+** first row returned is assigned rowid value 1, and each subsequent
+** row a value 1 more than that of the previous.
+*/
+static int fsdirRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
+  fsdir_cursor *pCur = (fsdir_cursor*)cur;
+  *pRowid = pCur->iRowid;
+  return SQLITE_OK;
+}
+
+/*
+** Return TRUE if the cursor has been moved off of the last
+** row of output.
+*/
+static int fsdirEof(sqlite3_vtab_cursor *cur){
+  fsdir_cursor *pCur = (fsdir_cursor*)cur;
+  return (pCur->zPath==0);
+}
+
+/*
+** xFilter callback.
+**
+** idxNum==1   PATH parameter only
+** idxNum==2   Both PATH and DIR supplied
+*/
+static int fsdirFilter(
+  sqlite3_vtab_cursor *cur, 
+  int idxNum, const char *idxStr,
+  int argc, sqlite3_value **argv
+){
+  const char *zDir = 0;
+  fsdir_cursor *pCur = (fsdir_cursor*)cur;
+  (void)idxStr;
+  fsdirResetCursor(pCur);
+
+  if( idxNum==0 ){
+    fsdirSetErrmsg(pCur, "table function fsdir requires an argument");
+    return SQLITE_ERROR;
+  }
+
+  assert( argc==idxNum && (argc==1 || argc==2) );
+  zDir = (const char*)sqlite3_value_text(argv[0]);
+  if( zDir==0 ){
+    fsdirSetErrmsg(pCur, "table function fsdir requires a non-NULL argument");
+    return SQLITE_ERROR;
+  }
+  if( argc==2 ){
+    pCur->zBase = (const char*)sqlite3_value_text(argv[1]);
+  }
+  if( pCur->zBase ){
+    pCur->nBase = (int)strlen(pCur->zBase)+1;
+    pCur->zPath = sqlite3_mprintf("%s/%s", pCur->zBase, zDir);
+  }else{
+    pCur->zPath = sqlite3_mprintf("%s", zDir);
+  }
+
+  if( pCur->zPath==0 ){
+    return SQLITE_NOMEM;
+  }
+  if( fileLinkStat(pCur->zPath, &pCur->sStat) ){
+    fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zPath);
+    return SQLITE_ERROR;
+  }
+
+  return SQLITE_OK;
+}
+
+/*
+** SQLite will invoke this method one or more times while planning a query
+** that uses the generate_series virtual table.  This routine needs to create
+** a query plan for each invocation and compute an estimated cost for that
+** plan.
+**
+** In this implementation idxNum is used to represent the
+** query plan.  idxStr is unused.
+**
+** The query plan is represented by values of idxNum:
+**
+**  (1)  The path value is supplied by argv[0]
+**  (2)  Path is in argv[0] and dir is in argv[1]
+*/
+static int fsdirBestIndex(
+  sqlite3_vtab *tab,
+  sqlite3_index_info *pIdxInfo
+){
+  int i;                 /* Loop over constraints */
+  int idxPath = -1;      /* Index in pIdxInfo->aConstraint of PATH= */
+  int idxDir = -1;       /* Index in pIdxInfo->aConstraint of DIR= */
+  int seenPath = 0;      /* True if an unusable PATH= constraint is seen */
+  int seenDir = 0;       /* True if an unusable DIR= constraint is seen */
+  const struct sqlite3_index_constraint *pConstraint;
+
+  (void)tab;
+  pConstraint = pIdxInfo->aConstraint;
+  for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){
+    if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;
+    switch( pConstraint->iColumn ){
+      case FSDIR_COLUMN_PATH: {
+        if( pConstraint->usable ){
+          idxPath = i;
+          seenPath = 0;
+        }else if( idxPath<0 ){
+          seenPath = 1;
+        }
+        break;
+      }
+      case FSDIR_COLUMN_DIR: {
+        if( pConstraint->usable ){
+          idxDir = i;
+          seenDir = 0;
+        }else if( idxDir<0 ){
+          seenDir = 1;
+        }
+        break;
+      }
+    } 
+  }
+  if( seenPath || seenDir ){
+    /* If input parameters are unusable, disallow this plan */
+    return SQLITE_CONSTRAINT;
+  }
+
+  if( idxPath<0 ){
+    pIdxInfo->idxNum = 0;
+    /* The pIdxInfo->estimatedCost should have been initialized to a huge
+    ** number.  Leave it unchanged. */
+    pIdxInfo->estimatedRows = 0x7fffffff;
+  }else{
+    pIdxInfo->aConstraintUsage[idxPath].omit = 1;
+    pIdxInfo->aConstraintUsage[idxPath].argvIndex = 1;
+    if( idxDir>=0 ){
+      pIdxInfo->aConstraintUsage[idxDir].omit = 1;
+      pIdxInfo->aConstraintUsage[idxDir].argvIndex = 2;
+      pIdxInfo->idxNum = 2;
+      pIdxInfo->estimatedCost = 10.0;
+    }else{
+      pIdxInfo->idxNum = 1;
+      pIdxInfo->estimatedCost = 100.0;
+    }
+  }
+
+  return SQLITE_OK;
+}
+
+/*
+** Register the "fsdir" virtual table.
+*/
+static int fsdirRegister(sqlite3 *db){
+  static sqlite3_module fsdirModule = {
+    0,                         /* iVersion */
+    0,                         /* xCreate */
+    fsdirConnect,              /* xConnect */
+    fsdirBestIndex,            /* xBestIndex */
+    fsdirDisconnect,           /* xDisconnect */
+    0,                         /* xDestroy */
+    fsdirOpen,                 /* xOpen - open a cursor */
+    fsdirClose,                /* xClose - close a cursor */
+    fsdirFilter,               /* xFilter - configure scan constraints */
+    fsdirNext,                 /* xNext - advance a cursor */
+    fsdirEof,                  /* xEof - check for end of scan */
+    fsdirColumn,               /* xColumn - read data */
+    fsdirRowid,                /* xRowid - read data */
+    0,                         /* xUpdate */
+    0,                         /* xBegin */
+    0,                         /* xSync */
+    0,                         /* xCommit */
+    0,                         /* xRollback */
+    0,                         /* xFindMethod */
+    0,                         /* xRename */
+    0,                         /* xSavepoint */
+    0,                         /* xRelease */
+    0,                         /* xRollbackTo */
+    0,                         /* xShadowName */
+  };
+
+  int rc = sqlite3_create_module(db, "fsdir", &fsdirModule, 0);
+  return rc;
+}
+#else         /* SQLITE_OMIT_VIRTUALTABLE */
+# define fsdirRegister(x) SQLITE_OK
+#endif
 
 #ifdef _WIN32
 __declspec(dllexport)
@@ -95,8 +956,15 @@ int sqlite3_fileio_init(
   rc = sqlite3_create_function(db, "readfile", 1, SQLITE_UTF8, 0,
                                readfileFunc, 0, 0);
   if( rc==SQLITE_OK ){
-    rc = sqlite3_create_function(db, "writefile", 2, SQLITE_UTF8, 0,
+    rc = sqlite3_create_function(db, "writefile", -1, SQLITE_UTF8, 0,
                                  writefileFunc, 0, 0);
   }
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_create_function(db, "lsmode", 1, SQLITE_UTF8, 0,
+                                 lsModeFunc, 0, 0);
+  }
+  if( rc==SQLITE_OK ){
+    rc = fsdirRegister(db);
+  }
   return rc;
 }

ext/misc/json1.c

@@ -172,6 +172,7 @@ struct JsonParse {
   u8 nErr;           /* Number of errors seen */
   u16 iDepth;        /* Nesting depth */
   int nJson;         /* Length of the zJson string in bytes */
+  u32 iHold;         /* Replace cache line with the lowest iHold value */
 };
 
 /*
@@ -976,7 +977,8 @@ static int jsonParseFindParents(JsonParse *pParse){
 /*
 ** Magic number used for the JSON parse cache in sqlite3_get_auxdata()
 */
-#define JSON_CACHE_ID  (-429938)
+#define JSON_CACHE_ID  (-429938)  /* First cache entry */
+#define JSON_CACHE_SZ  4          /* Max number of cache entries */
 
 /*
 ** Obtain a complete parse of the JSON found in the first argument
@@ -988,16 +990,42 @@ static int jsonParseFindParents(JsonParse *pParse){
 */
 static JsonParse *jsonParseCached(
   sqlite3_context *pCtx,
-  sqlite3_value **argv
+  sqlite3_value **argv,
+  sqlite3_context *pErrCtx
 ){
   const char *zJson = (const char*)sqlite3_value_text(argv[0]);
   int nJson = sqlite3_value_bytes(argv[0]);
   JsonParse *p;
+  JsonParse *pMatch = 0;
+  int iKey;
+  int iMinKey = 0;
+  u32 iMinHold = 0xffffffff;
+  u32 iMaxHold = 0;
   if( zJson==0 ) return 0;
-  p = (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID);
+  for(iKey=0; iKey<JSON_CACHE_SZ; iKey++){
-  if( p && p->nJson==nJson && memcmp(p->zJson,zJson,nJson)==0 ){
+    p = (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID+iKey);
-    p->nErr = 0;
+    if( p==0 ){
-    return p; /* The cached entry matches, so return it */
+      iMinKey = iKey;
+      break;
+    }
+    if( pMatch==0
+     && p->nJson==nJson
+     && memcmp(p->zJson,zJson,nJson)==0
+    ){
+      p->nErr = 0;
+      pMatch = p;
+    }else if( p->iHold<iMinHold ){
+      iMinHold = p->iHold;
+      iMinKey = iKey;
+    }
+    if( p->iHold>iMaxHold ){
+      iMaxHold = p->iHold;
+    }
+  }
+  if( pMatch ){
+    pMatch->nErr = 0;
+    pMatch->iHold = iMaxHold+1;
+    return pMatch;
   }
   p = sqlite3_malloc( sizeof(*p) + nJson + 1 );
   if( p==0 ){
@@ -1007,13 +1035,15 @@ static JsonParse *jsonParseCached(
   memset(p, 0, sizeof(*p));
   p->zJson = (char*)&p[1];
   memcpy((char*)p->zJson, zJson, nJson+1);
-  if( jsonParse(p, pCtx, p->zJson) ){
+  if( jsonParse(p, pErrCtx, p->zJson) ){
     sqlite3_free(p);
     return 0;
   }
   p->nJson = nJson;
-  sqlite3_set_auxdata(pCtx, JSON_CACHE_ID, p, (void(*)(void*))jsonParseFree);
+  p->iHold = iMaxHold+1;
-  return (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID);
+  sqlite3_set_auxdata(pCtx, JSON_CACHE_ID+iMinKey, p,
+                      (void(*)(void*))jsonParseFree);
+  return (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID+iMinKey);
 }
 
 /*
@@ -1386,7 +1416,7 @@ static void jsonArrayLengthFunc(
   u32 i;
   JsonNode *pNode;
 
-  p = jsonParseCached(ctx, argv);
+  p = jsonParseCached(ctx, argv, ctx);
   if( p==0 ) return;
   assert( p->nNode );
   if( argc==2 ){
@@ -1427,7 +1457,7 @@ static void jsonExtractFunc(
   int i;
 
   if( argc<2 ) return;
-  p = jsonParseCached(ctx, argv);
+  p = jsonParseCached(ctx, argv, ctx);
   if( p==0 ) return;
   jsonInit(&jx, ctx);
   jsonAppendChar(&jx, '[');
@@ -1734,22 +1764,21 @@ static void jsonTypeFunc(
   int argc,
   sqlite3_value **argv
 ){
-  JsonParse x;          /* The parse */
+  JsonParse *p;          /* The parse */
   const char *zPath;
   JsonNode *pNode;
 
-  if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return;
+  p = jsonParseCached(ctx, argv, ctx);
-  assert( x.nNode );
+  if( p==0 ) return;
   if( argc==2 ){
     zPath = (const char*)sqlite3_value_text(argv[1]);
-    pNode = jsonLookup(&x, zPath, 0, ctx);
+    pNode = jsonLookup(p, zPath, 0, ctx);
   }else{
-    pNode = x.aNode;
+    pNode = p->aNode;
   }
   if( pNode ){
     sqlite3_result_text(ctx, jsonType[pNode->eType], -1, SQLITE_STATIC);
   }
-  jsonParseReset(&x);
 }
 
 /*
@@ -1763,15 +1792,10 @@ static void jsonValidFunc(
   int argc,
   sqlite3_value **argv
 ){
-  JsonParse x;          /* The parse */
+  JsonParse *p;          /* The parse */
-  int rc = 0;
-
   UNUSED_PARAM(argc);
-  if( jsonParse(&x, 0, (const char*)sqlite3_value_text(argv[0]))==0 ){
+  p = jsonParseCached(ctx, argv, 0);
-    rc = 1;
+  sqlite3_result_int(ctx, p!=0);
-  }
-  jsonParseReset(&x);
-  sqlite3_result_int(ctx, rc);
 }
 
 
@@ -1802,7 +1826,7 @@ static void jsonArrayStep(
     jsonAppendValue(pStr, argv[0]);
   }
 }
-static void jsonArrayFinal(sqlite3_context *ctx){
+static void jsonArrayCompute(sqlite3_context *ctx, int isFinal){
   JsonString *pStr;
   pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0);
   if( pStr ){
@@ -1811,16 +1835,66 @@ static void jsonArrayFinal(sqlite3_context *ctx){
     if( pStr->bErr ){
       if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx);
       assert( pStr->bStatic );
-    }else{
+    }else if( isFinal ){
-      sqlite3_result_text(ctx, pStr->zBuf, pStr->nUsed,
+      sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed,
                           pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free);
       pStr->bStatic = 1;
+    }else{
+      sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT);
+      pStr->nUsed--;
     }
   }else{
     sqlite3_result_text(ctx, "[]", 2, SQLITE_STATIC);
   }
   sqlite3_result_subtype(ctx, JSON_SUBTYPE);
 }
+static void jsonArrayValue(sqlite3_context *ctx){
+  jsonArrayCompute(ctx, 0);
+}
+static void jsonArrayFinal(sqlite3_context *ctx){
+  jsonArrayCompute(ctx, 1);
+}
+
+#ifndef SQLITE_OMIT_WINDOWFUNC
+/*
+** This method works for both json_group_array() and json_group_object().
+** It works by removing the first element of the group by searching forward
+** to the first comma (",") that is not within a string and deleting all
+** text through that comma.
+*/
+static void jsonGroupInverse(
+  sqlite3_context *ctx,
+  int argc,
+  sqlite3_value **argv
+){
+  int i;
+  int inStr = 0;
+  char *z;
+  JsonString *pStr;
+  UNUSED_PARAM(argc);
+  UNUSED_PARAM(argv);
+  pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0);
+#ifdef NEVER
+  /* pStr is always non-NULL since jsonArrayStep() or jsonObjectStep() will
+  ** always have been called to initalize it */
+  if( NEVER(!pStr) ) return;
+#endif
+  z = pStr->zBuf;
+  for(i=1; z[i]!=',' || inStr; i++){
+    assert( i<pStr->nUsed );
+    if( z[i]=='"' ){
+      inStr = !inStr;
+    }else if( z[i]=='\\' ){
+      i++;
+    }
+  }
+  pStr->nUsed -= i;      
+  memmove(&z[1], &z[i+1], (size_t)pStr->nUsed-1);
+}
+#else
+# define jsonGroupInverse 0
+#endif
+
 
 /*
 ** json_group_obj(NAME,VALUE)
@@ -1852,7 +1926,7 @@ static void jsonObjectStep(
     jsonAppendValue(pStr, argv[1]);
   }
 }
-static void jsonObjectFinal(sqlite3_context *ctx){
+static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){
   JsonString *pStr;
   pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0);
   if( pStr ){
@@ -1860,16 +1934,26 @@ static void jsonObjectFinal(sqlite3_context *ctx){
     if( pStr->bErr ){
       if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx);
       assert( pStr->bStatic );
-    }else{
+    }else if( isFinal ){
-      sqlite3_result_text(ctx, pStr->zBuf, pStr->nUsed,
+      sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed,
                           pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free);
       pStr->bStatic = 1;
+    }else{
+      sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT);
+      pStr->nUsed--;
     }
   }else{
     sqlite3_result_text(ctx, "{}", 2, SQLITE_STATIC);
   }
   sqlite3_result_subtype(ctx, JSON_SUBTYPE);
 }
+static void jsonObjectValue(sqlite3_context *ctx){
+  jsonObjectCompute(ctx, 0);
+}
+static void jsonObjectFinal(sqlite3_context *ctx){
+  jsonObjectCompute(ctx, 1);
+}
+
 
 
 #ifndef SQLITE_OMIT_VIRTUALTABLE
@@ -1910,6 +1994,9 @@ static int jsonEachConnect(
 #define JEACH_PARENT  5
 #define JEACH_FULLKEY 6
 #define JEACH_PATH    7
+/* The xBestIndex method assumes that the JSON and ROOT columns are
+** the last two columns in the table.  Should this ever changes, be
+** sure to update the xBestIndex method. */
 #define JEACH_JSON    8
 #define JEACH_ROOT    9
 
@@ -2118,7 +2205,7 @@ static int jsonEachColumn(
         }
         if( p->eType==JSON_ARRAY ){
           jsonPrintf(30, &x, "[%d]", p->iRowid);
-        }else{
+        }else if( p->eType==JSON_OBJECT ){
           jsonPrintf(pThis->n, &x, ".%.*s", pThis->n-2, pThis->u.zJContent+1);
         }
       }
@@ -2167,35 +2254,54 @@ static int jsonEachBestIndex(
   sqlite3_vtab *tab,
   sqlite3_index_info *pIdxInfo
 ){
-  int i;
+  int i;                     /* Loop counter or computed array index */
-  int jsonIdx = -1;
+  int aIdx[2];               /* Index of constraints for JSON and ROOT */
-  int rootIdx = -1;
+  int unusableMask = 0;      /* Mask of unusable JSON and ROOT constraints */
+  int idxMask = 0;           /* Mask of usable == constraints JSON and ROOT */
   const struct sqlite3_index_constraint *pConstraint;
 
+  /* This implementation assumes that JSON and ROOT are the last two
+  ** columns in the table */
+  assert( JEACH_ROOT == JEACH_JSON+1 );
   UNUSED_PARAM(tab);
+  aIdx[0] = aIdx[1] = -1;
   pConstraint = pIdxInfo->aConstraint;
   for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){
-    if( pConstraint->usable==0 ) continue;
+    int iCol;
-    if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;
+    int iMask;
-    switch( pConstraint->iColumn ){
+    if( pConstraint->iColumn < JEACH_JSON ) continue;
-      case JEACH_JSON:   jsonIdx = i;    break;
+    iCol = pConstraint->iColumn - JEACH_JSON;
-      case JEACH_ROOT:   rootIdx = i;    break;
+    assert( iCol==0 || iCol==1 );
-      default:           /* no-op */     break;
+    iMask = 1 << iCol;
+    if( pConstraint->usable==0 ){
+      unusableMask |= iMask;
+    }else if( pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){
+      aIdx[iCol] = i;
+      idxMask |= iMask;
     }
   }
-  if( jsonIdx<0 ){
+  if( (unusableMask & ~idxMask)!=0 ){
+    /* If there are any unusable constraints on JSON or ROOT, then reject
+    ** this entire plan */
+    return SQLITE_CONSTRAINT;
+  }
+  if( aIdx[0]<0 ){
+    /* No JSON input.  Leave estimatedCost at the huge value that it was
+    ** initialized to to discourage the query planner from selecting this
+    ** plan. */
     pIdxInfo->idxNum = 0;
-    pIdxInfo->estimatedCost = 1e99;
   }else{
     pIdxInfo->estimatedCost = 1.0;
-    pIdxInfo->aConstraintUsage[jsonIdx].argvIndex = 1;
+    i = aIdx[0];
-    pIdxInfo->aConstraintUsage[jsonIdx].omit = 1;
+    pIdxInfo->aConstraintUsage[i].argvIndex = 1;
-    if( rootIdx<0 ){
+    pIdxInfo->aConstraintUsage[i].omit = 1;
-      pIdxInfo->idxNum = 1;
+    if( aIdx[1]<0 ){
+      pIdxInfo->idxNum = 1;  /* Only JSON supplied.  Plan 1 */
     }else{
-      pIdxInfo->aConstraintUsage[rootIdx].argvIndex = 2;
+      i = aIdx[1];
-      pIdxInfo->aConstraintUsage[rootIdx].omit = 1;
+      pIdxInfo->aConstraintUsage[i].argvIndex = 2;
-      pIdxInfo->idxNum = 3;
+      pIdxInfo->aConstraintUsage[i].omit = 1;
+      pIdxInfo->idxNum = 3;  /* Both JSON and ROOT are supplied.  Plan 3 */
     }
   }
   return SQLITE_OK;
@@ -2304,7 +2410,8 @@ static sqlite3_module jsonEachModule = {
   0,                         /* xRename */
   0,                         /* xSavepoint */
   0,                         /* xRelease */
-  0                          /* xRollbackTo */
+  0,                         /* xRollbackTo */
+  0                          /* xShadowName */
 };
 
 /* The methods of the json_tree virtual table. */
@@ -2331,7 +2438,8 @@ static sqlite3_module jsonTreeModule = {
   0,                         /* xRename */
   0,                         /* xSavepoint */
   0,                         /* xRelease */
-  0                          /* xRollbackTo */
+  0,                         /* xRollbackTo */
+  0                          /* xShadowName */
 };
 #endif /* SQLITE_OMIT_VIRTUALTABLE */
 
@@ -2377,9 +2485,12 @@ int sqlite3Json1Init(sqlite3 *db){
      int nArg;
      void (*xStep)(sqlite3_context*,int,sqlite3_value**);
      void (*xFinal)(sqlite3_context*);
+     void (*xValue)(sqlite3_context*);
   } aAgg[] = {
-    { "json_group_array",     1,   jsonArrayStep,   jsonArrayFinal  },
+    { "json_group_array",     1,
-    { "json_group_object",    2,   jsonObjectStep,  jsonObjectFinal },
+      jsonArrayStep,   jsonArrayFinal,  jsonArrayValue  },
+    { "json_group_object",    2,
+      jsonObjectStep,  jsonObjectFinal, jsonObjectValue },
   };
 #ifndef SQLITE_OMIT_VIRTUALTABLE
   static const struct {
@@ -2396,11 +2507,14 @@ int sqlite3Json1Init(sqlite3 *db){
                                  (void*)&aFunc[i].flag,
                                  aFunc[i].xFunc, 0, 0);
   }
+#ifndef SQLITE_OMIT_WINDOWFUNC
   for(i=0; i<sizeof(aAgg)/sizeof(aAgg[0]) && rc==SQLITE_OK; i++){
-    rc = sqlite3_create_function(db, aAgg[i].zName, aAgg[i].nArg,
+    rc = sqlite3_create_window_function(db, aAgg[i].zName, aAgg[i].nArg,
                                  SQLITE_UTF8 | SQLITE_DETERMINISTIC, 0,
-                                 0, aAgg[i].xStep, aAgg[i].xFinal);
+                                 aAgg[i].xStep, aAgg[i].xFinal,
+                                 aAgg[i].xValue, jsonGroupInverse, 0);
   }
+#endif
 #ifndef SQLITE_OMIT_VIRTUALTABLE
   for(i=0; i<sizeof(aMod)/sizeof(aMod[0]) && rc==SQLITE_OK; i++){
     rc = sqlite3_create_module(db, aMod[i].zName, aMod[i].pModule, 0);

ext/misc/memstat.c

@@ -0,0 +1,428 @@
+/*
+** 2018-09-27
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** This file demonstrates an eponymous virtual table that returns information
+** from sqlite3_status64() and sqlite3_db_status().
+**
+** Usage example:
+**
+**     .load ./memstat
+**     .mode quote
+**     .header on
+**     SELECT * FROM memstat;
+*/
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_MEMSTATVTAB)
+#if !defined(SQLITEINT_H)
+#include "sqlite3ext.h"
+#endif
+SQLITE_EXTENSION_INIT1
+#include <assert.h>
+#include <string.h>
+
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+
+/* memstat_vtab is a subclass of sqlite3_vtab which will
+** serve as the underlying representation of a memstat virtual table
+*/
+typedef struct memstat_vtab memstat_vtab;
+struct memstat_vtab {
+  sqlite3_vtab base;  /* Base class - must be first */
+  sqlite3 *db;        /* Database connection for this memstat vtab */
+};
+
+/* memstat_cursor is a subclass of sqlite3_vtab_cursor which will
+** serve as the underlying representation of a cursor that scans
+** over rows of the result
+*/
+typedef struct memstat_cursor memstat_cursor;
+struct memstat_cursor {
+  sqlite3_vtab_cursor base;  /* Base class - must be first */
+  sqlite3 *db;               /* Database connection for this cursor */
+  int iRowid;                /* Current row in aMemstatColumn[] */
+  int iDb;                   /* Which schema we are looking at */
+  int nDb;                   /* Number of schemas */
+  char **azDb;               /* Names of all schemas */
+  sqlite3_int64 aVal[2];     /* Result values */
+};
+
+/*
+** The memstatConnect() method is invoked to create a new
+** memstat_vtab that describes the memstat virtual table.
+**
+** Think of this routine as the constructor for memstat_vtab objects.
+**
+** All this routine needs to do is:
+**
+**    (1) Allocate the memstat_vtab object and initialize all fields.
+**
+**    (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the
+**        result set of queries against memstat will look like.
+*/
+static int memstatConnect(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
+){
+  memstat_vtab *pNew;
+  int rc;
+
+/* Column numbers */
+#define MSV_COLUMN_NAME    0   /* Name of quantity being measured */
+#define MSV_COLUMN_SCHEMA  1   /* schema name */
+#define MSV_COLUMN_VALUE   2   /* Current value */
+#define MSV_COLUMN_HIWTR   3   /* Highwater mark */
+
+  rc = sqlite3_declare_vtab(db,"CREATE TABLE x(name,schema,value,hiwtr)");
+  if( rc==SQLITE_OK ){
+    pNew = sqlite3_malloc( sizeof(*pNew) );
+    *ppVtab = (sqlite3_vtab*)pNew;
+    if( pNew==0 ) return SQLITE_NOMEM;
+    memset(pNew, 0, sizeof(*pNew));
+    pNew->db = db;
+  }
+  return rc;
+}
+
+/*
+** This method is the destructor for memstat_cursor objects.
+*/
+static int memstatDisconnect(sqlite3_vtab *pVtab){
+  sqlite3_free(pVtab);
+  return SQLITE_OK;
+}
+
+/*
+** Constructor for a new memstat_cursor object.
+*/
+static int memstatOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
+  memstat_cursor *pCur;
+  pCur = sqlite3_malloc( sizeof(*pCur) );
+  if( pCur==0 ) return SQLITE_NOMEM;
+  memset(pCur, 0, sizeof(*pCur));
+  pCur->db = ((memstat_vtab*)p)->db;
+  *ppCursor = &pCur->base;
+  return SQLITE_OK;
+}
+
+/*
+** Clear all the schema names from a cursor
+*/
+static void memstatClearSchema(memstat_cursor *pCur){
+  int i;
+  if( pCur->azDb==0 ) return;
+  for(i=0; i<pCur->nDb; i++){
+    sqlite3_free(pCur->azDb[i]);
+  }
+  sqlite3_free(pCur->azDb);
+  pCur->azDb = 0;
+  pCur->nDb = 0;
+}
+
+/*
+** Fill in the azDb[] array for the cursor.
+*/
+static int memstatFindSchemas(memstat_cursor *pCur){
+  sqlite3_stmt *pStmt = 0;
+  int rc;
+  if( pCur->nDb ) return SQLITE_OK;
+  rc = sqlite3_prepare_v2(pCur->db, "PRAGMA database_list", -1, &pStmt, 0);
+  if( rc ){
+    sqlite3_finalize(pStmt);
+    return rc;
+  }
+  while( sqlite3_step(pStmt)==SQLITE_ROW ){
+    char **az, *z;
+    az = sqlite3_realloc(pCur->azDb, sizeof(char*)*(pCur->nDb+1));
+    if( az==0 ){
+      memstatClearSchema(pCur);
+      return SQLITE_NOMEM;
+    }
+    pCur->azDb = az;
+    z = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1));
+    if( z==0 ){
+      memstatClearSchema(pCur);
+      return SQLITE_NOMEM;
+    }
+    pCur->azDb[pCur->nDb] = z;
+    pCur->nDb++;
+  }
+  sqlite3_finalize(pStmt);
+  return SQLITE_OK;
+}
+
+
+/*
+** Destructor for a memstat_cursor.
+*/
+static int memstatClose(sqlite3_vtab_cursor *cur){
+  memstat_cursor *pCur = (memstat_cursor*)cur;
+  memstatClearSchema(pCur);
+  sqlite3_free(cur);
+  return SQLITE_OK;
+}
+
+
+/*
+** Allowed values for aMemstatColumn[].eType
+*/
+#define MSV_GSTAT   0          /* sqlite3_status64() information */
+#define MSV_DB      1          /* sqlite3_db_status() information */
+#define MSV_ZIPVFS  2          /* ZIPVFS file-control with 64-bit return */
+
+/*
+** An array of quantities that can be measured and reported by
+** this virtual table
+*/
+static const struct MemstatColumns {
+  const char *zName;    /* Symbolic name */
+  unsigned char eType;  /* Type of interface */
+  unsigned char mNull;  /* Bitmask of which columns are NULL */
+                        /* 2: dbname,  4: current,  8: hiwtr */
+  int eOp;              /* Opcode */
+} aMemstatColumn[] = {
+ {"MEMORY_USED",            MSV_GSTAT,  2, SQLITE_STATUS_MEMORY_USED          },
+ {"MALLOC_SIZE",            MSV_GSTAT,  6, SQLITE_STATUS_MALLOC_SIZE          },
+ {"MALLOC_COUNT",           MSV_GSTAT,  2, SQLITE_STATUS_MALLOC_COUNT         },
+ {"PAGECACHE_USED",         MSV_GSTAT,  2, SQLITE_STATUS_PAGECACHE_USED       },
+ {"PAGECACHE_OVERFLOW",     MSV_GSTAT,  2, SQLITE_STATUS_PAGECACHE_OVERFLOW   },
+ {"PAGECACHE_SIZE",         MSV_GSTAT,  6, SQLITE_STATUS_PAGECACHE_SIZE       },
+ {"PARSER_STACK",           MSV_GSTAT,  6, SQLITE_STATUS_PARSER_STACK         },
+ {"DB_LOOKASIDE_USED",      MSV_DB,     2, SQLITE_DBSTATUS_LOOKASIDE_USED     },
+ {"DB_LOOKASIDE_HIT",       MSV_DB,     6, SQLITE_DBSTATUS_LOOKASIDE_HIT      },
+ {"DB_LOOKASIDE_MISS_SIZE", MSV_DB,     6, SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE},
+ {"DB_LOOKASIDE_MISS_FULL", MSV_DB,     6, SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL},
+ {"DB_CACHE_USED",          MSV_DB,    10, SQLITE_DBSTATUS_CACHE_USED         },
+#if SQLITE_VERSION_NUMBER >= 3140000
+ {"DB_CACHE_USED_SHARED",   MSV_DB,    10, SQLITE_DBSTATUS_CACHE_USED_SHARED  },
+#endif
+ {"DB_SCHEMA_USED",         MSV_DB,    10, SQLITE_DBSTATUS_SCHEMA_USED        },
+ {"DB_STMT_USED",           MSV_DB,    10, SQLITE_DBSTATUS_STMT_USED          },
+ {"DB_CACHE_HIT",           MSV_DB,    10, SQLITE_DBSTATUS_CACHE_HIT          },
+ {"DB_CACHE_MISS",          MSV_DB,    10, SQLITE_DBSTATUS_CACHE_MISS         },
+ {"DB_CACHE_WRITE",         MSV_DB,    10, SQLITE_DBSTATUS_CACHE_WRITE        },
+#if SQLITE_VERSION_NUMBER >= 3230000
+ {"DB_CACHE_SPILL",         MSV_DB,    10, SQLITE_DBSTATUS_CACHE_SPILL        },
+#endif
+ {"DB_DEFERRED_FKS",        MSV_DB,    10, SQLITE_DBSTATUS_DEFERRED_FKS       },
+#ifdef SQLITE_ENABLE_ZIPVFS
+ {"ZIPVFS_CACHE_USED",      MSV_ZIPVFS, 8, 231454 },
+ {"ZIPVFS_CACHE_HIT",       MSV_ZIPVFS, 8, 231455 },
+ {"ZIPVFS_CACHE_MISS",      MSV_ZIPVFS, 8, 231456 },
+ {"ZIPVFS_CACHE_WRITE",     MSV_ZIPVFS, 8, 231457 },
+ {"ZIPVFS_DIRECT_READ",     MSV_ZIPVFS, 8, 231458 },
+ {"ZIPVFS_DIRECT_BYTES",    MSV_ZIPVFS, 8, 231459 },
+#endif /* SQLITE_ENABLE_ZIPVFS */
+};
+#define MSV_NROW (sizeof(aMemstatColumn)/sizeof(aMemstatColumn[0]))
+
+/*
+** Advance a memstat_cursor to its next row of output.
+*/
+static int memstatNext(sqlite3_vtab_cursor *cur){
+  memstat_cursor *pCur = (memstat_cursor*)cur;
+  int i;
+  assert( pCur->iRowid<=MSV_NROW );
+  while(1){
+    i = (int)pCur->iRowid - 1;
+    if( i<0 || (aMemstatColumn[i].mNull & 2)!=0 || (++pCur->iDb)>=pCur->nDb ){
+      pCur->iRowid++;
+      if( pCur->iRowid>MSV_NROW ) return SQLITE_OK;  /* End of the table */
+      pCur->iDb = 0;
+      i++;
+    }
+    pCur->aVal[0] = 0;
+    pCur->aVal[1] = 0;    
+    switch( aMemstatColumn[i].eType ){
+      case MSV_GSTAT: {
+        if( sqlite3_libversion_number()>=3010000 ){
+          sqlite3_status64(aMemstatColumn[i].eOp,
+                           &pCur->aVal[0], &pCur->aVal[1],0);
+        }else{
+          int xCur, xHiwtr;
+          sqlite3_status(aMemstatColumn[i].eOp, &xCur, &xHiwtr, 0);
+          pCur->aVal[0] = xCur;
+          pCur->aVal[1] = xHiwtr;
+        }
+        break;
+      }
+      case MSV_DB: {
+        int xCur, xHiwtr;
+        sqlite3_db_status(pCur->db, aMemstatColumn[i].eOp, &xCur, &xHiwtr, 0);
+        pCur->aVal[0] = xCur;
+        pCur->aVal[1] = xHiwtr;
+        break;
+      }
+      case MSV_ZIPVFS: {
+        int rc;
+        rc = sqlite3_file_control(pCur->db, pCur->azDb[pCur->iDb],
+                                  aMemstatColumn[i].eOp, (void*)&pCur->aVal[0]);
+        if( rc!=SQLITE_OK ) continue;
+        break;
+      }
+    }
+    break;
+  }
+  return SQLITE_OK;
+}
+  
+
+/*
+** Return values of columns for the row at which the memstat_cursor
+** is currently pointing.
+*/
+static int memstatColumn(
+  sqlite3_vtab_cursor *cur,   /* The cursor */
+  sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
+  int iCol                    /* Which column to return */
+){
+  memstat_cursor *pCur = (memstat_cursor*)cur;
+  int i;
+  assert( pCur->iRowid>0 && pCur->iRowid<=MSV_NROW );
+  i = (int)pCur->iRowid - 1;
+  if( (aMemstatColumn[i].mNull & (1<<iCol))!=0 ){
+    return SQLITE_OK;
+  }
+  switch( iCol ){
+    case MSV_COLUMN_NAME: {
+      sqlite3_result_text(ctx, aMemstatColumn[i].zName, -1, SQLITE_STATIC);
+      break;
+    }
+    case MSV_COLUMN_SCHEMA: {
+      sqlite3_result_text(ctx, pCur->azDb[pCur->iDb], -1, 0);
+      break;
+    }
+    case MSV_COLUMN_VALUE: {
+      sqlite3_result_int64(ctx, pCur->aVal[0]);
+      break;
+    }
+    case MSV_COLUMN_HIWTR: {
+      sqlite3_result_int64(ctx, pCur->aVal[1]);
+      break;
+    }
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Return the rowid for the current row.  In this implementation, the
+** rowid is the same as the output value.
+*/
+static int memstatRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
+  memstat_cursor *pCur = (memstat_cursor*)cur;
+  *pRowid = pCur->iRowid*1000 + pCur->iDb;
+  return SQLITE_OK;
+}
+
+/*
+** Return TRUE if the cursor has been moved off of the last
+** row of output.
+*/
+static int memstatEof(sqlite3_vtab_cursor *cur){
+  memstat_cursor *pCur = (memstat_cursor*)cur;
+  return pCur->iRowid>MSV_NROW;
+}
+
+/*
+** This method is called to "rewind" the memstat_cursor object back
+** to the first row of output.  This method is always called at least
+** once prior to any call to memstatColumn() or memstatRowid() or 
+** memstatEof().
+*/
+static int memstatFilter(
+  sqlite3_vtab_cursor *pVtabCursor, 
+  int idxNum, const char *idxStr,
+  int argc, sqlite3_value **argv
+){
+  memstat_cursor *pCur = (memstat_cursor *)pVtabCursor;
+  int rc = memstatFindSchemas(pCur);
+  if( rc ) return rc;
+  pCur->iRowid = 0;
+  pCur->iDb = 0;
+  return memstatNext(pVtabCursor);
+}
+
+/*
+** SQLite will invoke this method one or more times while planning a query
+** that uses the memstat virtual table.  This routine needs to create
+** a query plan for each invocation and compute an estimated cost for that
+** plan.
+*/
+static int memstatBestIndex(
+  sqlite3_vtab *tab,
+  sqlite3_index_info *pIdxInfo
+){
+  pIdxInfo->estimatedCost = (double)500;
+  pIdxInfo->estimatedRows = 500;
+  return SQLITE_OK;
+}
+
+/*
+** This following structure defines all the methods for the 
+** memstat virtual table.
+*/
+static sqlite3_module memstatModule = {
+  0,                         /* iVersion */
+  0,                         /* xCreate */
+  memstatConnect,            /* xConnect */
+  memstatBestIndex,          /* xBestIndex */
+  memstatDisconnect,         /* xDisconnect */
+  0,                         /* xDestroy */
+  memstatOpen,               /* xOpen - open a cursor */
+  memstatClose,              /* xClose - close a cursor */
+  memstatFilter,             /* xFilter - configure scan constraints */
+  memstatNext,               /* xNext - advance a cursor */
+  memstatEof,                /* xEof - check for end of scan */
+  memstatColumn,             /* xColumn - read data */
+  memstatRowid,              /* xRowid - read data */
+  0,                         /* xUpdate */
+  0,                         /* xBegin */
+  0,                         /* xSync */
+  0,                         /* xCommit */
+  0,                         /* xRollback */
+  0,                         /* xFindMethod */
+  0,                         /* xRename */
+  0,                         /* xSavepoint */
+  0,                         /* xRelease */
+  0,                         /* xRollbackTo */
+  0,                         /* xShadowName */
+};
+
+#endif /* SQLITE_OMIT_VIRTUALTABLE */
+
+int sqlite3MemstatVtabInit(sqlite3 *db){
+  int rc = SQLITE_OK;
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+  rc = sqlite3_create_module(db, "sqlite_memstat", &memstatModule, 0);
+#endif
+  return rc;
+}
+
+#ifndef SQLITE_CORE
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_memstat_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  int rc = SQLITE_OK;
+  SQLITE_EXTENSION_INIT2(pApi);
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+  rc = sqlite3MemstatVtabInit(db);
+#endif
+  return rc;
+}
+#endif /* SQLITE_CORE */
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_MEMSTATVTAB) */

ext/misc/memvfs.c

@@ -10,23 +10,33 @@
 **
 ******************************************************************************
 **
-** This is an in-memory read-only VFS implementation.  The application
+** This is an in-memory VFS implementation.  The application supplies
-** supplies a block of memory which is the database file, and this VFS
+** a chunk of memory to hold the database file.
-** uses that block of memory.
 **
-** Because there is no place to store journals and no good way to lock
+** Because there is place to store a rollback or wal journal, the database
-** the "file", this VFS is read-only.
+** must use one of journal_mode=MEMORY or journal_mode=NONE.
 **
 ** USAGE:
 **
-**    sqlite3_open_v2("file:/whatever?ptr=0xf05538&sz=14336", &db,
+**    sqlite3_open_v2("file:/whatever?ptr=0xf05538&sz=14336&max=65536", &db,
-**                    SQLITE_OPEN_READONLY | SQLITE_OPEN_URI,
+**                    SQLITE_OPEN_READWRITE | SQLITE_OPEN_URI,
 **                    "memvfs");
 **
-** The ptr= and sz= query parameters are required or the open will fail.
+** These are the query parameters:
-** The ptr= parameter gives the memory address of the buffer holding the
+**
-** read-only database and sz= gives the size of the database.  The parameter
+**    ptr=          The address of the memory buffer that holds the database.
-** values may be in hexadecimal or decimal.  The filename is ignored.
+**
+**    sz=           The current size the database file
+**
+**    maxsz=        The maximum size of the database.  In other words, the
+**                  amount of space allocated for the ptr= buffer.
+**
+**    freeonclose=  If true, then sqlite3_free() is called on the ptr=
+**                  value when the connection closes.
+**
+** The ptr= and sz= query parameters are required.  If maxsz= is omitted,
+** then it defaults to the sz= value.  Parameter values can be in either
+** decimal or hexadecimal.  The filename in the URI is ignored.
 */
 #include <sqlite3ext.h>
 SQLITE_EXTENSION_INIT1
@@ -49,7 +59,9 @@ typedef struct MemFile MemFile;
 struct MemFile {
   sqlite3_file base;              /* IO methods */
   sqlite3_int64 sz;               /* Size of the file */
+  sqlite3_int64 szMax;            /* Space allocated to aData */
   unsigned char *aData;           /* content of the file */
+  int bFreeOnClose;               /* Invoke sqlite3_free() on aData at close */
 };
 
 /*
@@ -144,6 +156,8 @@ static const sqlite3_io_methods mem_io_methods = {
 ** to free.
 */
 static int memClose(sqlite3_file *pFile){
+  MemFile *p = (MemFile *)pFile;
+  if( p->bFreeOnClose ) sqlite3_free(p->aData);
   return SQLITE_OK;
 }
 
@@ -170,21 +184,34 @@ static int memWrite(
   int iAmt,
   sqlite_int64 iOfst
 ){
-  return SQLITE_READONLY;
+  MemFile *p = (MemFile *)pFile;
+  if( iOfst+iAmt>p->sz ){
+    if( iOfst+iAmt>p->szMax ) return SQLITE_FULL;
+    if( iOfst>p->sz ) memset(p->aData+p->sz, 0, iOfst-p->sz);
+    p->sz = iOfst+iAmt;
+  }
+  memcpy(p->aData+iOfst, z, iAmt);
+  return SQLITE_OK;
 }
 
 /*
 ** Truncate an mem-file.
 */
 static int memTruncate(sqlite3_file *pFile, sqlite_int64 size){
-  return SQLITE_READONLY;
+  MemFile *p = (MemFile *)pFile;
+  if( size>p->sz ){
+    if( size>p->szMax ) return SQLITE_FULL;
+    memset(p->aData+p->sz, 0, size-p->sz);
+  }
+  p->sz = size; 
+  return SQLITE_OK;
 }
 
 /*
 ** Sync an mem-file.
 */
 static int memSync(sqlite3_file *pFile, int flags){
-  return SQLITE_READONLY;
+  return SQLITE_OK;
 }
 
 /*
@@ -200,7 +227,7 @@ static int memFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){
 ** Lock an mem-file.
 */
 static int memLock(sqlite3_file *pFile, int eLock){
-  return SQLITE_READONLY;
+  return SQLITE_OK;
 }
 
 /*
@@ -242,7 +269,10 @@ static int memSectorSize(sqlite3_file *pFile){
 ** Return the device characteristic flags supported by an mem-file.
 */
 static int memDeviceCharacteristics(sqlite3_file *pFile){
-  return SQLITE_IOCAP_IMMUTABLE;
+  return SQLITE_IOCAP_ATOMIC | 
+         SQLITE_IOCAP_POWERSAFE_OVERWRITE |
+         SQLITE_IOCAP_SAFE_APPEND |
+         SQLITE_IOCAP_SEQUENTIAL;
 }
 
 /* Create a shared memory file mapping */
@@ -253,12 +283,12 @@ static int memShmMap(
   int bExtend,
   void volatile **pp
 ){
-  return SQLITE_READONLY;
+  return SQLITE_IOERR_SHMMAP;
 }
 
 /* Perform locking on a shared-memory segment */
 static int memShmLock(sqlite3_file *pFile, int offset, int n, int flags){
-  return SQLITE_READONLY;
+  return SQLITE_IOERR_SHMLOCK;
 }
 
 /* Memory barrier operation on shared memory */
@@ -305,6 +335,9 @@ static int memOpen(
   if( p->aData==0 ) return SQLITE_CANTOPEN;
   p->sz = sqlite3_uri_int64(zName,"sz",0);
   if( p->sz<0 ) return SQLITE_CANTOPEN;
+  p->szMax = sqlite3_uri_int64(zName,"max",p->sz);
+  if( p->szMax<p->sz ) return SQLITE_CANTOPEN;
+  p->bFreeOnClose = sqlite3_uri_boolean(zName,"freeonclose",0);
   pFile->pMethods = &mem_io_methods;
   return SQLITE_OK;
 }
@@ -315,7 +348,7 @@ static int memOpen(
 ** returning.
 */
 static int memDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){
-  return SQLITE_READONLY;
+  return SQLITE_IOERR_DELETE;
 }
 
 /*
@@ -328,14 +361,7 @@ static int memAccess(
   int flags, 
   int *pResOut
 ){
-  /* The spec says there are three possible values for flags.  But only
+  *pResOut = 0;
-  ** two of them are actually used */
-  assert( flags==SQLITE_ACCESS_EXISTS || flags==SQLITE_ACCESS_READWRITE );
-  if( flags==SQLITE_ACCESS_READWRITE ){
-    *pResOut = 0;
-  }else{
-    *pResOut = 1;
-  }
   return SQLITE_OK;
 }
 
@@ -416,31 +442,43 @@ static int memCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){
 
 #ifdef MEMVFS_TEST
 /*
-**       memload(FILENAME)
+**       memvfs_from_file(FILENAME, MAXSIZE)
 **
 ** This an SQL function used to help in testing the memvfs VFS.  The
 ** function reads the content of a file into memory and then returns
-** a string that gives the locate and size of the in-memory buffer.
+** a URI that can be handed to ATTACH to attach the memory buffer as
+** a database.  Example:
+**
+**       ATTACH memvfs_from_file('test.db',1048576) AS inmem;
+**
+** The optional MAXSIZE argument gives the size of the memory allocation
+** used to hold the database.  If omitted, it defaults to the size of the
+** file on disk.
 */
 #include <stdio.h>
-static void memvfsMemloadFunc(
+static void memvfsFromFileFunc(
   sqlite3_context *context,
   int argc,
   sqlite3_value **argv
 ){
   unsigned char *p;
   sqlite3_int64 sz;
+  sqlite3_int64 szMax;
   FILE *in;
   const char *zFilename = (const char*)sqlite3_value_text(argv[0]);
-  char zReturn[100];
+  char *zUri;
 
   if( zFilename==0 ) return;
   in = fopen(zFilename, "rb");
   if( in==0 ) return;
   fseek(in, 0, SEEK_END);
-  sz = ftell(in);
+  szMax = sz = ftell(in);
   rewind(in);
-  p = sqlite3_malloc( sz );
+  if( argc>=2 ){
+    szMax = sqlite3_value_int64(argv[1]);
+    if( szMax<sz ) szMax = sz;
+  }
+  p = sqlite3_malloc64( szMax );
   if( p==0 ){
     fclose(in);
     sqlite3_result_error_nomem(context);
@@ -448,18 +486,60 @@ static void memvfsMemloadFunc(
   }
   fread(p, sz, 1, in);
   fclose(in);
-  sqlite3_snprintf(sizeof(zReturn),zReturn,"ptr=%lld&sz=%lld",
+  zUri = sqlite3_mprintf(
-                   (sqlite3_int64)p, sz);
+           "file:/mem?vfs=memvfs&ptr=%lld&sz=%lld&max=%lld&freeonclose=1",
-  sqlite3_result_text(context, zReturn, -1, SQLITE_TRANSIENT);
+                         (sqlite3_int64)p, sz, szMax);
+  sqlite3_result_text(context, zUri, -1, sqlite3_free);
 }
+#endif /* MEMVFS_TEST */
+
+#ifdef MEMVFS_TEST
+/*
+**       memvfs_to_file(SCHEMA, FILENAME)
+**
+** The schema identified by SCHEMA must be a memvfs database.  Write
+** the content of this database into FILENAME.
+*/
+static void memvfsToFileFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  MemFile *p = 0;
+  FILE *out;
+  int rc;
+  sqlite3 *db = sqlite3_context_db_handle(context);
+  sqlite3_vfs *pVfs = 0;
+  const char *zSchema = (const char*)sqlite3_value_text(argv[0]);
+  const char *zFilename = (const char*)sqlite3_value_text(argv[1]);
+
+  if( zFilename==0 ) return;
+  out = fopen(zFilename, "wb");
+  if( out==0 ) return;
+  rc = sqlite3_file_control(db, zSchema, SQLITE_FCNTL_VFS_POINTER, &pVfs);
+  if( rc || pVfs==0 ) return;
+  if( strcmp(pVfs->zName,"memvfs")!=0 ) return;
+  rc = sqlite3_file_control(db, zSchema, SQLITE_FCNTL_FILE_POINTER, &p);
+  if( rc ) return;
+  fwrite(p->aData, 1, (size_t)p->sz, out);
+  fclose(out);
+}
+#endif /* MEMVFS_TEST */
+
+#ifdef MEMVFS_TEST
 /* Called for each new database connection */
 static int memvfsRegister(
   sqlite3 *db,
-  const char **pzErrMsg,
+  char **pzErrMsg,
   const struct sqlite3_api_routines *pThunk
 ){
-  return sqlite3_create_function(db, "memload", 1, SQLITE_UTF8, 0,
+  sqlite3_create_function(db, "memvfs_from_file", 1, SQLITE_UTF8, 0,
-                                 memvfsMemloadFunc, 0, 0);
+                          memvfsFromFileFunc, 0, 0);
+  sqlite3_create_function(db, "memvfs_from_file", 2, SQLITE_UTF8, 0,
+                          memvfsFromFileFunc, 0, 0);
+  sqlite3_create_function(db, "memvfs_to_file", 2, SQLITE_UTF8, 0,
+                          memvfsToFileFunc, 0, 0);
+  return SQLITE_OK;
 }
 #endif /* MEMVFS_TEST */
 
@@ -485,6 +565,9 @@ int sqlite3_memvfs_init(
   if( rc==SQLITE_OK ){
     rc = sqlite3_auto_extension((void(*)(void))memvfsRegister);
   }
+  if( rc==SQLITE_OK ){
+    rc = memvfsRegister(db, pzErrMsg, pApi);
+  }
 #endif
   if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY;
   return rc;

ext/misc/normalize.c

@@ -0,0 +1,707 @@
+/*
+** 2018-01-08
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This file contains code to implement the sqlite3_normalize() function.
+**
+**    char *sqlite3_normalize(const char *zSql);
+**
+** This function takes an SQL string as input and returns a "normalized"
+** version of that string in memory obtained from sqlite3_malloc64().  The
+** caller is responsible for ensuring that the returned memory is freed.
+**
+** If a memory allocation error occurs, this routine returns NULL.
+**
+** The normalization consists of the following transformations:
+**
+**   (1)  Convert every literal (string, blob literal, numeric constant,
+**        or "NULL" constant) into a ?
+**
+**   (2)  Remove all superfluous whitespace, including comments.  Change
+**        all required whitespace to a single space character.
+**
+**   (3)  Lowercase all ASCII characters.
+**
+**   (4)  If an IN or NOT IN operator is followed by a list of 1 or more
+**        values, convert that list into "(?,?,?)".
+**
+** The purpose of normalization is two-fold:
+**
+**   (1)  Sanitize queries by removing potentially private or sensitive
+**        information contained in literals.
+**
+**   (2)  Identify structurally identical queries by comparing their
+**        normalized forms.
+**
+** Command-Line Utility
+** --------------------
+**
+** This file also contains code for a command-line utility that converts
+** SQL queries in text files into their normalized forms.  To build the
+** command-line program, compile this file with -DSQLITE_NORMALIZE_CLI
+** and link it against the SQLite library.
+*/
+#include <sqlite3.h>
+#include <string.h>
+
+/*
+** Implementation note:
+**
+** Much of the tokenizer logic is copied out of the tokenize.c source file
+** of SQLite.  That logic could be simplified for this particular application,
+** but that would impose a risk of introducing subtle errors.  It is best to
+** keep the code as close to the original as possible.
+**
+** The tokenize code is in sync with the SQLite core as of 2018-01-08.
+** Any future changes to the core tokenizer might require corresponding
+** adjustments to the tokenizer logic in this module.
+*/
+
+
+/* Character classes for tokenizing
+**
+** In the sqlite3GetToken() function, a switch() on aiClass[c] is implemented
+** using a lookup table, whereas a switch() directly on c uses a binary search.
+** The lookup table is much faster.  To maximize speed, and to ensure that
+** a lookup table is used, all of the classes need to be small integers and
+** all of them need to be used within the switch.
+*/
+#define CC_X          0    /* The letter 'x', or start of BLOB literal */
+#define CC_KYWD       1    /* Alphabetics or '_'.  Usable in a keyword */
+#define CC_ID         2    /* unicode characters usable in IDs */
+#define CC_DIGIT      3    /* Digits */
+#define CC_DOLLAR     4    /* '$' */
+#define CC_VARALPHA   5    /* '@', '#', ':'.  Alphabetic SQL variables */
+#define CC_VARNUM     6    /* '?'.  Numeric SQL variables */
+#define CC_SPACE      7    /* Space characters */
+#define CC_QUOTE      8    /* '"', '\'', or '`'.  String literals, quoted ids */
+#define CC_QUOTE2     9    /* '['.   [...] style quoted ids */
+#define CC_PIPE      10    /* '|'.   Bitwise OR or concatenate */
+#define CC_MINUS     11    /* '-'.  Minus or SQL-style comment */
+#define CC_LT        12    /* '<'.  Part of < or <= or <> */
+#define CC_GT        13    /* '>'.  Part of > or >= */
+#define CC_EQ        14    /* '='.  Part of = or == */
+#define CC_BANG      15    /* '!'.  Part of != */
+#define CC_SLASH     16    /* '/'.  / or c-style comment */
+#define CC_LP        17    /* '(' */
+#define CC_RP        18    /* ')' */
+#define CC_SEMI      19    /* ';' */
+#define CC_PLUS      20    /* '+' */
+#define CC_STAR      21    /* '*' */
+#define CC_PERCENT   22    /* '%' */
+#define CC_COMMA     23    /* ',' */
+#define CC_AND       24    /* '&' */
+#define CC_TILDA     25    /* '~' */
+#define CC_DOT       26    /* '.' */
+#define CC_ILLEGAL   27    /* Illegal character */
+
+static const unsigned char aiClass[] = {
+/*         x0  x1  x2  x3  x4  x5  x6  x7  x8  x9  xa  xb  xc  xd  xe  xf */
+/* 0x */   27, 27, 27, 27, 27, 27, 27, 27, 27,  7,  7, 27,  7,  7, 27, 27,
+/* 1x */   27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
+/* 2x */    7, 15,  8,  5,  4, 22, 24,  8, 17, 18, 21, 20, 23, 11, 26, 16,
+/* 3x */    3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  5, 19, 12, 14, 13,  6,
+/* 4x */    5,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,
+/* 5x */    1,  1,  1,  1,  1,  1,  1,  1,  0,  1,  1,  9, 27, 27, 27,  1,
+/* 6x */    8,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,
+/* 7x */    1,  1,  1,  1,  1,  1,  1,  1,  0,  1,  1, 27, 10, 27, 25, 27,
+/* 8x */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
+/* 9x */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
+/* Ax */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
+/* Bx */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
+/* Cx */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
+/* Dx */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
+/* Ex */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
+/* Fx */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2
+};
+
+/* An array to map all upper-case characters into their corresponding
+** lower-case character. 
+**
+** SQLite only considers US-ASCII (or EBCDIC) characters.  We do not
+** handle case conversions for the UTF character set since the tables
+** involved are nearly as big or bigger than SQLite itself.
+*/
+static const unsigned char sqlite3UpperToLower[] = {
+      0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 16, 17,
+     18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35,
+     36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53,
+     54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 97, 98, 99,100,101,102,103,
+    104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,
+    122, 91, 92, 93, 94, 95, 96, 97, 98, 99,100,101,102,103,104,105,106,107,
+    108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,
+    126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,
+    144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,
+    162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,
+    180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,
+    198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,
+    216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,
+    234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,
+    252,253,254,255
+};
+
+/*
+** The following 256 byte lookup table is used to support SQLites built-in
+** equivalents to the following standard library functions:
+**
+**   isspace()                        0x01
+**   isalpha()                        0x02
+**   isdigit()                        0x04
+**   isalnum()                        0x06
+**   isxdigit()                       0x08
+**   toupper()                        0x20
+**   SQLite identifier character      0x40
+**   Quote character                  0x80
+**
+** Bit 0x20 is set if the mapped character requires translation to upper
+** case. i.e. if the character is a lower-case ASCII character.
+** If x is a lower-case ASCII character, then its upper-case equivalent
+** is (x - 0x20). Therefore toupper() can be implemented as:
+**
+**   (x & ~(map[x]&0x20))
+**
+** The equivalent of tolower() is implemented using the sqlite3UpperToLower[]
+** array. tolower() is used more often than toupper() by SQLite.
+**
+** Bit 0x40 is set if the character is non-alphanumeric and can be used in an 
+** SQLite identifier.  Identifiers are alphanumerics, "_", "$", and any
+** non-ASCII UTF character. Hence the test for whether or not a character is
+** part of an identifier is 0x46.
+*/
+static const unsigned char sqlite3CtypeMap[256] = {
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,  /* 00..07    ........ */
+  0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00,  /* 08..0f    ........ */
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,  /* 10..17    ........ */
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,  /* 18..1f    ........ */
+  0x01, 0x00, 0x80, 0x00, 0x40, 0x00, 0x00, 0x80,  /* 20..27     !"#$%&' */
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,  /* 28..2f    ()*+,-./ */
+  0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c,  /* 30..37    01234567 */
+  0x0c, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,  /* 38..3f    89:;<=>? */
+
+  0x00, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x02,  /* 40..47    @ABCDEFG */
+  0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,  /* 48..4f    HIJKLMNO */
+  0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,  /* 50..57    PQRSTUVW */
+  0x02, 0x02, 0x02, 0x80, 0x00, 0x00, 0x00, 0x40,  /* 58..5f    XYZ[\]^_ */
+  0x80, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x22,  /* 60..67    `abcdefg */
+  0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22,  /* 68..6f    hijklmno */
+  0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22,  /* 70..77    pqrstuvw */
+  0x22, 0x22, 0x22, 0x00, 0x00, 0x00, 0x00, 0x00,  /* 78..7f    xyz{|}~. */
+
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* 80..87    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* 88..8f    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* 90..97    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* 98..9f    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* a0..a7    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* a8..af    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* b0..b7    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* b8..bf    ........ */
+
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* c0..c7    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* c8..cf    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* d0..d7    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* d8..df    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* e0..e7    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* e8..ef    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* f0..f7    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40   /* f8..ff    ........ */
+};
+#define sqlite3Toupper(x)   ((x)&~(sqlite3CtypeMap[(unsigned char)(x)]&0x20))
+#define sqlite3Isspace(x)   (sqlite3CtypeMap[(unsigned char)(x)]&0x01)
+#define sqlite3Isalnum(x)   (sqlite3CtypeMap[(unsigned char)(x)]&0x06)
+#define sqlite3Isalpha(x)   (sqlite3CtypeMap[(unsigned char)(x)]&0x02)
+#define sqlite3Isdigit(x)   (sqlite3CtypeMap[(unsigned char)(x)]&0x04)
+#define sqlite3Isxdigit(x)  (sqlite3CtypeMap[(unsigned char)(x)]&0x08)
+#define sqlite3Tolower(x)   (sqlite3UpperToLower[(unsigned char)(x)])
+#define sqlite3Isquote(x)   (sqlite3CtypeMap[(unsigned char)(x)]&0x80)
+
+
+/*
+** If X is a character that can be used in an identifier then
+** IdChar(X) will be true.  Otherwise it is false.
+**
+** For ASCII, any character with the high-order bit set is
+** allowed in an identifier.  For 7-bit characters, 
+** sqlite3IsIdChar[X] must be 1.
+**
+** For EBCDIC, the rules are more complex but have the same
+** end result.
+**
+** Ticket #1066.  the SQL standard does not allow '$' in the
+** middle of identifiers.  But many SQL implementations do. 
+** SQLite will allow '$' in identifiers for compatibility.
+** But the feature is undocumented.
+*/
+#define IdChar(C)  ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0)
+
+/*
+** Ignore testcase() macros
+*/
+#define testcase(X)
+
+/*
+** Token values
+*/
+#define TK_SPACE    0
+#define TK_NAME     1
+#define TK_LITERAL  2
+#define TK_PUNCT    3
+#define TK_ERROR    4
+
+#define TK_MINUS    TK_PUNCT
+#define TK_LP       TK_PUNCT
+#define TK_RP       TK_PUNCT
+#define TK_SEMI     TK_PUNCT
+#define TK_PLUS     TK_PUNCT
+#define TK_STAR     TK_PUNCT
+#define TK_SLASH    TK_PUNCT
+#define TK_REM      TK_PUNCT
+#define TK_EQ       TK_PUNCT
+#define TK_LE       TK_PUNCT
+#define TK_NE       TK_PUNCT
+#define TK_LSHIFT   TK_PUNCT
+#define TK_LT       TK_PUNCT
+#define TK_GE       TK_PUNCT
+#define TK_RSHIFT   TK_PUNCT
+#define TK_GT       TK_PUNCT
+#define TK_GE       TK_PUNCT
+#define TK_BITOR    TK_PUNCT
+#define TK_CONCAT   TK_PUNCT
+#define TK_COMMA    TK_PUNCT
+#define TK_BITAND   TK_PUNCT
+#define TK_BITNOT   TK_PUNCT
+#define TK_STRING   TK_LITERAL
+#define TK_ID       TK_NAME
+#define TK_ILLEGAL  TK_ERROR
+#define TK_DOT      TK_PUNCT
+#define TK_INTEGER  TK_LITERAL
+#define TK_FLOAT    TK_LITERAL
+#define TK_VARIABLE TK_LITERAL
+#define TK_BLOB     TK_LITERAL
+
+/*
+** Return the length (in bytes) of the token that begins at z[0]. 
+** Store the token type in *tokenType before returning.
+*/
+static int sqlite3GetToken(const unsigned char *z, int *tokenType){
+  int i, c;
+  switch( aiClass[*z] ){  /* Switch on the character-class of the first byte
+                          ** of the token. See the comment on the CC_ defines
+                          ** above. */
+    case CC_SPACE: {
+      for(i=1; sqlite3Isspace(z[i]); i++){}
+      *tokenType = TK_SPACE;
+      return i;
+    }
+    case CC_MINUS: {
+      if( z[1]=='-' ){
+        for(i=2; (c=z[i])!=0 && c!='\n'; i++){}
+        *tokenType = TK_SPACE;
+        return i;
+      }
+      *tokenType = TK_MINUS;
+      return 1;
+    }
+    case CC_LP: {
+      *tokenType = TK_LP;
+      return 1;
+    }
+    case CC_RP: {
+      *tokenType = TK_RP;
+      return 1;
+    }
+    case CC_SEMI: {
+      *tokenType = TK_SEMI;
+      return 1;
+    }
+    case CC_PLUS: {
+      *tokenType = TK_PLUS;
+      return 1;
+    }
+    case CC_STAR: {
+      *tokenType = TK_STAR;
+      return 1;
+    }
+    case CC_SLASH: {
+      if( z[1]!='*' || z[2]==0 ){
+        *tokenType = TK_SLASH;
+        return 1;
+      }
+      for(i=3, c=z[2]; (c!='*' || z[i]!='/') && (c=z[i])!=0; i++){}
+      if( c ) i++;
+      *tokenType = TK_SPACE;
+      return i;
+    }
+    case CC_PERCENT: {
+      *tokenType = TK_REM;
+      return 1;
+    }
+    case CC_EQ: {
+      *tokenType = TK_EQ;
+      return 1 + (z[1]=='=');
+    }
+    case CC_LT: {
+      if( (c=z[1])=='=' ){
+        *tokenType = TK_LE;
+        return 2;
+      }else if( c=='>' ){
+        *tokenType = TK_NE;
+        return 2;
+      }else if( c=='<' ){
+        *tokenType = TK_LSHIFT;
+        return 2;
+      }else{
+        *tokenType = TK_LT;
+        return 1;
+      }
+    }
+    case CC_GT: {
+      if( (c=z[1])=='=' ){
+        *tokenType = TK_GE;
+        return 2;
+      }else if( c=='>' ){
+        *tokenType = TK_RSHIFT;
+        return 2;
+      }else{
+        *tokenType = TK_GT;
+        return 1;
+      }
+    }
+    case CC_BANG: {
+      if( z[1]!='=' ){
+        *tokenType = TK_ILLEGAL;
+        return 1;
+      }else{
+        *tokenType = TK_NE;
+        return 2;
+      }
+    }
+    case CC_PIPE: {
+      if( z[1]!='|' ){
+        *tokenType = TK_BITOR;
+        return 1;
+      }else{
+        *tokenType = TK_CONCAT;
+        return 2;
+      }
+    }
+    case CC_COMMA: {
+      *tokenType = TK_COMMA;
+      return 1;
+    }
+    case CC_AND: {
+      *tokenType = TK_BITAND;
+      return 1;
+    }
+    case CC_TILDA: {
+      *tokenType = TK_BITNOT;
+      return 1;
+    }
+    case CC_QUOTE: {
+      int delim = z[0];
+      testcase( delim=='`' );
+      testcase( delim=='\'' );
+      testcase( delim=='"' );
+      for(i=1; (c=z[i])!=0; i++){
+        if( c==delim ){
+          if( z[i+1]==delim ){
+            i++;
+          }else{
+            break;
+          }
+        }
+      }
+      if( c=='\'' ){
+        *tokenType = TK_STRING;
+        return i+1;
+      }else if( c!=0 ){
+        *tokenType = TK_ID;
+        return i+1;
+      }else{
+        *tokenType = TK_ILLEGAL;
+        return i;
+      }
+    }
+    case CC_DOT: {
+      if( !sqlite3Isdigit(z[1]) ){
+        *tokenType = TK_DOT;
+        return 1;
+      }
+      /* If the next character is a digit, this is a floating point
+      ** number that begins with ".".  Fall thru into the next case */
+    }
+    case CC_DIGIT: {
+      *tokenType = TK_INTEGER;
+      if( z[0]=='0' && (z[1]=='x' || z[1]=='X') && sqlite3Isxdigit(z[2]) ){
+        for(i=3; sqlite3Isxdigit(z[i]); i++){}
+        return i;
+      }
+      for(i=0; sqlite3Isdigit(z[i]); i++){}
+      if( z[i]=='.' ){
+        i++;
+        while( sqlite3Isdigit(z[i]) ){ i++; }
+        *tokenType = TK_FLOAT;
+      }
+      if( (z[i]=='e' || z[i]=='E') &&
+           ( sqlite3Isdigit(z[i+1]) 
+            || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2]))
+           )
+      ){
+        i += 2;
+        while( sqlite3Isdigit(z[i]) ){ i++; }
+        *tokenType = TK_FLOAT;
+      }
+      while( IdChar(z[i]) ){
+        *tokenType = TK_ILLEGAL;
+        i++;
+      }
+      return i;
+    }
+    case CC_QUOTE2: {
+      for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){}
+      *tokenType = c==']' ? TK_ID : TK_ILLEGAL;
+      return i;
+    }
+    case CC_VARNUM: {
+      *tokenType = TK_VARIABLE;
+      for(i=1; sqlite3Isdigit(z[i]); i++){}
+      return i;
+    }
+    case CC_DOLLAR:
+    case CC_VARALPHA: {
+      int n = 0;
+      testcase( z[0]=='$' );  testcase( z[0]=='@' );
+      testcase( z[0]==':' );  testcase( z[0]=='#' );
+      *tokenType = TK_VARIABLE;
+      for(i=1; (c=z[i])!=0; i++){
+        if( IdChar(c) ){
+          n++;
+        }else if( c=='(' && n>0 ){
+          do{
+            i++;
+          }while( (c=z[i])!=0 && !sqlite3Isspace(c) && c!=')' );
+          if( c==')' ){
+            i++;
+          }else{
+            *tokenType = TK_ILLEGAL;
+          }
+          break;
+        }else if( c==':' && z[i+1]==':' ){
+          i++;
+        }else{
+          break;
+        }
+      }
+      if( n==0 ) *tokenType = TK_ILLEGAL;
+      return i;
+    }
+    case CC_KYWD: {
+      for(i=1; aiClass[z[i]]<=CC_KYWD; i++){}
+      if( IdChar(z[i]) ){
+        /* This token started out using characters that can appear in keywords,
+        ** but z[i] is a character not allowed within keywords, so this must
+        ** be an identifier instead */
+        i++;
+        break;
+      }
+      *tokenType = TK_ID;
+      return i;
+    }
+    case CC_X: {
+      testcase( z[0]=='x' ); testcase( z[0]=='X' );
+      if( z[1]=='\'' ){
+        *tokenType = TK_BLOB;
+        for(i=2; sqlite3Isxdigit(z[i]); i++){}
+        if( z[i]!='\'' || i%2 ){
+          *tokenType = TK_ILLEGAL;
+          while( z[i] && z[i]!='\'' ){ i++; }
+        }
+        if( z[i] ) i++;
+        return i;
+      }
+      /* If it is not a BLOB literal, then it must be an ID, since no
+      ** SQL keywords start with the letter 'x'.  Fall through */
+    }
+    case CC_ID: {
+      i = 1;
+      break;
+    }
+    default: {
+      *tokenType = TK_ILLEGAL;
+      return 1;
+    }
+  }
+  while( IdChar(z[i]) ){ i++; }
+  *tokenType = TK_ID;
+  return i;
+}
+
+char *sqlite3_normalize(const char *zSql){
+  char *z;              /* The output string */
+  sqlite3_int64 nZ;     /* Size of the output string in bytes */
+  sqlite3_int64 nSql;   /* Size of the input string in bytes */
+  int i;                /* Next character to read from zSql[] */
+  int j;                /* Next slot to fill in on z[] */
+  int tokenType;        /* Type of the next token */
+  int n;                /* Size of the next token */
+  int k;                /* Loop counter */
+
+  nSql = strlen(zSql);
+  nZ = nSql;
+  z = sqlite3_malloc64( nZ+2 );
+  if( z==0 ) return 0;
+  for(i=j=0; zSql[i]; i += n){
+    n = sqlite3GetToken((unsigned char*)zSql+i, &tokenType);
+    switch( tokenType ){
+      case TK_SPACE: {
+        break;
+      }
+      case TK_ERROR: {
+        sqlite3_free(z);
+        return 0;
+      }
+      case TK_LITERAL: {
+        z[j++] = '?';
+        break;
+      }
+      case TK_PUNCT:
+      case TK_NAME: {
+        if( n==4 && sqlite3_strnicmp(zSql+i,"NULL",4)==0 ){
+          if( (j>=3 && strncmp(z+j-2,"is",2)==0 && !IdChar(z[j-3]))
+           || (j>=4 && strncmp(z+j-3,"not",3)==0 && !IdChar(z[j-4]))
+          ){
+            /* NULL is a keyword in this case, not a literal value */
+          }else{
+            /* Here the NULL is a literal value */
+            z[j++] = '?';
+            break;
+          }
+        }
+        if( j>0 && IdChar(z[j-1]) && IdChar(zSql[i]) ) z[j++] = ' ';
+        for(k=0; k<n; k++){
+          z[j++] = sqlite3Tolower(zSql[i+k]);
+        }
+        break;
+      }
+    }
+  }
+  while( j>0 && z[j-1]==' ' ){ j--; }
+  if( j>0 && z[j-1]!=';' ){ z[j++] = ';'; }
+  z[j] = 0;
+
+  /* Make a second pass converting "in(...)" where the "..." is not a
+  ** SELECT statement into "in(?,?,?)" */
+  for(i=0; i<j; i=n){
+    char *zIn = strstr(z+i, "in(");
+    int nParen;
+    if( zIn==0 ) break;
+    n = (int)(zIn-z)+3;  /* Index of first char past "in(" */
+    if( n && IdChar(zIn[-1]) ) continue;
+    if( strncmp(zIn, "in(select",9)==0 && !IdChar(zIn[9]) ) continue;
+    if( strncmp(zIn, "in(with",7)==0 && !IdChar(zIn[7]) ) continue;
+    for(nParen=1, k=0; z[n+k]; k++){
+      if( z[n+k]=='(' ) nParen++;
+      if( z[n+k]==')' ){
+        nParen--;
+        if( nParen==0 ) break;
+      }
+    }
+    /* k is the number of bytes in the "..." within "in(...)" */
+    if( k<5 ){
+      z = sqlite3_realloc64(z, j+(5-k)+1);
+      if( z==0 ) return 0;
+      memmove(z+n+5, z+n+k, j-(n+k));
+    }else if( k>5 ){
+      memmove(z+n+5, z+n+k, j-(n+k));
+    }
+    j = j-k+5;
+    z[j] = 0;
+    memcpy(z+n, "?,?,?", 5);
+  }
+  return z;
+}
+
+/*
+** For testing purposes, or to build a stand-alone SQL normalizer program,
+** compile this one source file with the -DSQLITE_NORMALIZE_CLI and link
+** it against any SQLite library.  The resulting command-line program will
+** run sqlite3_normalize() over the text of all files named on the command-
+** line and show the result on standard output.
+*/
+#ifdef SQLITE_NORMALIZE_CLI
+#include <stdio.h>
+#include <stdlib.h>
+
+/*
+** Break zIn up into separate SQL statements and run sqlite3_normalize()
+** on each one.  Print the result of each run.
+*/
+static void normalizeFile(char *zIn){
+  int i;
+  if( zIn==0 ) return;
+  for(i=0; zIn[i]; i++){
+    char cSaved;
+    if( zIn[i]!=';' ) continue;
+    cSaved = zIn[i+1];
+    zIn[i+1] = 0;
+    if( sqlite3_complete(zIn) ){
+      char *zOut = sqlite3_normalize(zIn);
+      if( zOut ){
+        printf("%s\n", zOut);
+        sqlite3_free(zOut);
+      }else{
+        fprintf(stderr, "ERROR: %s\n", zIn);
+      }
+      zIn[i+1] = cSaved;
+      zIn += i+1;
+      i = -1;
+    }else{
+      zIn[i+1] = cSaved;
+    }
+  }
+}
+
+/*
+** The main routine for "sql_normalize".  Read files named on the
+** command-line and run the text of each through sqlite3_normalize().
+*/
+int main(int argc, char **argv){
+  int i;
+  FILE *in;
+  char *zBuf = 0;
+  sqlite3_int64 sz, got;
+
+  for(i=1; i<argc; i++){
+    in = fopen(argv[i], "rb");
+    if( in==0 ){
+      fprintf(stderr, "cannot open \"%s\"\n", argv[i]);
+      continue;
+    }
+    fseek(in, 0, SEEK_END);
+    sz = ftell(in);
+    rewind(in);
+    zBuf = sqlite3_realloc64(zBuf, sz+1);
+    if( zBuf==0 ){
+      fprintf(stderr, "failed to malloc for %lld bytes\n", sz);
+      exit(1);
+    }
+    got = fread(zBuf, 1, sz, in);
+    fclose(in);
+    if( got!=sz ){
+      fprintf(stderr, "only able to read %lld of %lld bytes from \"%s\"\n",
+              got, sz, argv[i]);
+    }else{
+      zBuf[got] = 0;
+      normalizeFile(zBuf);
+    }
+  }
+  sqlite3_free(zBuf);
+}
+#endif /* SQLITE_NORMALIZE_CLI */

ext/misc/rot13.c

@@ -47,9 +47,9 @@ static void rot13func(
   const unsigned char *zIn;
   int nIn;
   unsigned char *zOut;
-  char *zToFree = 0;
+  unsigned char *zToFree = 0;
   int i;
-  char zTemp[100];
+  unsigned char zTemp[100];
   assert( argc==1 );
   if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return;
   zIn = (const unsigned char*)sqlite3_value_text(argv[0]);
@@ -57,7 +57,7 @@ static void rot13func(
   if( nIn<sizeof(zTemp)-1 ){
     zOut = zTemp;
   }else{
-    zOut = zToFree = sqlite3_malloc( nIn+1 );
+    zOut = zToFree = (unsigned char*)sqlite3_malloc64( nIn+1 );
     if( zOut==0 ){
       sqlite3_result_error_nomem(context);
       return;

ext/misc/scrub.c

@@ -131,7 +131,7 @@ static void scrubBackupWrite(ScrubState *p, int pgno, const u8 *pData){
     scrubBackupErr(p, "write failed for page %d", pgno);
     p->rcErr = SQLITE_IOERR;
   }
-  if( pgno>p->iLastPage ) p->iLastPage = pgno;
+  if( (u32)pgno>p->iLastPage ) p->iLastPage = pgno;
 }
 
 /* Prepare a statement against the "db" database. */
@@ -459,7 +459,7 @@ static void scrubBackupBtree(ScrubState *p, int pgno, int iDepth){
     nLocal = K<=X ? K : M;
     if( pc+nLocal > p->szUsable-4 ){ ln=__LINE__; goto btree_corrupt; }
     iChild = scrubBackupInt32(&a[pc+nLocal]);
-    scrubBackupOverflow(p, iChild, P-nLocal);
+    scrubBackupOverflow(p, iChild, (u32)(P-nLocal));
   }
 
   /* Walk the right-most tree */

ext/misc/series.c

@@ -270,6 +270,15 @@ static int seriesFilter(
   }else{
     pCur->iStep = 1;
   }
+  for(i=0; i<argc; i++){
+    if( sqlite3_value_type(argv[i])==SQLITE_NULL ){
+      /* If any of the constraints have a NULL value, then return no rows.
+      ** See ticket https://www.sqlite.org/src/info/fac496b61722daf2 */
+      pCur->mnValue = 1;
+      pCur->mxValue = 0;
+      break;
+    }
+  }
   if( idxNum & 8 ){
     pCur->isDesc = 1;
     pCur->iValue = pCur->mxValue;
@@ -304,44 +313,45 @@ static int seriesBestIndex(
   sqlite3_vtab *tab,
   sqlite3_index_info *pIdxInfo
 ){
-  int i;                 /* Loop over constraints */
+  int i, j;              /* Loop over constraints */
   int idxNum = 0;        /* The query plan bitmask */
-  int startIdx = -1;     /* Index of the start= constraint, or -1 if none */
+  int unusableMask = 0;  /* Mask of unusable constraints */
-  int stopIdx = -1;      /* Index of the stop= constraint, or -1 if none */
-  int stepIdx = -1;      /* Index of the step= constraint, or -1 if none */
   int nArg = 0;          /* Number of arguments that seriesFilter() expects */
-
+  int aIdx[3];           /* Constraints on start, stop, and step */
   const struct sqlite3_index_constraint *pConstraint;
+
+  /* This implementation assumes that the start, stop, and step columns
+  ** are the last three columns in the virtual table. */
+  assert( SERIES_COLUMN_STOP == SERIES_COLUMN_START+1 );
+  assert( SERIES_COLUMN_STEP == SERIES_COLUMN_START+2 );
+  aIdx[0] = aIdx[1] = aIdx[2] = -1;
   pConstraint = pIdxInfo->aConstraint;
   for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){
-    if( pConstraint->usable==0 ) continue;
+    int iCol;    /* 0 for start, 1 for stop, 2 for step */
-    if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;
+    int iMask;   /* bitmask for those column */
-    switch( pConstraint->iColumn ){
+    if( pConstraint->iColumn<SERIES_COLUMN_START ) continue;
-      case SERIES_COLUMN_START:
+    iCol = pConstraint->iColumn - SERIES_COLUMN_START;
-        startIdx = i;
+    assert( iCol>=0 && iCol<=2 );
-        idxNum |= 1;
+    iMask = 1 << iCol;
-        break;
+    if( pConstraint->usable==0 ){
-      case SERIES_COLUMN_STOP:
+      unusableMask |=  iMask;
-        stopIdx = i;
+      continue;
-        idxNum |= 2;
+    }else if( pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){
-        break;
+      idxNum |= iMask;
-      case SERIES_COLUMN_STEP:
+      aIdx[iCol] = i;
-        stepIdx = i;
-        idxNum |= 4;
-        break;
     }
   }
-  if( startIdx>=0 ){
+  for(i=0; i<3; i++){
-    pIdxInfo->aConstraintUsage[startIdx].argvIndex = ++nArg;
+    if( (j = aIdx[i])>=0 ){
-    pIdxInfo->aConstraintUsage[startIdx].omit= !SQLITE_SERIES_CONSTRAINT_VERIFY;
+      pIdxInfo->aConstraintUsage[j].argvIndex = ++nArg;
+      pIdxInfo->aConstraintUsage[j].omit = !SQLITE_SERIES_CONSTRAINT_VERIFY;
+    }
   }
-  if( stopIdx>=0 ){
+  if( (unusableMask & ~idxNum)!=0 ){
-    pIdxInfo->aConstraintUsage[stopIdx].argvIndex = ++nArg;
+    /* The start, stop, and step columns are inputs.  Therefore if there
-    pIdxInfo->aConstraintUsage[stopIdx].omit = !SQLITE_SERIES_CONSTRAINT_VERIFY;
+    ** are unusable constraints on any of start, stop, or step then
-  }
+    ** this plan is unusable */
-  if( stepIdx>=0 ){
+    return SQLITE_CONSTRAINT;
-    pIdxInfo->aConstraintUsage[stepIdx].argvIndex = ++nArg;
-    pIdxInfo->aConstraintUsage[stepIdx].omit = !SQLITE_SERIES_CONSTRAINT_VERIFY;
   }
   if( (idxNum & 3)==3 ){
     /* Both start= and stop= boundaries are available.  This is the 
@@ -356,7 +366,6 @@ static int seriesBestIndex(
     /* If either boundary is missing, we have to generate a huge span
     ** of numbers.  Make this case very expensive so that the query
     ** planner will work hard to avoid it. */
-    pIdxInfo->estimatedCost = (double)2147483647;
     pIdxInfo->estimatedRows = 2147483647;
   }
   pIdxInfo->idxNum = idxNum;

ext/misc/sha1.c

@@ -10,7 +10,7 @@
 **
 ******************************************************************************
 **
-** This SQLite extension implements a functions that compute SHA1 hashes.
+** This SQLite extension implements functions that compute SHA1 hashes.
 ** Two SQL functions are implemented:
 **
 **     sha1(X)

ext/misc/shathree.c

@@ -10,7 +10,7 @@
 **
 ******************************************************************************
 **
-** This SQLite extension implements a functions that compute SHA1 hashes.
+** This SQLite extension implements functions that compute SHA3 hashes.
 ** Two SQL functions are implemented:
 **
 **     sha3(X,SIZE)
@@ -78,9 +78,9 @@ struct SHA3Context {
 */
 static void KeccakF1600Step(SHA3Context *p){
   int i;
-  u64 B0, B1, B2, B3, B4;
+  u64 b0, b1, b2, b3, b4;
-  u64 C0, C1, C2, C3, C4;
+  u64 c0, c1, c2, c3, c4;
-  u64 D0, D1, D2, D3, D4;
+  u64 d0, d1, d2, d3, d4;
   static const u64 RC[] = {
     0x0000000000000001ULL,  0x0000000000008082ULL,
     0x800000000000808aULL,  0x8000000080008000ULL,
@@ -95,301 +95,301 @@ static void KeccakF1600Step(SHA3Context *p){
     0x8000000080008081ULL,  0x8000000000008080ULL,
     0x0000000080000001ULL,  0x8000000080008008ULL
   };
-# define A00 (p->u.s[0])
+# define a00 (p->u.s[0])
-# define A01 (p->u.s[1])
+# define a01 (p->u.s[1])
-# define A02 (p->u.s[2])
+# define a02 (p->u.s[2])
-# define A03 (p->u.s[3])
+# define a03 (p->u.s[3])
-# define A04 (p->u.s[4])
+# define a04 (p->u.s[4])
-# define A10 (p->u.s[5])
+# define a10 (p->u.s[5])
-# define A11 (p->u.s[6])
+# define a11 (p->u.s[6])
-# define A12 (p->u.s[7])
+# define a12 (p->u.s[7])
-# define A13 (p->u.s[8])
+# define a13 (p->u.s[8])
-# define A14 (p->u.s[9])
+# define a14 (p->u.s[9])
-# define A20 (p->u.s[10])
+# define a20 (p->u.s[10])
-# define A21 (p->u.s[11])
+# define a21 (p->u.s[11])
-# define A22 (p->u.s[12])
+# define a22 (p->u.s[12])
-# define A23 (p->u.s[13])
+# define a23 (p->u.s[13])
-# define A24 (p->u.s[14])
+# define a24 (p->u.s[14])
-# define A30 (p->u.s[15])
+# define a30 (p->u.s[15])
-# define A31 (p->u.s[16])
+# define a31 (p->u.s[16])
-# define A32 (p->u.s[17])
+# define a32 (p->u.s[17])
-# define A33 (p->u.s[18])
+# define a33 (p->u.s[18])
-# define A34 (p->u.s[19])
+# define a34 (p->u.s[19])
-# define A40 (p->u.s[20])
+# define a40 (p->u.s[20])
-# define A41 (p->u.s[21])
+# define a41 (p->u.s[21])
-# define A42 (p->u.s[22])
+# define a42 (p->u.s[22])
-# define A43 (p->u.s[23])
+# define a43 (p->u.s[23])
-# define A44 (p->u.s[24])
+# define a44 (p->u.s[24])
 # define ROL64(a,x) ((a<<x)|(a>>(64-x)))
 
   for(i=0; i<24; i+=4){
-    C0 = A00^A10^A20^A30^A40;
+    c0 = a00^a10^a20^a30^a40;
-    C1 = A01^A11^A21^A31^A41;
+    c1 = a01^a11^a21^a31^a41;
-    C2 = A02^A12^A22^A32^A42;
+    c2 = a02^a12^a22^a32^a42;
-    C3 = A03^A13^A23^A33^A43;
+    c3 = a03^a13^a23^a33^a43;
-    C4 = A04^A14^A24^A34^A44;
+    c4 = a04^a14^a24^a34^a44;
-    D0 = C4^ROL64(C1, 1);
+    d0 = c4^ROL64(c1, 1);
-    D1 = C0^ROL64(C2, 1);
+    d1 = c0^ROL64(c2, 1);
-    D2 = C1^ROL64(C3, 1);
+    d2 = c1^ROL64(c3, 1);
-    D3 = C2^ROL64(C4, 1);
+    d3 = c2^ROL64(c4, 1);
-    D4 = C3^ROL64(C0, 1);
+    d4 = c3^ROL64(c0, 1);
 
-    B0 = (A00^D0);
+    b0 = (a00^d0);
-    B1 = ROL64((A11^D1), 44);
+    b1 = ROL64((a11^d1), 44);
-    B2 = ROL64((A22^D2), 43);
+    b2 = ROL64((a22^d2), 43);
-    B3 = ROL64((A33^D3), 21);
+    b3 = ROL64((a33^d3), 21);
-    B4 = ROL64((A44^D4), 14);
+    b4 = ROL64((a44^d4), 14);
-    A00 =   B0 ^((~B1)&  B2 );
+    a00 =   b0 ^((~b1)&  b2 );
-    A00 ^= RC[i];
+    a00 ^= RC[i];
-    A11 =   B1 ^((~B2)&  B3 );
+    a11 =   b1 ^((~b2)&  b3 );
-    A22 =   B2 ^((~B3)&  B4 );
+    a22 =   b2 ^((~b3)&  b4 );
-    A33 =   B3 ^((~B4)&  B0 );
+    a33 =   b3 ^((~b4)&  b0 );
-    A44 =   B4 ^((~B0)&  B1 );
+    a44 =   b4 ^((~b0)&  b1 );
 
-    B2 = ROL64((A20^D0), 3);
+    b2 = ROL64((a20^d0), 3);
-    B3 = ROL64((A31^D1), 45);
+    b3 = ROL64((a31^d1), 45);
-    B4 = ROL64((A42^D2), 61);
+    b4 = ROL64((a42^d2), 61);
-    B0 = ROL64((A03^D3), 28);
+    b0 = ROL64((a03^d3), 28);
-    B1 = ROL64((A14^D4), 20);
+    b1 = ROL64((a14^d4), 20);
-    A20 =   B0 ^((~B1)&  B2 );
+    a20 =   b0 ^((~b1)&  b2 );
-    A31 =   B1 ^((~B2)&  B3 );
+    a31 =   b1 ^((~b2)&  b3 );
-    A42 =   B2 ^((~B3)&  B4 );
+    a42 =   b2 ^((~b3)&  b4 );
-    A03 =   B3 ^((~B4)&  B0 );
+    a03 =   b3 ^((~b4)&  b0 );
-    A14 =   B4 ^((~B0)&  B1 );
+    a14 =   b4 ^((~b0)&  b1 );
 
-    B4 = ROL64((A40^D0), 18);
+    b4 = ROL64((a40^d0), 18);
-    B0 = ROL64((A01^D1), 1);
+    b0 = ROL64((a01^d1), 1);
-    B1 = ROL64((A12^D2), 6);
+    b1 = ROL64((a12^d2), 6);
-    B2 = ROL64((A23^D3), 25);
+    b2 = ROL64((a23^d3), 25);
-    B3 = ROL64((A34^D4), 8);
+    b3 = ROL64((a34^d4), 8);
-    A40 =   B0 ^((~B1)&  B2 );
+    a40 =   b0 ^((~b1)&  b2 );
-    A01 =   B1 ^((~B2)&  B3 );
+    a01 =   b1 ^((~b2)&  b3 );
-    A12 =   B2 ^((~B3)&  B4 );
+    a12 =   b2 ^((~b3)&  b4 );
-    A23 =   B3 ^((~B4)&  B0 );
+    a23 =   b3 ^((~b4)&  b0 );
-    A34 =   B4 ^((~B0)&  B1 );
+    a34 =   b4 ^((~b0)&  b1 );
 
-    B1 = ROL64((A10^D0), 36);
+    b1 = ROL64((a10^d0), 36);
-    B2 = ROL64((A21^D1), 10);
+    b2 = ROL64((a21^d1), 10);
-    B3 = ROL64((A32^D2), 15);
+    b3 = ROL64((a32^d2), 15);
-    B4 = ROL64((A43^D3), 56);
+    b4 = ROL64((a43^d3), 56);
-    B0 = ROL64((A04^D4), 27);
+    b0 = ROL64((a04^d4), 27);
-    A10 =   B0 ^((~B1)&  B2 );
+    a10 =   b0 ^((~b1)&  b2 );
-    A21 =   B1 ^((~B2)&  B3 );
+    a21 =   b1 ^((~b2)&  b3 );
-    A32 =   B2 ^((~B3)&  B4 );
+    a32 =   b2 ^((~b3)&  b4 );
-    A43 =   B3 ^((~B4)&  B0 );
+    a43 =   b3 ^((~b4)&  b0 );
-    A04 =   B4 ^((~B0)&  B1 );
+    a04 =   b4 ^((~b0)&  b1 );
 
-    B3 = ROL64((A30^D0), 41);
+    b3 = ROL64((a30^d0), 41);
-    B4 = ROL64((A41^D1), 2);
+    b4 = ROL64((a41^d1), 2);
-    B0 = ROL64((A02^D2), 62);
+    b0 = ROL64((a02^d2), 62);
-    B1 = ROL64((A13^D3), 55);
+    b1 = ROL64((a13^d3), 55);
-    B2 = ROL64((A24^D4), 39);
+    b2 = ROL64((a24^d4), 39);
-    A30 =   B0 ^((~B1)&  B2 );
+    a30 =   b0 ^((~b1)&  b2 );
-    A41 =   B1 ^((~B2)&  B3 );
+    a41 =   b1 ^((~b2)&  b3 );
-    A02 =   B2 ^((~B3)&  B4 );
+    a02 =   b2 ^((~b3)&  b4 );
-    A13 =   B3 ^((~B4)&  B0 );
+    a13 =   b3 ^((~b4)&  b0 );
-    A24 =   B4 ^((~B0)&  B1 );
+    a24 =   b4 ^((~b0)&  b1 );
 
-    C0 = A00^A20^A40^A10^A30;
+    c0 = a00^a20^a40^a10^a30;
-    C1 = A11^A31^A01^A21^A41;
+    c1 = a11^a31^a01^a21^a41;
-    C2 = A22^A42^A12^A32^A02;
+    c2 = a22^a42^a12^a32^a02;
-    C3 = A33^A03^A23^A43^A13;
+    c3 = a33^a03^a23^a43^a13;
-    C4 = A44^A14^A34^A04^A24;
+    c4 = a44^a14^a34^a04^a24;
-    D0 = C4^ROL64(C1, 1);
+    d0 = c4^ROL64(c1, 1);
-    D1 = C0^ROL64(C2, 1);
+    d1 = c0^ROL64(c2, 1);
-    D2 = C1^ROL64(C3, 1);
+    d2 = c1^ROL64(c3, 1);
-    D3 = C2^ROL64(C4, 1);
+    d3 = c2^ROL64(c4, 1);
-    D4 = C3^ROL64(C0, 1);
+    d4 = c3^ROL64(c0, 1);
 
-    B0 = (A00^D0);
+    b0 = (a00^d0);
-    B1 = ROL64((A31^D1), 44);
+    b1 = ROL64((a31^d1), 44);
-    B2 = ROL64((A12^D2), 43);
+    b2 = ROL64((a12^d2), 43);
-    B3 = ROL64((A43^D3), 21);
+    b3 = ROL64((a43^d3), 21);
-    B4 = ROL64((A24^D4), 14);
+    b4 = ROL64((a24^d4), 14);
-    A00 =   B0 ^((~B1)&  B2 );
+    a00 =   b0 ^((~b1)&  b2 );
-    A00 ^= RC[i+1];
+    a00 ^= RC[i+1];
-    A31 =   B1 ^((~B2)&  B3 );
+    a31 =   b1 ^((~b2)&  b3 );
-    A12 =   B2 ^((~B3)&  B4 );
+    a12 =   b2 ^((~b3)&  b4 );
-    A43 =   B3 ^((~B4)&  B0 );
+    a43 =   b3 ^((~b4)&  b0 );
-    A24 =   B4 ^((~B0)&  B1 );
+    a24 =   b4 ^((~b0)&  b1 );
 
-    B2 = ROL64((A40^D0), 3);
+    b2 = ROL64((a40^d0), 3);
-    B3 = ROL64((A21^D1), 45);
+    b3 = ROL64((a21^d1), 45);
-    B4 = ROL64((A02^D2), 61);
+    b4 = ROL64((a02^d2), 61);
-    B0 = ROL64((A33^D3), 28);
+    b0 = ROL64((a33^d3), 28);
-    B1 = ROL64((A14^D4), 20);
+    b1 = ROL64((a14^d4), 20);
-    A40 =   B0 ^((~B1)&  B2 );
+    a40 =   b0 ^((~b1)&  b2 );
-    A21 =   B1 ^((~B2)&  B3 );
+    a21 =   b1 ^((~b2)&  b3 );
-    A02 =   B2 ^((~B3)&  B4 );
+    a02 =   b2 ^((~b3)&  b4 );
-    A33 =   B3 ^((~B4)&  B0 );
+    a33 =   b3 ^((~b4)&  b0 );
-    A14 =   B4 ^((~B0)&  B1 );
+    a14 =   b4 ^((~b0)&  b1 );
 
-    B4 = ROL64((A30^D0), 18);
+    b4 = ROL64((a30^d0), 18);
-    B0 = ROL64((A11^D1), 1);
+    b0 = ROL64((a11^d1), 1);
-    B1 = ROL64((A42^D2), 6);
+    b1 = ROL64((a42^d2), 6);
-    B2 = ROL64((A23^D3), 25);
+    b2 = ROL64((a23^d3), 25);
-    B3 = ROL64((A04^D4), 8);
+    b3 = ROL64((a04^d4), 8);
-    A30 =   B0 ^((~B1)&  B2 );
+    a30 =   b0 ^((~b1)&  b2 );
-    A11 =   B1 ^((~B2)&  B3 );
+    a11 =   b1 ^((~b2)&  b3 );
-    A42 =   B2 ^((~B3)&  B4 );
+    a42 =   b2 ^((~b3)&  b4 );
-    A23 =   B3 ^((~B4)&  B0 );
+    a23 =   b3 ^((~b4)&  b0 );
-    A04 =   B4 ^((~B0)&  B1 );
+    a04 =   b4 ^((~b0)&  b1 );
 
-    B1 = ROL64((A20^D0), 36);
+    b1 = ROL64((a20^d0), 36);
-    B2 = ROL64((A01^D1), 10);
+    b2 = ROL64((a01^d1), 10);
-    B3 = ROL64((A32^D2), 15);
+    b3 = ROL64((a32^d2), 15);
-    B4 = ROL64((A13^D3), 56);
+    b4 = ROL64((a13^d3), 56);
-    B0 = ROL64((A44^D4), 27);
+    b0 = ROL64((a44^d4), 27);
-    A20 =   B0 ^((~B1)&  B2 );
+    a20 =   b0 ^((~b1)&  b2 );
-    A01 =   B1 ^((~B2)&  B3 );
+    a01 =   b1 ^((~b2)&  b3 );
-    A32 =   B2 ^((~B3)&  B4 );
+    a32 =   b2 ^((~b3)&  b4 );
-    A13 =   B3 ^((~B4)&  B0 );
+    a13 =   b3 ^((~b4)&  b0 );
-    A44 =   B4 ^((~B0)&  B1 );
+    a44 =   b4 ^((~b0)&  b1 );
 
-    B3 = ROL64((A10^D0), 41);
+    b3 = ROL64((a10^d0), 41);
-    B4 = ROL64((A41^D1), 2);
+    b4 = ROL64((a41^d1), 2);
-    B0 = ROL64((A22^D2), 62);
+    b0 = ROL64((a22^d2), 62);
-    B1 = ROL64((A03^D3), 55);
+    b1 = ROL64((a03^d3), 55);
-    B2 = ROL64((A34^D4), 39);
+    b2 = ROL64((a34^d4), 39);
-    A10 =   B0 ^((~B1)&  B2 );
+    a10 =   b0 ^((~b1)&  b2 );
-    A41 =   B1 ^((~B2)&  B3 );
+    a41 =   b1 ^((~b2)&  b3 );
-    A22 =   B2 ^((~B3)&  B4 );
+    a22 =   b2 ^((~b3)&  b4 );
-    A03 =   B3 ^((~B4)&  B0 );
+    a03 =   b3 ^((~b4)&  b0 );
-    A34 =   B4 ^((~B0)&  B1 );
+    a34 =   b4 ^((~b0)&  b1 );
 
-    C0 = A00^A40^A30^A20^A10;
+    c0 = a00^a40^a30^a20^a10;
-    C1 = A31^A21^A11^A01^A41;
+    c1 = a31^a21^a11^a01^a41;
-    C2 = A12^A02^A42^A32^A22;
+    c2 = a12^a02^a42^a32^a22;
-    C3 = A43^A33^A23^A13^A03;
+    c3 = a43^a33^a23^a13^a03;
-    C4 = A24^A14^A04^A44^A34;
+    c4 = a24^a14^a04^a44^a34;
-    D0 = C4^ROL64(C1, 1);
+    d0 = c4^ROL64(c1, 1);
-    D1 = C0^ROL64(C2, 1);
+    d1 = c0^ROL64(c2, 1);
-    D2 = C1^ROL64(C3, 1);
+    d2 = c1^ROL64(c3, 1);
-    D3 = C2^ROL64(C4, 1);
+    d3 = c2^ROL64(c4, 1);
-    D4 = C3^ROL64(C0, 1);
+    d4 = c3^ROL64(c0, 1);
 
-    B0 = (A00^D0);
+    b0 = (a00^d0);
-    B1 = ROL64((A21^D1), 44);
+    b1 = ROL64((a21^d1), 44);
-    B2 = ROL64((A42^D2), 43);
+    b2 = ROL64((a42^d2), 43);
-    B3 = ROL64((A13^D3), 21);
+    b3 = ROL64((a13^d3), 21);
-    B4 = ROL64((A34^D4), 14);
+    b4 = ROL64((a34^d4), 14);
-    A00 =   B0 ^((~B1)&  B2 );
+    a00 =   b0 ^((~b1)&  b2 );
-    A00 ^= RC[i+2];
+    a00 ^= RC[i+2];
-    A21 =   B1 ^((~B2)&  B3 );
+    a21 =   b1 ^((~b2)&  b3 );
-    A42 =   B2 ^((~B3)&  B4 );
+    a42 =   b2 ^((~b3)&  b4 );
-    A13 =   B3 ^((~B4)&  B0 );
+    a13 =   b3 ^((~b4)&  b0 );
-    A34 =   B4 ^((~B0)&  B1 );
+    a34 =   b4 ^((~b0)&  b1 );
 
-    B2 = ROL64((A30^D0), 3);
+    b2 = ROL64((a30^d0), 3);
-    B3 = ROL64((A01^D1), 45);
+    b3 = ROL64((a01^d1), 45);
-    B4 = ROL64((A22^D2), 61);
+    b4 = ROL64((a22^d2), 61);
-    B0 = ROL64((A43^D3), 28);
+    b0 = ROL64((a43^d3), 28);
-    B1 = ROL64((A14^D4), 20);
+    b1 = ROL64((a14^d4), 20);
-    A30 =   B0 ^((~B1)&  B2 );
+    a30 =   b0 ^((~b1)&  b2 );
-    A01 =   B1 ^((~B2)&  B3 );
+    a01 =   b1 ^((~b2)&  b3 );
-    A22 =   B2 ^((~B3)&  B4 );
+    a22 =   b2 ^((~b3)&  b4 );
-    A43 =   B3 ^((~B4)&  B0 );
+    a43 =   b3 ^((~b4)&  b0 );
-    A14 =   B4 ^((~B0)&  B1 );
+    a14 =   b4 ^((~b0)&  b1 );
 
-    B4 = ROL64((A10^D0), 18);
+    b4 = ROL64((a10^d0), 18);
-    B0 = ROL64((A31^D1), 1);
+    b0 = ROL64((a31^d1), 1);
-    B1 = ROL64((A02^D2), 6);
+    b1 = ROL64((a02^d2), 6);
-    B2 = ROL64((A23^D3), 25);
+    b2 = ROL64((a23^d3), 25);
-    B3 = ROL64((A44^D4), 8);
+    b3 = ROL64((a44^d4), 8);
-    A10 =   B0 ^((~B1)&  B2 );
+    a10 =   b0 ^((~b1)&  b2 );
-    A31 =   B1 ^((~B2)&  B3 );
+    a31 =   b1 ^((~b2)&  b3 );
-    A02 =   B2 ^((~B3)&  B4 );
+    a02 =   b2 ^((~b3)&  b4 );
-    A23 =   B3 ^((~B4)&  B0 );
+    a23 =   b3 ^((~b4)&  b0 );
-    A44 =   B4 ^((~B0)&  B1 );
+    a44 =   b4 ^((~b0)&  b1 );
 
-    B1 = ROL64((A40^D0), 36);
+    b1 = ROL64((a40^d0), 36);
-    B2 = ROL64((A11^D1), 10);
+    b2 = ROL64((a11^d1), 10);
-    B3 = ROL64((A32^D2), 15);
+    b3 = ROL64((a32^d2), 15);
-    B4 = ROL64((A03^D3), 56);
+    b4 = ROL64((a03^d3), 56);
-    B0 = ROL64((A24^D4), 27);
+    b0 = ROL64((a24^d4), 27);
-    A40 =   B0 ^((~B1)&  B2 );
+    a40 =   b0 ^((~b1)&  b2 );
-    A11 =   B1 ^((~B2)&  B3 );
+    a11 =   b1 ^((~b2)&  b3 );
-    A32 =   B2 ^((~B3)&  B4 );
+    a32 =   b2 ^((~b3)&  b4 );
-    A03 =   B3 ^((~B4)&  B0 );
+    a03 =   b3 ^((~b4)&  b0 );
-    A24 =   B4 ^((~B0)&  B1 );
+    a24 =   b4 ^((~b0)&  b1 );
 
-    B3 = ROL64((A20^D0), 41);
+    b3 = ROL64((a20^d0), 41);
-    B4 = ROL64((A41^D1), 2);
+    b4 = ROL64((a41^d1), 2);
-    B0 = ROL64((A12^D2), 62);
+    b0 = ROL64((a12^d2), 62);
-    B1 = ROL64((A33^D3), 55);
+    b1 = ROL64((a33^d3), 55);
-    B2 = ROL64((A04^D4), 39);
+    b2 = ROL64((a04^d4), 39);
-    A20 =   B0 ^((~B1)&  B2 );
+    a20 =   b0 ^((~b1)&  b2 );
-    A41 =   B1 ^((~B2)&  B3 );
+    a41 =   b1 ^((~b2)&  b3 );
-    A12 =   B2 ^((~B3)&  B4 );
+    a12 =   b2 ^((~b3)&  b4 );
-    A33 =   B3 ^((~B4)&  B0 );
+    a33 =   b3 ^((~b4)&  b0 );
-    A04 =   B4 ^((~B0)&  B1 );
+    a04 =   b4 ^((~b0)&  b1 );
 
-    C0 = A00^A30^A10^A40^A20;
+    c0 = a00^a30^a10^a40^a20;
-    C1 = A21^A01^A31^A11^A41;
+    c1 = a21^a01^a31^a11^a41;
-    C2 = A42^A22^A02^A32^A12;
+    c2 = a42^a22^a02^a32^a12;
-    C3 = A13^A43^A23^A03^A33;
+    c3 = a13^a43^a23^a03^a33;
-    C4 = A34^A14^A44^A24^A04;
+    c4 = a34^a14^a44^a24^a04;
-    D0 = C4^ROL64(C1, 1);
+    d0 = c4^ROL64(c1, 1);
-    D1 = C0^ROL64(C2, 1);
+    d1 = c0^ROL64(c2, 1);
-    D2 = C1^ROL64(C3, 1);
+    d2 = c1^ROL64(c3, 1);
-    D3 = C2^ROL64(C4, 1);
+    d3 = c2^ROL64(c4, 1);
-    D4 = C3^ROL64(C0, 1);
+    d4 = c3^ROL64(c0, 1);
 
-    B0 = (A00^D0);
+    b0 = (a00^d0);
-    B1 = ROL64((A01^D1), 44);
+    b1 = ROL64((a01^d1), 44);
-    B2 = ROL64((A02^D2), 43);
+    b2 = ROL64((a02^d2), 43);
-    B3 = ROL64((A03^D3), 21);
+    b3 = ROL64((a03^d3), 21);
-    B4 = ROL64((A04^D4), 14);
+    b4 = ROL64((a04^d4), 14);
-    A00 =   B0 ^((~B1)&  B2 );
+    a00 =   b0 ^((~b1)&  b2 );
-    A00 ^= RC[i+3];
+    a00 ^= RC[i+3];
-    A01 =   B1 ^((~B2)&  B3 );
+    a01 =   b1 ^((~b2)&  b3 );
-    A02 =   B2 ^((~B3)&  B4 );
+    a02 =   b2 ^((~b3)&  b4 );
-    A03 =   B3 ^((~B4)&  B0 );
+    a03 =   b3 ^((~b4)&  b0 );
-    A04 =   B4 ^((~B0)&  B1 );
+    a04 =   b4 ^((~b0)&  b1 );
 
-    B2 = ROL64((A10^D0), 3);
+    b2 = ROL64((a10^d0), 3);
-    B3 = ROL64((A11^D1), 45);
+    b3 = ROL64((a11^d1), 45);
-    B4 = ROL64((A12^D2), 61);
+    b4 = ROL64((a12^d2), 61);
-    B0 = ROL64((A13^D3), 28);
+    b0 = ROL64((a13^d3), 28);
-    B1 = ROL64((A14^D4), 20);
+    b1 = ROL64((a14^d4), 20);
-    A10 =   B0 ^((~B1)&  B2 );
+    a10 =   b0 ^((~b1)&  b2 );
-    A11 =   B1 ^((~B2)&  B3 );
+    a11 =   b1 ^((~b2)&  b3 );
-    A12 =   B2 ^((~B3)&  B4 );
+    a12 =   b2 ^((~b3)&  b4 );
-    A13 =   B3 ^((~B4)&  B0 );
+    a13 =   b3 ^((~b4)&  b0 );
-    A14 =   B4 ^((~B0)&  B1 );
+    a14 =   b4 ^((~b0)&  b1 );
 
-    B4 = ROL64((A20^D0), 18);
+    b4 = ROL64((a20^d0), 18);
-    B0 = ROL64((A21^D1), 1);
+    b0 = ROL64((a21^d1), 1);
-    B1 = ROL64((A22^D2), 6);
+    b1 = ROL64((a22^d2), 6);
-    B2 = ROL64((A23^D3), 25);
+    b2 = ROL64((a23^d3), 25);
-    B3 = ROL64((A24^D4), 8);
+    b3 = ROL64((a24^d4), 8);
-    A20 =   B0 ^((~B1)&  B2 );
+    a20 =   b0 ^((~b1)&  b2 );
-    A21 =   B1 ^((~B2)&  B3 );
+    a21 =   b1 ^((~b2)&  b3 );
-    A22 =   B2 ^((~B3)&  B4 );
+    a22 =   b2 ^((~b3)&  b4 );
-    A23 =   B3 ^((~B4)&  B0 );
+    a23 =   b3 ^((~b4)&  b0 );
-    A24 =   B4 ^((~B0)&  B1 );
+    a24 =   b4 ^((~b0)&  b1 );
 
-    B1 = ROL64((A30^D0), 36);
+    b1 = ROL64((a30^d0), 36);
-    B2 = ROL64((A31^D1), 10);
+    b2 = ROL64((a31^d1), 10);
-    B3 = ROL64((A32^D2), 15);
+    b3 = ROL64((a32^d2), 15);
-    B4 = ROL64((A33^D3), 56);
+    b4 = ROL64((a33^d3), 56);
-    B0 = ROL64((A34^D4), 27);
+    b0 = ROL64((a34^d4), 27);
-    A30 =   B0 ^((~B1)&  B2 );
+    a30 =   b0 ^((~b1)&  b2 );
-    A31 =   B1 ^((~B2)&  B3 );
+    a31 =   b1 ^((~b2)&  b3 );
-    A32 =   B2 ^((~B3)&  B4 );
+    a32 =   b2 ^((~b3)&  b4 );
-    A33 =   B3 ^((~B4)&  B0 );
+    a33 =   b3 ^((~b4)&  b0 );
-    A34 =   B4 ^((~B0)&  B1 );
+    a34 =   b4 ^((~b0)&  b1 );
 
-    B3 = ROL64((A40^D0), 41);
+    b3 = ROL64((a40^d0), 41);
-    B4 = ROL64((A41^D1), 2);
+    b4 = ROL64((a41^d1), 2);
-    B0 = ROL64((A42^D2), 62);
+    b0 = ROL64((a42^d2), 62);
-    B1 = ROL64((A43^D3), 55);
+    b1 = ROL64((a43^d3), 55);
-    B2 = ROL64((A44^D4), 39);
+    b2 = ROL64((a44^d4), 39);
-    A40 =   B0 ^((~B1)&  B2 );
+    a40 =   b0 ^((~b1)&  b2 );
-    A41 =   B1 ^((~B2)&  B3 );
+    a41 =   b1 ^((~b2)&  b3 );
-    A42 =   B2 ^((~B3)&  B4 );
+    a42 =   b2 ^((~b3)&  b4 );
-    A43 =   B3 ^((~B4)&  B0 );
+    a43 =   b3 ^((~b4)&  b0 );
-    A44 =   B4 ^((~B0)&  B1 );
+    a44 =   b4 ^((~b0)&  b1 );
   }
 }