Merge latest changes from wal2 into this branch.

FossilOrigin-Name: c9c9bc097a622bfb6e094b1178cd44da19418cf19cb2d6ad54b9bb868de51f10

Makefile.in

@@ -57,6 +57,7 @@ LIBTCL = @TCL_LIB_SPEC@
 #
 READLINE_FLAGS = -DHAVE_READLINE=@TARGET_HAVE_READLINE@ @TARGET_READLINE_INC@
 READLINE_FLAGS += -DHAVE_EDITLINE=@TARGET_HAVE_EDITLINE@
+READLINE_FLAGS += -DHAVE_LINENOISE=@TARGET_HAVE_LINENOISE@
 
 # The library that programs using readline() must link against.
 #
@@ -767,13 +768,22 @@ mptest:	mptester$(TEXE)
 	$(MPTEST2) --journalmode DELETE
 
 
+has_tclsh84:
+	sh $(TOP)/tool/cktclsh.sh 8.4 $(TCLSH_CMD)
+	touch has_tclsh84
+
+has_tclsh85:
+	sh $(TOP)/tool/cktclsh.sh 8.5 $(TCLSH_CMD)
+	touch has_tclsh85
+
+
 # This target creates a directory named "tsrc" and fills it with
 # copies of all of the C source code and header files needed to
 # build on the target system.  Some of the C source code and header
 # files are automatically generated.  This target takes care of
 # all that automatic generation.
 #
-.target_source:	$(SRC) $(TOP)/tool/vdbe-compress.tcl fts5.c
+.target_source:	$(SRC) $(TOP)/tool/vdbe-compress.tcl has_tclsh84 fts5.c
 	rm -rf tsrc
 	mkdir tsrc
 	cp -f $(SRC) tsrc
@@ -783,15 +793,15 @@ mptest:	mptester$(TEXE)
 	cp fts5.c fts5.h tsrc
 	touch .target_source
 
-sqlite3.c:	.target_source $(TOP)/tool/mksqlite3c.tcl src-verify
+sqlite3.c:	.target_source $(TOP)/tool/mksqlite3c.tcl src-verify has_tclsh84
 	$(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl $(AMALGAMATION_LINE_MACROS)
 	cp tsrc/sqlite3ext.h .
 	cp $(TOP)/ext/session/sqlite3session.h .
 
-sqlite3r.h: sqlite3.h
+sqlite3r.h: sqlite3.h has_tclsh84
 	$(TCLSH_CMD) $(TOP)/tool/mksqlite3h.tcl $(TOP) --enable-recover >sqlite3r.h
 
-sqlite3r.c: sqlite3.c sqlite3r.h
+sqlite3r.c: sqlite3.c sqlite3r.h has_tclsh84
 	cp $(TOP)/ext/recover/sqlite3recover.c tsrc/
 	cp $(TOP)/ext/recover/sqlite3recover.h tsrc/
 	cp $(TOP)/ext/recover/dbdata.c tsrc/
@@ -806,7 +816,7 @@ tclsqlite3.c:	sqlite3.c
 	echo '#endif /* USE_SYSTEM_SQLITE */' >>tclsqlite3.c
 	cat $(TOP)/src/tclsqlite.c >>tclsqlite3.c
 
-sqlite3-all.c:	sqlite3.c $(TOP)/tool/split-sqlite3c.tcl
+sqlite3-all.c:	sqlite3.c $(TOP)/tool/split-sqlite3c.tcl has_tclsh84
 	$(TCLSH_CMD) $(TOP)/tool/split-sqlite3c.tcl
 
 # Rule to build the amalgamation
@@ -1094,10 +1104,10 @@ tclsqlite3$(TEXE):	tclsqlite-shell.lo libsqlite3.la
 
 # Rules to build opcodes.c and opcodes.h
 #
-opcodes.c:	opcodes.h $(TOP)/tool/mkopcodec.tcl
+opcodes.c:	opcodes.h $(TOP)/tool/mkopcodec.tcl has_tclsh84
 	$(TCLSH_CMD) $(TOP)/tool/mkopcodec.tcl opcodes.h >opcodes.c
 
-opcodes.h:	parse.h $(TOP)/src/vdbe.c $(TOP)/tool/mkopcodeh.tcl
+opcodes.h:	parse.h $(TOP)/src/vdbe.c $(TOP)/tool/mkopcodeh.tcl has_tclsh84
 	cat parse.h $(TOP)/src/vdbe.c | $(TCLSH_CMD) $(TOP)/tool/mkopcodeh.tcl >opcodes.h
 
 # Rules to build parse.c and parse.h - the outputs of lemon.
@@ -1108,10 +1118,10 @@ parse.c:	$(TOP)/src/parse.y lemon$(BEXE)
 	cp $(TOP)/src/parse.y .
 	./lemon$(BEXE) $(OPT_FEATURE_FLAGS) $(OPTS) -S parse.y
 
-sqlite3.h:	$(TOP)/src/sqlite.h.in $(TOP)/manifest mksourceid$(BEXE) $(TOP)/VERSION
+sqlite3.h:	$(TOP)/src/sqlite.h.in $(TOP)/manifest mksourceid$(BEXE) $(TOP)/VERSION has_tclsh84
 	$(TCLSH_CMD) $(TOP)/tool/mksqlite3h.tcl $(TOP) >sqlite3.h
 
-sqlite3rc.h:	$(TOP)/src/sqlite3.rc $(TOP)/VERSION
+sqlite3rc.h:	$(TOP)/src/sqlite3.rc $(TOP)/VERSION has_tclsh84
 	echo '#ifndef SQLITE_RESOURCE_VERSION' >$@
 	echo -n '#define SQLITE_RESOURCE_VERSION ' >>$@
 	cat $(TOP)/VERSION | $(TCLSH_CMD) $(TOP)/tool/replace.tcl exact . , >>$@
@@ -1147,7 +1157,7 @@ SHELL_SRC = \
 	$(TOP)/ext/recover/sqlite3recover.h \
         $(TOP)/src/test_windirent.c
 
-shell.c:	$(SHELL_SRC) $(TOP)/tool/mkshellc.tcl
+shell.c:	$(SHELL_SRC) $(TOP)/tool/mkshellc.tcl has_tclsh84
 	$(TCLSH_CMD) $(TOP)/tool/mkshellc.tcl >shell.c
 
 
@@ -1235,7 +1245,7 @@ fts5parse.c:	$(TOP)/ext/fts5/fts5parse.y lemon$(BEXE)
 
 fts5parse.h: fts5parse.c
 
-fts5.c: $(FTS5_SRC)
+fts5.c: $(FTS5_SRC) has_tclsh84
 	$(TCLSH_CMD) $(TOP)/ext/fts5/tool/mkfts5c.tcl
 	cp $(TOP)/ext/fts5/fts5.h .
 
@@ -1269,7 +1279,7 @@ TESTFIXTURE_SRC1 = sqlite3.c
 TESTFIXTURE_SRC = $(TESTSRC) $(TOP)/src/tclsqlite.c
 TESTFIXTURE_SRC += $(TESTFIXTURE_SRC$(USE_AMALGAMATION))
 
-testfixture$(TEXE):	$(TESTFIXTURE_SRC)
+testfixture$(TEXE):	has_tclsh85 $(TESTFIXTURE_SRC)
 	$(LTLINK) -DSQLITE_NO_SYNC=1 $(TEMP_STORE) $(TESTFIXTURE_FLAGS) \
 		-o $@ $(TESTFIXTURE_SRC) $(LIBTCL) $(TLIBS)
 
@@ -1293,11 +1303,17 @@ fulltestonly:	$(TESTPROGS) fuzztest
 	./testfixture$(TEXE) $(TOP)/test/full.test
 
 # Fuzz testing
-fuzztest:	fuzzcheck$(TEXE) $(FUZZDATA) sessionfuzz$(TEXE) $(TOP)/test/sessionfuzz-data1.db
+#
+# WARNING: When the "fuzztest" target is run by the testrunner.tcl script,
+# it does not actually run this code. Instead, it schedules equivalent 
+# commands. Therefore, if this target is updated, then code in
+# testrunner_data.tcl (search for "trd_fuzztest_data") must also be updated.
+#
+fuzztest:	fuzzcheck$(TEXE) $(FUZZDATA) sessionfuzz$(TEXE)
 	./fuzzcheck$(TEXE) $(FUZZDATA)
 	./sessionfuzz$(TEXE) run $(TOP)/test/sessionfuzz-data1.db
 
-valgrindfuzz:	fuzzcheck$(TEXT) $(FUZZDATA) sessionfuzz$(TEXE) $(TOP)/test/sessionfuzz-data1.db
+valgrindfuzz:	fuzzcheck$(TEXT) $(FUZZDATA) sessionfuzz$(TEXE)
 	valgrind ./fuzzcheck$(TEXE) --cell-size-check --limit-mem 10M $(FUZZDATA)
 	valgrind ./sessionfuzz$(TEXE) run $(TOP)/test/sessionfuzz-data1.db
 
@@ -1314,17 +1330,23 @@ testrunner:	testfixture$(TEXE)
 
 # Runs both fuzztest and testrunner, consecutively.
 #
-devtest:	testfixture$(TEXE) fuzztest testrunner
+devtest:	srctree-check testfixture$(TEXE) fuzztest testrunner
 
-mdevtest:
+mdevtest: srctree-check has_tclsh85
 	$(TCLSH_CMD) $(TOP)/test/testrunner.tcl mdevtest
 
-sdevtest:
+sdevtest: has_tclsh85
 	$(TCLSH_CMD) $(TOP)/test/testrunner.tcl sdevtest
 
+# Validate that various generated files in the source tree
+# are up-to-date.
+#
+srctree-check:	$(TOP)/tool/srctree-check.tcl
+	$(TCLSH_CMD) $(TOP)/tool/srctree-check.tcl
+
 # Testing for a release
 #
-releasetest: testfixture$(TEXE)
+releasetest: srctree-check testfixture$(TEXE)
 	./testfixture$(TEXE) $(TOP)/test/testrunner.tcl release
 
 # Minimal testing that runs in less than 3 minutes
@@ -1335,7 +1357,7 @@ quicktest:	./testfixture$(TEXE)
 # This is the common case.  Run many tests that do not take too long,
 # including fuzzcheck, sqlite3_analyzer, and sqldiff tests.
 #
-test:	fuzztest sourcetest $(TESTPROGS) tcltest
+test:	srctree-check fuzztest sourcetest $(TESTPROGS) tcltest
 
 # Run a test using valgrind.  This can take a really long time
 # because valgrind is so much slower than a native machine.
@@ -1353,13 +1375,13 @@ smoketest:	$(TESTPROGS) fuzzcheck$(TEXE)
 shelltest: $(TESTPROGS)
 	./testfixture$(TEXT) $(TOP)/test/permutations.test shell
 
-sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in
+sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in has_tclsh85
 	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in >sqlite3_analyzer.c
 
 sqlite3_analyzer$(TEXE): sqlite3_analyzer.c
 	$(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS)
 
-sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in
+sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in has_tclsh85
 	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in >sqltclsh.c
 
 sqltclsh$(TEXE): sqltclsh.c
@@ -1378,7 +1400,7 @@ CHECKER_DEPS =\
   $(TOP)/ext/misc/btreeinfo.c \
   $(TOP)/ext/repair/sqlite3_checker.c.in
 
-sqlite3_checker.c:	$(CHECKER_DEPS)
+sqlite3_checker.c:	$(CHECKER_DEPS) has_tclsh85
 	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/ext/repair/sqlite3_checker.c.in >$@
 
 sqlite3_checker$(TEXE):	sqlite3_checker.c
@@ -1464,6 +1486,11 @@ amalgamation-tarball: sqlite3.c sqlite3rc.h
 snapshot-tarball: sqlite3.c sqlite3rc.h
 	TOP=$(TOP) sh $(TOP)/tool/mkautoconfamal.sh --snapshot
 
+# Build a ZIP archive containing various command-line tools.
+#
+tool-zip:	testfixture sqlite3 sqldiff sqlite3_analyzer tool/mktoolzip.tcl
+	./testfixture $(TOP)/tool/mktoolzip.tcl
+
 # The next two rules are used to support the "threadtest" target. Building
 # threadtest runs a few thread-safety tests that are implemented in C. This
 # target is invoked by the releasetest.tcl script.
@@ -1539,6 +1566,7 @@ clean:
 	rm -f threadtest5
 	rm -f src-verify
 	rm -f custom.rws
+	rm -f has_tclsh84 has_tclsh85
 
 distclean:	clean
 	rm -f sqlite_cfg.h config.log config.status libtool Makefile sqlite3.pc \
@@ -1583,7 +1611,7 @@ fiddle: sqlite3.c shell.c
 	@echo 'Updating custom dictionary from tool/custom.txt'
 	aspell --lang=en create master ./custom.rws < $<
 
-misspell: ./custom.rws
+misspell: ./custom.rws has_tclsh84
 	$(TCLSH_CMD) ./tool/spellsift.tcl ./src/*.c ./src/*.h ./src/*.in
 
 #

Makefile.msc

@@ -52,8 +52,8 @@ MINIMAL_AMALGAMATION = 0
 USE_STDCALL = 0
 !ENDIF
 
-# Set this non-0 to use structured exception handling (SEH) for WAL mode
+# Use the USE_SEH=0 option on the nmake command line to omit structured
-# in the core library.
+# exception handling (SEH) support.  SEH is on by default.
 #
 !IFNDEF USE_SEH
 USE_SEH = 1
@@ -403,10 +403,11 @@ OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RBU=1
 !ENDIF
 
 # Should structured exception handling (SEH) be enabled for WAL mode in
-# the core library?
+# the core library?  It is on by default.  Only omit it if the
+# USE_SEH=0 option is provided on the nmake command-line.
 #
-!IF $(USE_SEH)!=0
+!IF $(USE_SEH)==0
-OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_USE_SEH=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_OMIT_SEH=1
 !ENDIF
 
 # These are the "extended" SQLite compilation options used when compiling for
@@ -2467,6 +2468,9 @@ extensiontest:	testfixture.exe testloadext.dll
 	@set PATH=$(LIBTCLPATH);$(PATH)
 	.\testfixture.exe $(TOP)\test\loadext.test $(TESTOPTS)
 
+tool-zip:	testfixture.exe sqlite3.exe sqldiff.exe sqlite3_analyzer.exe tool\mktoolzip.tcl
+	.\testfixture.exe $(TOP)\tool\mktoolzip.tcl
+
 coretestprogs:	$(TESTPROGS)
 
 testprogs:	coretestprogs srcck1.exe fuzzcheck.exe sessionfuzz.exe

VERSION

@@ -1 +1 @@
-3.43.0
+3.44.0

autoconf/Makefile.msc

@@ -52,8 +52,8 @@ MINIMAL_AMALGAMATION = 0
 USE_STDCALL = 0
 !ENDIF
 
-# Set this non-0 to use structured exception handling (SEH) for WAL mode
+# Use the USE_SEH=0 option on the nmake command line to omit structured
-# in the core library.
+# exception handling (SEH) support.  SEH is on by default.
 #
 !IFNDEF USE_SEH
 USE_SEH = 1
@@ -325,10 +325,11 @@ OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RBU=1
 !ENDIF
 
 # Should structured exception handling (SEH) be enabled for WAL mode in
-# the core library?
+# the core library?  It is on by default.  Only omit it if the
+# USE_SEH=0 option is provided on the nmake command-line.
 #
-!IF $(USE_SEH)!=0
+!IF $(USE_SEH)==0
-OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_USE_SEH=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_OMIT_SEH=1
 !ENDIF
 
 # These are the "extended" SQLite compilation options used when compiling for

autoconf/tea/configure.ac

@@ -19,7 +19,7 @@ dnl	to configure the system for the local environment.
 # so that we create the export library with the dll.
 #-----------------------------------------------------------------------
 
-AC_INIT([sqlite],[3.43.0])
+AC_INIT([sqlite],[3.44.0])
 
 #--------------------------------------------------------------------
 # Call TEA_INIT as the first TEA_ macro to set up initial vars.

configure

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for sqlite 3.43.0.
+# Generated by GNU Autoconf 2.69 for sqlite 3.44.0.
 #
 #
 # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -726,8 +726,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='sqlite'
 PACKAGE_TARNAME='sqlite'
-PACKAGE_VERSION='3.43.0'
+PACKAGE_VERSION='3.44.0'
-PACKAGE_STRING='sqlite 3.43.0'
+PACKAGE_STRING='sqlite 3.44.0'
 PACKAGE_BUGREPORT=''
 PACKAGE_URL=''
 
@@ -776,6 +776,7 @@ OPT_FEATURE_FLAGS
 HAVE_ZLIB
 USE_AMALGAMATION
 TARGET_DEBUG
+TARGET_HAVE_LINENOISE
 TARGET_HAVE_EDITLINE
 TARGET_HAVE_READLINE
 TARGET_READLINE_INC
@@ -903,6 +904,7 @@ enable_editline
 enable_readline
 with_readline_lib
 with_readline_inc
+with_linenoise
 enable_debug
 enable_amalgamation
 enable_load_extension
@@ -1470,7 +1472,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures sqlite 3.43.0 to adapt to many kinds of systems.
+\`configure' configures sqlite 3.44.0 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1535,7 +1537,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of sqlite 3.43.0:";;
+     short | recursive ) echo "Configuration of sqlite 3.44.0:";;
    esac
   cat <<\_ACEOF
 
@@ -1587,6 +1589,7 @@ Optional Packages:
                           (tclConfig.sh)
   --with-readline-lib     specify readline library
   --with-readline-inc     specify readline include paths
+  --with-linenoise=DIR    source directory for linenoise library
 
 Some influential environment variables:
   CC          C compiler command
@@ -1665,7 +1668,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-sqlite configure 3.43.0
+sqlite configure 3.44.0
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -2084,7 +2087,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by sqlite $as_me 3.43.0, which was
+It was created by sqlite $as_me 3.44.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -3942,13 +3945,13 @@ if ${lt_cv_nm_interface+:} false; then :
 else
   lt_cv_nm_interface="BSD nm"
   echo "int some_variable = 0;" > conftest.$ac_ext
-  (eval echo "\"\$as_me:3945: $ac_compile\"" >&5)
+  (eval echo "\"\$as_me:3948: $ac_compile\"" >&5)
   (eval "$ac_compile" 2>conftest.err)
   cat conftest.err >&5
-  (eval echo "\"\$as_me:3948: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
+  (eval echo "\"\$as_me:3951: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
   (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out)
   cat conftest.err >&5
-  (eval echo "\"\$as_me:3951: output\"" >&5)
+  (eval echo "\"\$as_me:3954: output\"" >&5)
   cat conftest.out >&5
   if $GREP 'External.*some_variable' conftest.out > /dev/null; then
     lt_cv_nm_interface="MS dumpbin"
@@ -5154,7 +5157,7 @@ ia64-*-hpux*)
   ;;
 *-*-irix6*)
   # Find out which ABI we are using.
-  echo '#line 5157 "configure"' > conftest.$ac_ext
+  echo '#line 5160 "configure"' > conftest.$ac_ext
   if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5
   (eval $ac_compile) 2>&5
   ac_status=$?
@@ -6679,11 +6682,11 @@ else
    -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
    -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
    -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:6682: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:6685: $lt_compile\"" >&5)
    (eval "$lt_compile" 2>conftest.err)
    ac_status=$?
    cat conftest.err >&5
-   echo "$as_me:6686: \$? = $ac_status" >&5
+   echo "$as_me:6689: \$? = $ac_status" >&5
    if (exit $ac_status) && test -s "$ac_outfile"; then
      # The compiler can only warn and ignore the option if not recognized
      # So say no if there are warnings other than the usual output.
@@ -7018,11 +7021,11 @@ else
    -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
    -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
    -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7021: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7024: $lt_compile\"" >&5)
    (eval "$lt_compile" 2>conftest.err)
    ac_status=$?
    cat conftest.err >&5
-   echo "$as_me:7025: \$? = $ac_status" >&5
+   echo "$as_me:7028: \$? = $ac_status" >&5
    if (exit $ac_status) && test -s "$ac_outfile"; then
      # The compiler can only warn and ignore the option if not recognized
      # So say no if there are warnings other than the usual output.
@@ -7123,11 +7126,11 @@ else
    -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
    -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
    -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7126: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7129: $lt_compile\"" >&5)
    (eval "$lt_compile" 2>out/conftest.err)
    ac_status=$?
    cat out/conftest.err >&5
-   echo "$as_me:7130: \$? = $ac_status" >&5
+   echo "$as_me:7133: \$? = $ac_status" >&5
    if (exit $ac_status) && test -s out/conftest2.$ac_objext
    then
      # The compiler can only warn and ignore the option if not recognized
@@ -7178,11 +7181,11 @@ else
    -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
    -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
    -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7181: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7184: $lt_compile\"" >&5)
    (eval "$lt_compile" 2>out/conftest.err)
    ac_status=$?
    cat out/conftest.err >&5
-   echo "$as_me:7185: \$? = $ac_status" >&5
+   echo "$as_me:7188: \$? = $ac_status" >&5
    if (exit $ac_status) && test -s out/conftest2.$ac_objext
    then
      # The compiler can only warn and ignore the option if not recognized
@@ -9558,7 +9561,7 @@ else
   lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
   lt_status=$lt_dlunknown
   cat > conftest.$ac_ext <<_LT_EOF
-#line 9561 "configure"
+#line 9564 "configure"
 #include "confdefs.h"
 
 #if HAVE_DLFCN_H
@@ -9654,7 +9657,7 @@ else
   lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
   lt_status=$lt_dlunknown
   cat > conftest.$ac_ext <<_LT_EOF
-#line 9657 "configure"
+#line 9660 "configure"
 #include "confdefs.h"
 
 #if HAVE_DLFCN_H
@@ -11245,6 +11248,27 @@ fi
 	fi
 fi
 
+# Check whether --with-linenoise was given.
+if test "${with_linenoise+set}" = set; then :
+  withval=$with_linenoise; with_linenoise=$withval
+else
+  with_linenoise="no"
+fi
+
+if test "x$with_linenoise" != "xno"; then
+   TARGET_HAVE_READLINE=0
+   TARGET_HAVE_EDITLINE=0
+   TARGET_HAVE_LINENOISE=1
+   TARGET_READLINE_INC="-I${with_linenoise}"
+   TARGET_READLINE_LIBS="${with_linenoise}/linenoise.c"
+   echo "using linenoise source code at ${with_linenoise}"
+else
+   TARGET_HAVE_LINENOISE=0
+   echo "not using linenoise"
+fi
+
+
+
 
 
 
@@ -11321,7 +11345,7 @@ fi
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking build type" >&5
 $as_echo_n "checking build type... " >&6; }
 if test "${enable_debug}" = "yes" ; then
-  TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0"
+  TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0 -Wall"
   { $as_echo "$as_me:${as_lineno-$LINENO}: result: debug" >&5
 $as_echo "debug" >&6; }
 else
@@ -12457,7 +12481,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by sqlite $as_me 3.43.0, which was
+This file was extended by sqlite $as_me 3.44.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -12523,7 +12547,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-sqlite config.status 3.43.0
+sqlite config.status 3.44.0
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 

configure.ac

@@ -598,11 +598,28 @@ if test x"$with_readline" != xno; then
 		TARGET_HAVE_READLINE=1
 	fi
 fi
+AC_ARG_WITH([linenoise],
+            [AS_HELP_STRING([--with-linenoise=DIR],[source directory for linenoise library])],
+            [with_linenoise=$withval],
+            [with_linenoise="no"])
+if test "x$with_linenoise" != "xno"; then
+   TARGET_HAVE_READLINE=0
+   TARGET_HAVE_EDITLINE=0
+   TARGET_HAVE_LINENOISE=1
+   TARGET_READLINE_INC="-I${with_linenoise}"
+   TARGET_READLINE_LIBS="${with_linenoise}/linenoise.c"
+   echo "using linenoise source code at ${with_linenoise}"
+else
+   TARGET_HAVE_LINENOISE=0
+   echo "not using linenoise"
+fi
 
 AC_SUBST(TARGET_READLINE_LIBS)
 AC_SUBST(TARGET_READLINE_INC)
 AC_SUBST(TARGET_HAVE_READLINE)
 AC_SUBST(TARGET_HAVE_EDITLINE)
+AC_SUBST(TARGET_HAVE_LINENOISE)
+
 
 ##########
 # Figure out what C libraries are required to compile programs
@@ -615,7 +632,7 @@ AC_SEARCH_LIBS(fdatasync, [rt])
 AC_ARG_ENABLE(debug, AS_HELP_STRING([--enable-debug],[enable debugging & verbose explain]))
 AC_MSG_CHECKING([build type])
 if test "${enable_debug}" = "yes" ; then
-  TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0"
+  TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0 -Wall"
   AC_MSG_RESULT([debug])
 else
   TARGET_DEBUG="-DNDEBUG"

doc/testrunner.md

@@ -0,0 +1,284 @@
+
+
+# The testrunner.tcl Script
+
+# 1. Overview
+
+testrunner.tcl is a Tcl script used to run multiple SQLite tests using 
+multiple jobs. It supports the following types of tests:
+
+  *  Tcl test scripts.
+
+  *  Tests run with [make] commands. Specifically, at time of writing, 
+     [make fuzztest], [make mptest], [make sourcetest] and [make threadtest].
+
+testrunner.tcl pipes the output of all tests and builds run into log file
+**testrunner.log**, created in the cwd directory. Searching this file for
+"failed" is a good way to find the output of a failed test.
+
+testrunner.tcl also populates SQLite database **testrunner.db**. This database
+contains details of all tests run, running and to be run. A useful query
+might be:
+
+```
+  SELECT * FROM script WHERE state='failed'
+```
+
+Running the command:
+
+```
+  ./testfixture $(TESTDIR)/testrunner.tcl status
+```
+
+in the directory containing the testrunner.db database runs various queries
+to produce a succinct report on the state of a running testrunner.tcl script.
+Running:
+
+```
+  watch ./testfixture $(TESTDIR)/testrunner.tcl status
+```
+
+in another terminal is a good way to keep an eye on a long running test.
+
+Sometimes testrunner.tcl uses the [testfixture] binary that it is run with
+to run tests (see "Binary Tests" below). Sometimes it builds testfixture and
+other binaries in specific configurations to test (see "Source Tests").
+
+# 2. Binary Tests
+
+The commands described in this section all run various combinations of the Tcl
+test scripts using the [testfixture] binary used to run the testrunner.tcl
+script (i.e. they do not invoke the compiler to build new binaries, or the
+[make] command to run tests that are not Tcl scripts). The procedure to run
+these tests is therefore:
+
+  1. Build the "testfixture" (or "testfixture.exe" for windows) binary using
+     whatever method seems convenient.
+
+  2. Test the binary built in step 1 by running testrunner.tcl with it, 
+     perhaps with various options.
+
+The following sub-sections describe the various options that can be
+passed to testrunner.tcl to test binary testfixture builds.
+
+## 2.1. Organization of Tcl Tests
+
+Tcl tests are stored in files that match the pattern *\*.test*. They are
+found in both the $TOP/test/ directory, and in the various sub-directories
+of the $TOP/ext/ directory of the source tree. Not all *\*.test* files
+contain Tcl tests - a handful are Tcl scripts designed to invoke other
+*\*.test* files.
+
+The **veryquick** set of tests is a subset of all Tcl test scripts in the
+source tree. In includes most tests, but excludes some that are very slow.
+Almost all fault-injection tests (those that test the response of the library
+to OOM or IO errors) are excluded. It is defined in source file 
+*test/permutations.test*.
+
+The **full** set of tests includes all Tcl test scripts in the source tree.
+To run a "full" test is to run all Tcl test scripts that can be found in the
+source tree.
+
+File *permutations.test* defines various test "permutations". A permutation
+consists of:
+
+  *  A subset of Tcl test scripts, and 
+
+  *  Runtime configuration to apply before running each test script 
+     (e.g. enabling auto-vacuum, or disable lookaside).
+
+Running **all** tests is to run all tests in the full test set, plus a dozen
+or so permutations. The specific permutations that are run as part of "all"
+are defined in file *testrunner_data.tcl*.
+
+## 2.2. Commands to Run Tests
+
+To run the "veryquick" test set, use either of the following:
+
+```
+  ./testfixture $TESTDIR/testrunner.tcl
+  ./testfixture $TESTDIR/testrunner.tcl veryquick
+```
+
+To run the "full" test suite:
+
+```
+  ./testfixture $TESTDIR/testrunner.tcl full
+```
+
+To run the subset of the "full" test suite for which the test file name matches
+a specified pattern (e.g. all tests that start with "fts5"), either of:
+
+```
+  ./testfixture $TESTDIR/testrunner.tcl fts5%
+  ./testfixture $TESTDIR/testrunner.tcl 'fts5*'
+```
+
+To run "all" tests (full + permutations):
+
+```
+  ./testfixture $TESTDIR/testrunner.tcl all
+```
+
+<a name=binary_test_failures></a>
+## 2.3. Investigating Binary Test Failures
+
+If a test fails, testrunner.tcl reports name of the Tcl test script and, if
+applicable, the name of the permutation, to stdout. This information can also
+be retrieved from either *testrunner.log* or *testrunner.db*.
+
+If there is no permutation, the individual test script may be run with:
+
+```
+  ./testfixture $PATH_TO_SCRIPT
+```
+
+Or, if the failure occured as part of a permutation:
+
+```
+  ./testfixture $TESTDIR/testrunner.tcl $PERMUTATION $PATH_TO_SCRIPT
+```
+
+TODO: An example instead of "$PERMUTATION" and $PATH\_TO\_SCRIPT?
+
+# 3. Source Code Tests
+
+The commands described in this section invoke the C compiler to build 
+binaries from the source tree, then use those binaries to run Tcl and
+other tests. The advantages of this are that:
+
+  *  it is possible to test multiple build configurations with a single
+     command, and 
+
+  *  it ensures that tests are always run using binaries created with the
+     same set of compiler options.
+
+The testrunner.tcl commands described in this section may be run using
+either a *testfixture* (or testfixture.exe) build, or with any other Tcl
+shell that supports SQLite 3.31.1 or newer via "package require sqlite3".
+
+TODO: ./configure + Makefile.msc build systems.
+
+## Commands to Run SQLite Tests
+
+The **mdevtest** command is equivalent to running the veryquick tests and
+the [make fuzztest] target once for each of two --enable-all builds - one 
+with debugging enabled and one without:
+
+```
+  tclsh $TESTDIR/testrunner.tcl mdevtest
+```
+
+In other words, it is equivalent to running:
+
+```
+  $TOP/configure --enable-all --enable-debug
+  make fuzztest
+  make testfixture
+  ./testfixture $TOP/test/testrunner.tcl veryquick
+
+  # Then, after removing files created by the tests above:
+  $TOP/configure --enable-all OPTS="-O0"
+  make fuzztest
+  make testfixture
+  ./testfixture $TOP/test/testrunner.tcl veryquick
+```
+
+The **sdevtest** command is identical to the mdevtest command, except that the
+second of the two builds is a sanitizer build. Specifically, this means that
+OPTS="-fsanitize=address,undefined" is specified instead of OPTS="-O0":
+
+```
+  tclsh $TESTDIR/testrunner.tcl sdevtest
+```
+
+The **release** command runs lots of tests under lots of builds. It runs
+different combinations of builds and tests depending on whether it is run
+on Linux, Windows or OSX. Refer to *testrunner\_data.tcl* for the details
+of the specific tests run.
+
+```
+  tclsh $TESTDIR/testrunner.tcl release
+```
+
+## Running ZipVFS Tests
+
+testrunner.tcl can build a zipvfs-enabled testfixture and use it to run
+tests from the Zipvfs project with the following command:
+
+```
+  tclsh $TESTDIR/testrunner.tcl --zipvfs $PATH_TO_ZIPVFS
+```
+
+This can be combined with any of "mdevtest", "sdevtest" or "release" to
+test both SQLite and Zipvfs with a single command:
+
+```
+  tclsh $TESTDIR/testrunner.tcl --zipvfs $PATH_TO_ZIPVFS mdevtest
+```
+
+## Investigating Source Code Test Failures
+
+Investigating a test failure that occurs during source code testing is a
+two step process:
+
+  1. Recreating the build configuration in which the test failed, and
+
+  2. Re-running the actual test.
+
+To recreate a build configuration, use the testrunner.tcl **script** command
+to create a build script. A build script is a bash script on Linux or OSX, or
+a dos \*.bat file on windows. For example:
+
+```
+  # Create a script that recreates build configuration "Device-One" on 
+  # Linux or OSX:
+  tclsh $TESTDIR/testrunner.tcl script Device-One > make.sh 
+
+  # Create a script that recreates build configuration "Have-Not" on Windows:
+  tclsh $TESTDIR/testrunner.tcl script Have-Not > make.bat 
+```
+
+The generated bash or \*.bat file script accepts a single argument - a makefile
+target to build. This may be used either to run a [make] command test directly,
+or else to build a testfixture (or testfixture.exe) binary with which to
+run a Tcl test script, as <a href=#binary_test_failures>described above</a>.
+
+
+
+# 4. Controlling CPU Core Utilization
+
+When running either binary or source code tests, testrunner.tcl reports the
+number of jobs it intends to use to stdout. e.g.
+
+```
+  $ ./testfixture $TESTDIR/testrunner.tcl
+  splitting work across 16 jobs
+  ... more output ...
+```
+
+By default, testfixture.tcl attempts to set the number of jobs to the number 
+of real cores on the machine. This can be overridden using the "--jobs" (or -j)
+switch:
+
+```
+  $ ./testfixture $TESTDIR/testrunner.tcl --jobs 8
+  splitting work across 8 jobs
+  ... more output ...
+```
+
+The number of jobs may also be changed while an instance of testrunner.tcl is
+running by exucuting the following command from the directory containing the
+testrunner.log and testrunner.db files:
+
+```
+  $ ./testfixture $TESTDIR/testrunner.tcl njob $NEW_NUMBER_OF_JOBS
+```
+
+
+
+
+
+
+
+

ext/expert/expert1.test

@@ -464,4 +464,23 @@ do_execsql_test 5.3 {
   t2 t2_idx_0001295b {100 20 5}
 }
 
+if 0 {
+do_test expert1-6.0 {
+  catchcmd :memory: {
+.expert
+select base64('');
+.expert
+select name from pragma_collation_list order by name collate uint;
+}
+} {0 {(no new indexes)
+
+SCAN CONSTANT ROW
+
+(no new indexes)
+
+SCAN pragma_collation_list VIRTUAL TABLE INDEX 0:
+USE TEMP B-TREE FOR ORDER BY
+}}
+}
+
 finish_test

ext/expert/sqlite3expert.c

@@ -662,6 +662,7 @@ static int idxRegisterVtab(sqlite3expert *p){
     0,                            /* xRelease */
     0,                            /* xRollbackTo */
     0,                            /* xShadowName */
+    0,                            /* xIntegrity */
   };
 
   return sqlite3_create_module(p->dbv, "expert", &expertModule, (void*)p);
@@ -1818,6 +1819,87 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){
   return rc;
 }
 
+/*
+** Define and possibly pretend to use a useless collation sequence.
+** This pretense allows expert to accept SQL using custom collations.
+*/
+int dummyCompare(void *up1, int up2, const void *up3, int up4, const void *up5){
+  (void)up1;
+  (void)up2;
+  (void)up3;
+  (void)up4;
+  (void)up5;
+  assert(0); /* VDBE should never be run. */
+  return 0;
+}
+/* And a callback to register above upon actual need */
+void useDummyCS(void *up1, sqlite3 *db, int etr, const char *zName){
+  (void)up1;
+  sqlite3_create_collation_v2(db, zName, etr, 0, dummyCompare, 0);
+}
+
+#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) \
+  && !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS)
+/*
+** dummy functions for no-op implementation of UDFs during expert's work
+*/
+void dummyUDF(sqlite3_context *up1, int up2, sqlite3_value **up3){
+  (void)up1;
+  (void)up2;
+  (void)up3;
+  assert(0); /* VDBE should never be run. */
+}
+void dummyUDFvalue(sqlite3_context *up1){
+  (void)up1;
+  assert(0); /* VDBE should never be run. */
+}
+
+/*
+** Register UDFs from user database with another.
+*/
+int registerUDFs(sqlite3 *dbSrc, sqlite3 *dbDst){
+  sqlite3_stmt *pStmt;
+  int rc = sqlite3_prepare_v2(dbSrc,
+            "SELECT name,type,enc,narg,flags "
+            "FROM pragma_function_list() "
+            "WHERE builtin==0", -1, &pStmt, 0);
+  if( rc==SQLITE_OK ){
+    while( SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){
+      int nargs = sqlite3_column_int(pStmt,3);
+      int flags = sqlite3_column_int(pStmt,4);
+      const char *name = (char*)sqlite3_column_text(pStmt,0);
+      const char *type = (char*)sqlite3_column_text(pStmt,1);
+      const char *enc = (char*)sqlite3_column_text(pStmt,2);
+      if( name==0 || type==0 || enc==0 ) rc = SQLITE_NOMEM;
+      else{
+        int ienc = SQLITE_UTF8;
+        if( strcmp(enc,"utf16le")==0 ) ienc = SQLITE_UTF16LE;
+        else if( strcmp(enc,"utf16be")==0 ) ienc = SQLITE_UTF16BE;
+        int rcf = SQLITE_ERROR;
+        ienc |= (flags & (SQLITE_DETERMINISTIC|SQLITE_DIRECTONLY));
+        if( strcmp(type,"w")==0 ){
+          rcf = sqlite3_create_window_function(dbDst,name,nargs,ienc,0,
+                                               dummyUDF,dummyUDFvalue,0,0,0);
+        }else if( strcmp(type,"a")==0 ){
+          rcf = sqlite3_create_function(dbDst,name,nargs,ienc,0,
+                                        0,dummyUDF,dummyUDFvalue);
+        }else if( strcmp(type,"s")==0 ){
+          rcf = sqlite3_create_function(dbDst,name,nargs,ienc,0,
+                                        dummyUDF,0,0);
+        }
+        if( rcf!=SQLITE_OK ){
+          rc = rcf;
+          break;
+        }
+      }
+    }
+    sqlite3_finalize(pStmt);
+    if( rc==SQLITE_DONE ) rc = SQLITE_OK;
+  }
+  return rc;
+}
+#endif
+
 /*
 ** Allocate a new sqlite3expert object.
 */
@@ -1845,6 +1927,20 @@ sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){
     }
   }
 
+  /* Allow custom collations to be dealt with through prepare. */
+  if( rc==SQLITE_OK ) rc = sqlite3_collation_needed(pNew->dbm,0,useDummyCS);
+  if( rc==SQLITE_OK ) rc = sqlite3_collation_needed(pNew->dbv,0,useDummyCS);
+
+#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) \
+  && !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS)
+  /* Register UDFs from database [db] with [dbm] and [dbv]. */
+  if( rc==SQLITE_OK ){
+    rc = registerUDFs(pNew->db, pNew->dbm);
+  }
+  if( rc==SQLITE_OK ){
+    rc = registerUDFs(pNew->db, pNew->dbv);
+  }
+#endif
 
   /* Copy the entire schema of database [db] into [dbm]. */
   if( rc==SQLITE_OK ){
@@ -1920,6 +2016,10 @@ int sqlite3_expert_sql(
 
   while( rc==SQLITE_OK && zStmt && zStmt[0] ){
     sqlite3_stmt *pStmt = 0;
+    /* Ensure that the provided statement compiles against user's DB. */
+    rc = idxPrepareStmt(p->db, &pStmt, pzErr, zStmt);
+    if( rc!=SQLITE_OK ) break;
+    sqlite3_finalize(pStmt);
     rc = sqlite3_prepare_v2(p->dbv, zStmt, -1, &pStmt, &zStmt);
     if( rc==SQLITE_OK ){
       if( pStmt ){

ext/fts3/fts3.c

@@ -640,6 +640,7 @@ static void fts3DeclareVtab(int *pRc, Fts3Table *p){
 
     zLanguageid = (p->zLanguageid ? p->zLanguageid : "__langid");
     sqlite3_vtab_config(p->db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1);
+    sqlite3_vtab_config(p->db, SQLITE_VTAB_INNOCUOUS);
 
     /* Create a list of user columns for the virtual table */
     zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]);
@@ -3889,6 +3890,8 @@ static int fts3RenameMethod(
     rc = sqlite3Fts3PendingTermsFlush(p);
   }
 
+  p->bIgnoreSavepoint = 1;
+
   if( p->zContentTbl==0 ){
     fts3DbExec(&rc, db,
       "ALTER TABLE %Q.'%q_content'  RENAME TO '%q_content';",
@@ -3916,6 +3919,8 @@ static int fts3RenameMethod(
     "ALTER TABLE %Q.'%q_segdir'   RENAME TO '%q_segdir';",
     p->zDb, p->zName, zName
   );
+
+  p->bIgnoreSavepoint = 0;
   return rc;
 }
 
@@ -3926,12 +3931,28 @@ static int fts3RenameMethod(
 */
 static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
   int rc = SQLITE_OK;
-  UNUSED_PARAMETER(iSavepoint);
+  Fts3Table *pTab = (Fts3Table*)pVtab;
-  assert( ((Fts3Table *)pVtab)->inTransaction );
+  assert( pTab->inTransaction );
-  assert( ((Fts3Table *)pVtab)->mxSavepoint <= iSavepoint );
+  assert( pTab->mxSavepoint<=iSavepoint );
-  TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint );
+  TESTONLY( pTab->mxSavepoint = iSavepoint );
-  if( ((Fts3Table *)pVtab)->bIgnoreSavepoint==0 ){
+
-    rc = fts3SyncMethod(pVtab);
+  if( pTab->bIgnoreSavepoint==0 ){
+    if( fts3HashCount(&pTab->aIndex[0].hPending)>0 ){
+      char *zSql = sqlite3_mprintf("INSERT INTO %Q.%Q(%Q) VALUES('flush')",
+          pTab->zDb, pTab->zName, pTab->zName
+          );
+      if( zSql ){
+        pTab->bIgnoreSavepoint = 1;
+        rc = sqlite3_exec(pTab->db, zSql, 0, 0, 0);
+        pTab->bIgnoreSavepoint = 0;
+        sqlite3_free(zSql);
+      }else{
+        rc = SQLITE_NOMEM;
+      }
+    }
+    if( rc==SQLITE_OK ){
+      pTab->iSavepoint = iSavepoint+1;
+    }
   }
   return rc;
 }
@@ -3942,12 +3963,11 @@ static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
 ** This is a no-op.
 */
 static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
-  TESTONLY( Fts3Table *p = (Fts3Table*)pVtab );
+  Fts3Table *pTab = (Fts3Table*)pVtab;
-  UNUSED_PARAMETER(iSavepoint);
+  assert( pTab->inTransaction );
-  UNUSED_PARAMETER(pVtab);
+  assert( pTab->mxSavepoint >= iSavepoint );
-  assert( p->inTransaction );
+  TESTONLY( pTab->mxSavepoint = iSavepoint-1 );
-  assert( p->mxSavepoint >= iSavepoint );
+  pTab->iSavepoint = iSavepoint;
-  TESTONLY( p->mxSavepoint = iSavepoint-1 );
   return SQLITE_OK;
 }
 
@@ -3957,11 +3977,13 @@ static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
 ** Discard the contents of the pending terms table.
 */
 static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){
-  Fts3Table *p = (Fts3Table*)pVtab;
+  Fts3Table *pTab = (Fts3Table*)pVtab;
   UNUSED_PARAMETER(iSavepoint);
-  assert( p->inTransaction );
+  assert( pTab->inTransaction );
-  TESTONLY( p->mxSavepoint = iSavepoint );
+  TESTONLY( pTab->mxSavepoint = iSavepoint );
-  sqlite3Fts3PendingTermsClear(p);
+  if( (iSavepoint+1)<=pTab->iSavepoint ){
+    sqlite3Fts3PendingTermsClear(pTab);
+  }
   return SQLITE_OK;
 }
 
@@ -3980,8 +4002,40 @@ static int fts3ShadowName(const char *zName){
   return 0;
 }
 
+/*
+** Implementation of the xIntegrity() method on the FTS3/FTS4 virtual
+** table.
+*/
+static int fts3Integrity(sqlite3_vtab *pVtab, char **pzErr){
+  Fts3Table *p = (Fts3Table*)pVtab;
+  char *zSql;
+  int rc;
+  char *zErr = 0;
+
+  zSql = sqlite3_mprintf(
+            "INSERT INTO \"%w\".\"%w\"(\"%w\") VALUES('integrity-check');",
+            p->zDb, p->zName, p->zName);
+  if( zSql==0 ){
+    return SQLITE_NOMEM;
+  }
+  rc = sqlite3_exec(p->db, zSql, 0, 0, &zErr);
+  sqlite3_free(zSql);
+  if( (rc&0xff)==SQLITE_CORRUPT ){
+    *pzErr = sqlite3_mprintf("malformed inverted index for FTS%d table %s.%s",
+                p->bFts4 ? 4 : 3, p->zDb, p->zName);
+  }else if( rc!=SQLITE_OK ){
+    *pzErr = sqlite3_mprintf("unable to validate the inverted index for"
+                             " FTS%d table %s.%s: %s",
+                p->bFts4 ? 4 : 3, p->zDb, p->zName, zErr);
+  }
+  sqlite3_free(zErr);
+  return SQLITE_OK;
+}
+
+
+
 static const sqlite3_module fts3Module = {
-  /* iVersion      */ 3,
+  /* iVersion      */ 4,
   /* xCreate       */ fts3CreateMethod,
   /* xConnect      */ fts3ConnectMethod,
   /* xBestIndex    */ fts3BestIndexMethod,
@@ -4005,6 +4059,7 @@ static const sqlite3_module fts3Module = {
   /* xRelease      */ fts3ReleaseMethod,
   /* xRollbackTo   */ fts3RollbackToMethod,
   /* xShadowName   */ fts3ShadowName,
+  /* xIntegrity    */ fts3Integrity,
 };
 
 /*

ext/fts3/fts3Int.h

@@ -265,6 +265,7 @@ struct Fts3Table {
   int nPgsz;                      /* Page size for host database */
   char *zSegmentsTbl;             /* Name of %_segments table */
   sqlite3_blob *pSegments;        /* Blob handle open on %_segments table */
+  int iSavepoint;
 
   /* 
   ** The following array of hash tables is used to buffer pending index 

ext/fts3/fts3_aux.c

@@ -545,7 +545,8 @@ int sqlite3Fts3InitAux(sqlite3 *db){
      0,                           /* xSavepoint    */
      0,                           /* xRelease      */
      0,                           /* xRollbackTo   */
-     0                            /* xShadowName   */
+     0,                           /* xShadowName   */
+     0                            /* xIntegrity    */
   };
   int rc;                         /* Return code */
 

ext/fts3/fts3_term.c

@@ -362,7 +362,8 @@ int sqlite3Fts3InitTerm(sqlite3 *db){
      0,                           /* xSavepoint    */
      0,                           /* xRelease      */
      0,                           /* xRollbackTo   */
-     0                            /* xShadowName   */
+     0,                           /* xShadowName   */
+     0                            /* xIntegrity    */
   };
   int rc;                         /* Return code */
 

ext/fts3/fts3_tokenize_vtab.c

@@ -445,7 +445,8 @@ int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash, void(*xDestroy)(void*)){
      0,                           /* xSavepoint    */
      0,                           /* xRelease      */
      0,                           /* xRollbackTo   */
-     0                            /* xShadowName   */
+     0,                           /* xShadowName   */
+     0                            /* xIntegrity    */
   };
   int rc;                         /* Return code */
 

ext/fts3/fts3_write.c

@@ -3325,7 +3325,6 @@ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){
     rc = fts3SegmentMerge(p, p->iPrevLangid, i, FTS3_SEGCURSOR_PENDING);
     if( rc==SQLITE_DONE ) rc = SQLITE_OK;
   }
-  sqlite3Fts3PendingTermsClear(p);
 
   /* Determine the auto-incr-merge setting if unknown.  If enabled,
   ** estimate the number of leaf blocks of content to be written
@@ -3347,6 +3346,10 @@ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){
       rc = sqlite3_reset(pStmt);
     }
   }
+
+  if( rc==SQLITE_OK ){
+    sqlite3Fts3PendingTermsClear(p);
+  }
   return rc;
 }
 
@@ -4034,9 +4037,13 @@ static int fts3IncrmergeAppend(
   nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist;
 
   /* If the current block is not empty, and if adding this term/doclist
-  ** to the current block would make it larger than Fts3Table.nNodeSize
+  ** to the current block would make it larger than Fts3Table.nNodeSize bytes,
-  ** bytes, write this block out to the database. */
+  ** and if there is still room for another leaf page, write this block out to
-  if( pLeaf->block.n>0 && (pLeaf->block.n + nSpace)>p->nNodeSize ){
+  ** the database. */
+  if( pLeaf->block.n>0 
+   && (pLeaf->block.n + nSpace)>p->nNodeSize 
+   && pLeaf->iBlock < (pWriter->iStart + pWriter->nLeafEst)
+  ){
     rc = fts3WriteSegment(p, pLeaf->iBlock, pLeaf->block.a, pLeaf->block.n);
     pWriter->nWork++;
 
@@ -5218,7 +5225,7 @@ static u64 fts3ChecksumIndex(
   int rc;
   u64 cksum = 0;
 
-  assert( *pRc==SQLITE_OK );
+  if( *pRc ) return 0;
 
   memset(&filter, 0, sizeof(filter));
   memset(&csr, 0, sizeof(csr));
@@ -5433,8 +5440,11 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){
     rc = fts3DoIncrmerge(p, &zVal[6]);
   }else if( nVal>10 && 0==sqlite3_strnicmp(zVal, "automerge=", 10) ){
     rc = fts3DoAutoincrmerge(p, &zVal[10]);
+  }else if( nVal==5 && 0==sqlite3_strnicmp(zVal, "flush", 5) ){
+    rc = sqlite3Fts3PendingTermsFlush(p);
+  }
 #if defined(SQLITE_DEBUG) || defined(SQLITE_TEST)
-  }else{
+  else{
     int v;
     if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){
       v = atoi(&zVal[9]);
@@ -5452,8 +5462,8 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){
       if( v>=4 && v<=FTS3_MERGE_COUNT && (v&1)==0 ) p->nMergeCount = v;
       rc = SQLITE_OK;
     }
-#endif
   }
+#endif
   return rc;
 }
 

ext/fts5/fts5_index.c

@@ -2904,7 +2904,6 @@ static int fts5MultiIterDoCompare(Fts5Iter *pIter, int iOut){
       assert_nc( i2!=0 );
       pRes->bTermEq = 1;
       if( p1->iRowid==p2->iRowid ){
-        p1->bDel = p2->bDel;
         return i2;
       }
       res = ((p1->iRowid > p2->iRowid)==pIter->bRev) ? -1 : +1;
@@ -3272,7 +3271,7 @@ static Fts5Iter *fts5MultiIterAlloc(
   int nSeg
 ){
   Fts5Iter *pNew;
-  int nSlot;                      /* Power of two >= nSeg */
+  i64 nSlot;                      /* Power of two >= nSeg */
 
   for(nSlot=2; nSlot<nSeg; nSlot=nSlot*2);
   pNew = fts5IdxMalloc(p, 
@@ -5048,7 +5047,6 @@ static void fts5DoSecureDelete(
   int iPgIdx = pSeg->pLeaf->szLeaf;
 
   u64 iDelta = 0;
-  u64 iNextDelta = 0;
   int iNextOff = 0;
   int iOff = 0;
   int nIdx = 0;
@@ -5056,8 +5054,6 @@ static void fts5DoSecureDelete(
   int bLastInDoclist = 0;
   int iIdx = 0;
   int iStart = 0;
-  int iKeyOff = 0;
-  int iPrevKeyOff = 0;
   int iDelKeyOff = 0;       /* Offset of deleted key, if any */
 
   nIdx = nPg-iPgIdx;
@@ -5082,10 +5078,21 @@ static void fts5DoSecureDelete(
   ** This block sets the following variables:
   **
   **   iStart:
+  **     The offset of the first byte of the rowid or delta-rowid
+  **     value for the doclist entry being removed.
+  **
   **   iDelta:
+  **     The value of the rowid or delta-rowid value for the doclist
+  **     entry being removed.
+  **
+  **   iNextOff:
+  **     The offset of the next entry following the position list
+  **     for the one being removed. If the position list for this
+  **     entry overflows onto the next leaf page, this value will be
+  **     greater than pLeaf->szLeaf.
   */
   {
-    int iSOP;
+    int iSOP;                     /* Start-Of-Position-list */
     if( pSeg->iLeafPgno==pSeg->iTermLeafPgno ){
       iStart = pSeg->iTermLeafOffset;
     }else{
@@ -5121,14 +5128,21 @@ static void fts5DoSecureDelete(
   }
 
   iOff = iStart;
+
+  /* Set variable bLastInDoclist to true if this entry happens to be
+  ** the last rowid in the doclist for its term.  */
+  if( pSeg->bDel==0 ){
     if( iNextOff>=iPgIdx ){
       int pgno = pSeg->iLeafPgno+1;
       fts5SecureDeleteOverflow(p, pSeg->pSeg, pgno, &bLastInDoclist);
       iNextOff = iPgIdx;
     }else{
-    /* Set bLastInDoclist to true if the entry being removed is the last
+      /* Loop through the page-footer. If iNextOff (offset of the
+      ** entry following the one we are removing) is equal to the 
+      ** offset of a key on this page, then the entry is the last 
       ** in its doclist.  */
-    for(iIdx=0, iKeyOff=0; iIdx<nIdx; /* no-op */){
+      int iKeyOff = 0;
+      for(iIdx=0; iIdx<nIdx; /* no-op */){
         u32 iVal = 0;
         iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
         iKeyOff += iVal;
@@ -5138,30 +5152,51 @@ static void fts5DoSecureDelete(
       }
     }
 
-  if( fts5GetU16(&aPg[0])==iStart && (bLastInDoclist||iNextOff==iPgIdx) ){
+    /* If this is (a) the first rowid on a page and (b) is not followed by
+    ** another position list on the same page, set the "first-rowid" field
+    ** of the header to 0.  */
+    if( fts5GetU16(&aPg[0])==iStart && (bLastInDoclist || iNextOff==iPgIdx) ){
       fts5PutU16(&aPg[0], 0);
     }
+  }
 
-  if( bLastInDoclist==0 ){
+  if( pSeg->bDel ){
+    iOff += sqlite3Fts5PutVarint(&aPg[iOff], iDelta);
+    aPg[iOff++] = 0x01;
+  }else if( bLastInDoclist==0 ){
     if( iNextOff!=iPgIdx ){
+      u64 iNextDelta = 0;
       iNextOff += fts5GetVarint(&aPg[iNextOff], &iNextDelta);
       iOff += sqlite3Fts5PutVarint(&aPg[iOff], iDelta + iNextDelta);
     }
   }else if( 
-      iStart==pSeg->iTermLeafOffset && pSeg->iLeafPgno==pSeg->iTermLeafPgno 
+      pSeg->iLeafPgno==pSeg->iTermLeafPgno 
+   && iStart==pSeg->iTermLeafOffset 
   ){
     /* The entry being removed was the only position list in its
     ** doclist. Therefore the term needs to be removed as well. */
     int iKey = 0;
-    for(iIdx=0, iKeyOff=0; iIdx<nIdx; iKey++){
+    int iKeyOff = 0;
+
+    /* Set iKeyOff to the offset of the term that will be removed - the
+    ** last offset in the footer that is not greater than iStart. */
+    for(iIdx=0; iIdx<nIdx; iKey++){
       u32 iVal = 0;
       iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
       if( (iKeyOff+iVal)>(u32)iStart ) break;
       iKeyOff += iVal;
     }
+    assert_nc( iKey>=1 );
 
+    /* Set iDelKeyOff to the value of the footer entry to remove from 
+    ** the page. */
     iDelKeyOff = iOff = iKeyOff;
+
     if( iNextOff!=iPgIdx ){
+      /* This is the only position-list associated with the term, and there
+      ** is another term following it on this page. So the subsequent term
+      ** needs to be moved to replace the term associated with the entry
+      ** being removed. */
       int nPrefix = 0;
       int nSuffix = 0;
       int nPrefix2 = 0;
@@ -5240,27 +5275,35 @@ static void fts5DoSecureDelete(
     }
   }
 
+  /* Assuming no error has occurred, this block does final edits to the
+  ** leaf page before writing it back to disk. Input variables are:
+  **
+  **   nPg: Total initial size of leaf page.
+  **   iPgIdx: Initial offset of page footer.
+  **
+  **   iOff: Offset to move data to
+  **   iNextOff: Offset to move data from
+  */
   if( p->rc==SQLITE_OK ){
-      const int nMove = nPg - iNextOff;
+    const int nMove = nPg - iNextOff;     /* Number of bytes to move */
-      int nShift = 0;
+    int nShift = iNextOff - iOff;         /* Distance to move them */
+
+    int iPrevKeyOut = 0;
+    int iKeyIn = 0;
 
     memmove(&aPg[iOff], &aPg[iNextOff], nMove);
-      iPgIdx -= (iNextOff - iOff);
+    iPgIdx -= nShift;
     nPg = iPgIdx;
     fts5PutU16(&aPg[2], iPgIdx);
 
-      nShift = iNextOff - iOff;
+    for(iIdx=0; iIdx<nIdx; /* no-op */){
-      for(iIdx=0, iKeyOff=0, iPrevKeyOff=0; iIdx<nIdx; /* no-op */){
       u32 iVal = 0;
       iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
-        iKeyOff += iVal;
+      iKeyIn += iVal;
-        if( iKeyOff!=iDelKeyOff ){
+      if( iKeyIn!=iDelKeyOff ){
-          if( iKeyOff>iOff ){
+        int iKeyOut = (iKeyIn - (iKeyIn>iOff ? nShift : 0));
-            iKeyOff -= nShift;
+        nPg += sqlite3Fts5PutVarint(&aPg[nPg], iKeyOut - iPrevKeyOut);
-            nShift = 0;
+        iPrevKeyOut = iKeyOut;
-          }
-          nPg += sqlite3Fts5PutVarint(&aPg[nPg], iKeyOff - iPrevKeyOff);
-          iPrevKeyOff = iKeyOff;
       }
     }
 
@@ -5269,7 +5312,7 @@ static void fts5DoSecureDelete(
     }
 
     assert_nc( nPg>4 || fts5GetU16(aPg)==0 );
-      fts5DataWrite(p, FTS5_SEGMENT_ROWID(iSegid,pSeg->iLeafPgno), aPg,nPg);
+    fts5DataWrite(p, FTS5_SEGMENT_ROWID(iSegid,pSeg->iLeafPgno), aPg, nPg);
   }
   sqlite3_free(aIdx);
 }
@@ -5441,10 +5484,16 @@ static void fts5FlushOneHash(Fts5Index *p){
                 fts5WriteFlushLeaf(p, &writer);
               }
             }else{
-              int bDummy;
+              int bDel = 0;
-              int nPos;
+              int nPos = 0;
-              int nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDummy);
+              int nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDel);
+              if( bDel && bSecureDelete ){
+                fts5BufferAppendVarint(&p->rc, pBuf, nPos*2);
+                iOff += nCopy;
+                nCopy = nPos;
+              }else{
                 nCopy += nPos;
+              }
               if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){
                 /* The entire poslist will fit on the current leaf. So copy
                 ** it in one go. */
@@ -5482,7 +5531,6 @@ static void fts5FlushOneHash(Fts5Index *p){
         assert( pBuf->n<=pBuf->nSpace );
         if( p->rc==SQLITE_OK ) sqlite3Fts5HashScanNext(pHash);
       }
-      sqlite3Fts5HashClear(pHash);
       fts5WriteFinish(p, &writer, &pgnoLast);
   
       assert( p->rc!=SQLITE_OK || bSecureDelete || pgnoLast>0 );
@@ -5515,7 +5563,6 @@ static void fts5FlushOneHash(Fts5Index *p){
   fts5IndexCrisismerge(p, &pStruct);
   fts5StructureWrite(p, pStruct);
   fts5StructureRelease(pStruct);
-  p->nContentlessDelete = 0;
 }
 
 /*
@@ -5526,8 +5573,12 @@ static void fts5IndexFlush(Fts5Index *p){
   if( p->nPendingData || p->nContentlessDelete ){
     assert( p->pHash );
     fts5FlushOneHash(p);
+    if( p->rc==SQLITE_OK ){
+      sqlite3Fts5HashClear(p->pHash);
       p->nPendingData = 0;
       p->nPendingRow = 0;
+      p->nContentlessDelete = 0;
+    }
   }
 }
 
@@ -8269,7 +8320,8 @@ int sqlite3Fts5IndexInit(sqlite3 *db){
       0,                           /* xSavepoint    */
       0,                           /* xRelease      */
       0,                           /* xRollbackTo   */
-      0                            /* xShadowName   */
+      0,                           /* xShadowName   */
+      0                            /* xIntegrity    */
     };
     rc = sqlite3_create_module(db, "fts5_structure", &fts5structure_module, 0);
   }

ext/fts5/fts5_main.c

@@ -117,6 +117,8 @@ struct Fts5FullTable {
   Fts5Storage *pStorage;          /* Document store */
   Fts5Global *pGlobal;            /* Global (connection wide) data */
   Fts5Cursor *pSortCsr;           /* Sort data from this cursor */
+  int iSavepoint;                 /* Successful xSavepoint()+1 */
+  int bInSavepoint;
 #ifdef SQLITE_DEBUG
   struct Fts5TransactionState ts;
 #endif
@@ -405,6 +407,13 @@ static int fts5InitVtab(
     pConfig->pzErrmsg = 0;
   }
 
+  if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){
+    rc = sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, (int)1);
+  }
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS);
+  }
+
   if( rc!=SQLITE_OK ){
     fts5FreeVtab(pTab);
     pTab = 0;
@@ -1329,6 +1338,9 @@ static int fts5FilterMethod(
     pCsr->iFirstRowid = fts5GetRowidLimit(pRowidGe, SMALLEST_INT64);
   }
 
+  rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex);
+  if( rc!=SQLITE_OK ) goto filter_out;
+
   if( pTab->pSortCsr ){
     /* If pSortCsr is non-NULL, then this call is being made as part of 
     ** processing for a "... MATCH <expr> ORDER BY rank" query (ePlan is
@@ -1351,6 +1363,7 @@ static int fts5FilterMethod(
     pCsr->pExpr = pTab->pSortCsr->pExpr;
     rc = fts5CursorFirst(pTab, pCsr, bDesc);
   }else if( pCsr->pExpr ){
+    assert( rc==SQLITE_OK );
     rc = fts5CursorParseRank(pConfig, pCsr, pRank);
     if( rc==SQLITE_OK ){
       if( bOrderByRank ){
@@ -1522,6 +1535,7 @@ static int fts5SpecialInsert(
   Fts5Config *pConfig = pTab->p.pConfig;
   int rc = SQLITE_OK;
   int bError = 0;
+  int bLoadConfig = 0;
 
   if( 0==sqlite3_stricmp("delete-all", zCmd) ){
     if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
@@ -1533,6 +1547,7 @@ static int fts5SpecialInsert(
     }else{
       rc = sqlite3Fts5StorageDeleteAll(pTab->pStorage);
     }
+    bLoadConfig = 1;
   }else if( 0==sqlite3_stricmp("rebuild", zCmd) ){
     if( pConfig->eContent==FTS5_CONTENT_NONE ){
       fts5SetVtabError(pTab, 
@@ -1542,6 +1557,7 @@ static int fts5SpecialInsert(
     }else{
       rc = sqlite3Fts5StorageRebuild(pTab->pStorage);
     }
+    bLoadConfig = 1;
   }else if( 0==sqlite3_stricmp("optimize", zCmd) ){
     rc = sqlite3Fts5StorageOptimize(pTab->pStorage);
   }else if( 0==sqlite3_stricmp("merge", zCmd) ){
@@ -1554,6 +1570,8 @@ static int fts5SpecialInsert(
   }else if( 0==sqlite3_stricmp("prefix-index", zCmd) ){
     pConfig->bPrefixIndex = sqlite3_value_int(pVal);
 #endif
+  }else if( 0==sqlite3_stricmp("flush", zCmd) ){
+    rc = sqlite3Fts5FlushToDisk(&pTab->p);
   }else{
     rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex);
     if( rc==SQLITE_OK ){
@@ -1567,6 +1585,12 @@ static int fts5SpecialInsert(
       }
     }
   }
+
+  if( rc==SQLITE_OK && bLoadConfig ){
+    pTab->p.pConfig->iCookie--;
+    rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex);
+  }
+
   return rc;
 }
 
@@ -1685,7 +1709,7 @@ static int fts5UpdateMethod(
     assert( nArg!=1 || eType0==SQLITE_INTEGER );
 
     /* Filter out attempts to run UPDATE or DELETE on contentless tables.
-    ** This is not suported. Except - DELETE is supported if the CREATE
+    ** This is not suported. Except - they are both supported if the CREATE
     ** VIRTUAL TABLE statement contained "contentless_delete=1". */
     if( eType0==SQLITE_INTEGER 
      && pConfig->eContent==FTS5_CONTENT_NONE 
@@ -1714,7 +1738,8 @@ static int fts5UpdateMethod(
       }
 
       else if( eType0!=SQLITE_INTEGER ){     
-        /* If this is a REPLACE, first remove the current entry (if any) */
+        /* An INSERT statement. If the conflict-mode is REPLACE, first remove
+        ** the current entry (if any). */
         if( eConflict==SQLITE_REPLACE && eType1==SQLITE_INTEGER ){
           i64 iNew = sqlite3_value_int64(apVal[1]);  /* Rowid to delete */
           rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0);
@@ -2589,8 +2614,12 @@ static int fts5RenameMethod(
   sqlite3_vtab *pVtab,            /* Virtual table handle */
   const char *zName               /* New name of table */
 ){
+  int rc;
   Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
-  return sqlite3Fts5StorageRename(pTab->pStorage, zName);
+  pTab->bInSavepoint = 1;
+  rc = sqlite3Fts5StorageRename(pTab->pStorage, zName);
+  pTab->bInSavepoint = 0;
+  return rc;
 }
 
 int sqlite3Fts5FlushToDisk(Fts5Table *pTab){
@@ -2604,9 +2633,29 @@ int sqlite3Fts5FlushToDisk(Fts5Table *pTab){
 ** Flush the contents of the pending-terms table to disk.
 */
 static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
-  UNUSED_PARAM(iSavepoint);  /* Call below is a no-op for NDEBUG builds */
+  Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
-  fts5CheckTransactionState((Fts5FullTable*)pVtab, FTS5_SAVEPOINT, iSavepoint);
+  int rc = SQLITE_OK;
-  return sqlite3Fts5FlushToDisk((Fts5Table*)pVtab);
+  char *zSql = 0;
+  fts5CheckTransactionState(pTab, FTS5_SAVEPOINT, iSavepoint);
+
+  if( pTab->bInSavepoint==0 ){
+    zSql = sqlite3_mprintf("INSERT INTO %Q.%Q(%Q) VALUES('flush')",
+        pTab->p.pConfig->zDb, pTab->p.pConfig->zName, pTab->p.pConfig->zName
+    );
+    if( zSql ){
+      pTab->bInSavepoint = 1;
+      rc = sqlite3_exec(pTab->p.pConfig->db, zSql, 0, 0, 0);
+      pTab->bInSavepoint = 0;
+      sqlite3_free(zSql);
+    }else{
+      rc = SQLITE_NOMEM;
+    }
+    if( rc==SQLITE_OK ){
+      pTab->iSavepoint = iSavepoint+1;
+    }
+  }
+
+  return rc;
 }
 
 /*
@@ -2615,9 +2664,16 @@ static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
 ** This is a no-op.
 */
 static int fts5ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
-  UNUSED_PARAM(iSavepoint);  /* Call below is a no-op for NDEBUG builds */
+  Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
-  fts5CheckTransactionState((Fts5FullTable*)pVtab, FTS5_RELEASE, iSavepoint);
+  int rc = SQLITE_OK;
-  return sqlite3Fts5FlushToDisk((Fts5Table*)pVtab);
+  fts5CheckTransactionState(pTab, FTS5_RELEASE, iSavepoint);
+  if( (iSavepoint+1)<pTab->iSavepoint ){
+    rc = sqlite3Fts5FlushToDisk(&pTab->p);
+    if( rc==SQLITE_OK ){
+      pTab->iSavepoint = iSavepoint;
+    }
+  }
+  return rc;
 }
 
 /*
@@ -2627,11 +2683,14 @@ static int fts5ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
 */
 static int fts5RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){
   Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
-  UNUSED_PARAM(iSavepoint);  /* Call below is a no-op for NDEBUG builds */
+  int rc = SQLITE_OK;
   fts5CheckTransactionState(pTab, FTS5_ROLLBACKTO, iSavepoint);
   fts5TripCursors(pTab);
   pTab->p.pConfig->pgsz = 0;
-  return sqlite3Fts5StorageRollback(pTab->pStorage);
+  if( (iSavepoint+1)<=pTab->iSavepoint ){
+    rc = sqlite3Fts5StorageRollback(pTab->pStorage);
+  }
+  return rc;
 }
 
 /*
@@ -2851,9 +2910,38 @@ static int fts5ShadowName(const char *zName){
   return 0;
 }
 
+/*
+** Run an integrity check on the FTS5 data structures.  Return a string
+** if anything is found amiss.  Return a NULL pointer if everything is
+** OK.
+*/
+static int fts5Integrity(sqlite3_vtab *pVtab, char **pzErr){
+  Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
+  Fts5Config *pConfig = pTab->p.pConfig;
+  char *zSql;
+  char *zErr = 0;
+  int rc;
+  zSql = sqlite3_mprintf(
+            "INSERT INTO \"%w\".\"%w\"(\"%w\") VALUES('integrity-check');",
+            pConfig->zDb, pConfig->zName, pConfig->zName);
+  if( zSql==0 ) return SQLITE_NOMEM;
+  rc = sqlite3_exec(pConfig->db, zSql, 0, 0, &zErr);
+  sqlite3_free(zSql);
+  if( (rc&0xff)==SQLITE_CORRUPT ){
+    *pzErr = sqlite3_mprintf("malformed inverted index for FTS5 table %s.%s",
+                pConfig->zDb, pConfig->zName);
+  }else if( rc!=SQLITE_OK ){
+    *pzErr = sqlite3_mprintf("unable to validate the inverted index for"
+                             " FTS5 table %s.%s: %s",
+                pConfig->zDb, pConfig->zName, zErr);
+  }
+  sqlite3_free(zErr);
+  return SQLITE_OK;
+}
+
 static int fts5Init(sqlite3 *db){
   static const sqlite3_module fts5Mod = {
-    /* iVersion      */ 3,
+    /* iVersion      */ 4,
     /* xCreate       */ fts5CreateMethod,
     /* xConnect      */ fts5ConnectMethod,
     /* xBestIndex    */ fts5BestIndexMethod,
@@ -2876,7 +2964,8 @@ static int fts5Init(sqlite3 *db){
     /* xSavepoint    */ fts5SavepointMethod,
     /* xRelease      */ fts5ReleaseMethod,
     /* xRollbackTo   */ fts5RollbackToMethod,
-    /* xShadowName   */ fts5ShadowName
+    /* xShadowName   */ fts5ShadowName,
+    /* xIntegrity    */ fts5Integrity
   };
 
   int rc;

ext/fts5/fts5_storage.c

@@ -1184,8 +1184,10 @@ int sqlite3Fts5StorageSync(Fts5Storage *p){
   i64 iLastRowid = sqlite3_last_insert_rowid(p->pConfig->db);
   if( p->bTotalsValid ){
     rc = fts5StorageSaveTotals(p);
+    if( rc==SQLITE_OK ){
       p->bTotalsValid = 0;
     }
+  }
   if( rc==SQLITE_OK ){
     rc = sqlite3Fts5IndexSync(p->pIndex);
   }

ext/fts5/fts5_test_tok.c

@@ -472,7 +472,8 @@ int sqlite3Fts5TestRegisterTok(sqlite3 *db, fts5_api *pApi){
      0,                           /* xSavepoint    */
      0,                           /* xRelease      */
      0,                           /* xRollbackTo   */
-     0                            /* xShadowName   */
+     0,                           /* xShadowName   */
+     0                            /* xIntegrity    */
   };
   int rc;                         /* Return code */
 

ext/fts5/fts5_vocab.c

@@ -783,7 +783,8 @@ int sqlite3Fts5VocabInit(Fts5Global *pGlobal, sqlite3 *db){
     /* xSavepoint    */ 0,
     /* xRelease      */ 0,
     /* xRollbackTo   */ 0,
-    /* xShadowName   */ 0
+    /* xShadowName   */ 0,
+    /* xIntegrity    */ 0
   };
   void *p = (void*)pGlobal;
 

ext/fts5/test/fts5aa.test

@@ -65,7 +65,9 @@ foreach w {a b c d e f} {
 
 do_execsql_test 2.4 {
   INSERT INTO t1(t1) VALUES('integrity-check');
-}
+  PRAGMA integrity_check;
+  PRAGMA integrity_check(t1);
+} {ok ok}
 
 
 #-------------------------------------------------------------------------
@@ -88,6 +90,7 @@ foreach {i x y} {
 } {
   do_execsql_test 3.$i.1 { INSERT INTO t1 VALUES($x, $y) }
   do_execsql_test 3.$i.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
+  do_execsql_test 3.$i.3 { PRAGMA integrity_check(t1) } ok
   if {[set_test_counter errors]} break
 }
 
@@ -135,7 +138,7 @@ foreach {i x y} {
    10 {ddd abcde dddd dd c} {dddd c c d abcde}
 } {
   do_execsql_test 5.$i.1 { INSERT INTO t1 VALUES($x, $y) }
-  do_execsql_test 5.$i.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
+  do_execsql_test 5.$i.2 { PRAGMA integrity_check(t1) } ok
   if {[set_test_counter errors]} break
 }
 

ext/fts5/test/fts5aux.test

@@ -307,5 +307,31 @@ do_catchsql_test 10.1.4 {
   SELECT group_concat(firstcol(t1), '.') FROM t1 GROUP BY rowid
 } {1 {unable to use function firstcol in the requested context}}
 
-finish_test
+#-------------------------------------------------------------------------
+# Test that xInstCount() works from within an xPhraseQuery() callback.
+#
+reset_db
 
+proc xCallback {cmd} {
+  incr ::hitcount [$cmd xInstCount]
+  return SQLITE_OK
+}
+proc fts5_hitcount {cmd} {
+  set ::hitcount 0
+  $cmd xQueryPhrase 0 xCallback
+  return $::hitcount
+}
+sqlite3_fts5_create_function db fts5_hitcount fts5_hitcount
+
+do_execsql_test 11.1 {
+  CREATE VIRTUAL TABLE x1 USING fts5(z);
+  INSERT INTO x1 VALUES('one two three');
+  INSERT INTO x1 VALUES('one two one three one');
+  INSERT INTO x1 VALUES('one two three');
+}
+
+do_execsql_test 11.2 {
+  SELECT fts5_hitcount(x1) FROM x1('one') LIMIT 1;
+} {5}
+
+finish_test

ext/fts5/test/fts5conflict.test

@@ -65,4 +65,44 @@ do_execsql_test 2.1 {
   INSERT INTO fts_idx(fts_idx) VALUES('integrity-check');
 }
 
+#-------------------------------------------------------------------------
+# Tests for OR IGNORE conflict handling.
+#
+reset_db
+foreach_detail_mode $::testprefix {
+
+  do_execsql_test 3.0 {
+    CREATE VIRTUAL TABLE t1 USING fts5(xyz, detail=%DETAIL%);
+
+    BEGIN;
+    INSERT INTO t1(rowid, xyz) VALUES(13, 'thirteen documents');
+    INSERT INTO t1(rowid, xyz) VALUES(14, 'fourteen documents');
+    INSERT INTO t1(rowid, xyz) VALUES(15, 'fifteen documents');
+    COMMIT;
+  }
+
+  set db_cksum [cksum]
+    foreach {tn sql} {
+    1 {
+      INSERT OR IGNORE INTO t1(rowid, xyz) VALUES(14, 'new text');
+    }
+    2 {
+      UPDATE OR IGNORE t1 SET rowid=13 WHERE rowid=15;
+    }
+    3 {
+      INSERT OR IGNORE INTO t1(rowid, xyz) 
+      SELECT 13, 'some text'
+      UNION ALL
+      SELECT 14, 'some text'
+      UNION ALL
+      SELECT 15, 'some text'
+    }
+  } {
+    do_execsql_test 3.1.$tn.1 $sql
+    do_test 3.1.$tn.2 { cksum } $db_cksum
+  }
+
+}
+
+
 finish_test

ext/fts5/test/fts5content.test

@@ -294,4 +294,3 @@ do_catchsql_test 7.2.5 {
 } {1 {recursively defined fts5 content table}}
 
 finish_test
-

ext/fts5/test/fts5contentless.test

@@ -268,4 +268,3 @@ do_execsql_test 8.2 {
 } {}
 
 finish_test
-

ext/fts5/test/fts5contentless2.test

@@ -205,4 +205,3 @@ foreach {tn step} {
 
 
 finish_test
-

ext/fts5/test/fts5contentless3.test

@@ -193,4 +193,3 @@ do_execsql_test 3.7 {
 
 
 finish_test
-

ext/fts5/test/fts5contentless4.test

@@ -245,4 +245,3 @@ do_execsql_test 4.3 {
 }
 
 finish_test
-

ext/fts5/test/fts5contentless5.test

@@ -56,4 +56,3 @@ foreach {tn up err} {
 }
 
 finish_test
-

ext/fts5/test/fts5corrupt.test

@@ -48,6 +48,10 @@ do_test 1.3 {
   }
   catchsql { INSERT INTO t1(t1) VALUES('integrity-check') }
 } {1 {database disk image is malformed}}
+do_execsql_test 1.3b {
+  PRAGMA integrity_check(t1);
+} {{malformed inverted index for FTS5 table main.t1}}
+
 
 do_test 1.4 {
   db_restore_and_reopen

ext/fts5/test/fts5corrupt2.test

@@ -167,6 +167,9 @@ foreach {tn hdr} {
       do_test 3.$tn.$tn2.2 {
         catchsql { INSERT INTO x3(x3) VALUES('integrity-check') }
       } {1 {database disk image is malformed}}
+      do_execsql_test 3.$tn.$tn2.3 {
+        PRAGMA integrity_check(x3);
+      } {{malformed inverted index for FTS5 table main.x3}}
     }
 
     execsql ROLLBACK

ext/fts5/test/fts5faultG.test

@@ -0,0 +1,50 @@
+# 2010 June 15
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+source $testdir/malloc_common.tcl
+set testprefix fts5faultG
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+set ::testprefix fts5faultG
+
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a);
+  INSERT INTO t1 VALUES('test renaming the table');
+  INSERT INTO t1 VALUES(' after it has been written');
+  INSERT INTO t1 VALUES(' actually other stuff instead');
+}
+faultsim_save_and_close
+do_faultsim_test 1 -faults oom* -prep { 
+  faultsim_restore_and_reopen
+  execsql {
+    BEGIN;
+      DELETE FROM t1 WHERE rowid=2;
+  }
+} -body {
+  execsql {
+    DELETE FROM t1;
+  }
+} -test {
+  catchsql { COMMIT }
+  faultsim_integrity_check
+  faultsim_test_result {0 {}}
+}
+
+
+finish_test

ext/fts5/test/fts5integrity.test

@@ -77,6 +77,9 @@ do_catchsql_test 4.2 {
     UPDATE aa_docsize SET sz = X'44' WHERE rowid = 3;
     INSERT INTO aa(aa) VALUES('integrity-check'); 
 } {1 {database disk image is malformed}}
+do_execsql_test 4.2.1 {
+  PRAGMA integrity_check(aa);
+} {{malformed inverted index for FTS5 table main.aa}}
 
 do_catchsql_test 4.3 { 
   ROLLBACK;
@@ -317,4 +320,39 @@ do_catchsql_test 10.5.3 {
   INSERT INTO vt0(vt0) VALUES('integrity-check');
 } {0 {}}
 
+reset_db
+proc slang {in} {return [string map {th d e eh} $in]}
+db function slang -deterministic -innocuous slang
+do_execsql_test 11.0 {
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT, c TEXT AS (slang(b)));
+  INSERT INTO t1(b) VALUES('the quick fox jumps over the lazy brown dog');
+  SELECT c FROM t1;
+} {{deh quick fox jumps ovehr deh lazy brown dog}}
+
+do_execsql_test 11.1 {
+  CREATE VIRTUAL TABLE t2 USING fts5(content="t1", c);
+  INSERT INTO t2(t2) VALUES('rebuild');
+  SELECT rowid FROM t2 WHERE t2 MATCH 'deh';
+} {1}
+
+do_execsql_test 11.2 {
+  PRAGMA integrity_check(t2);
+} {ok}
+db close
+sqlite3 db test.db
+
+# FIX ME?
+#
+# FTS5 integrity-check does not care if the content table is unreadable or
+# does not exist.  It only looks for internal inconsistencies in the
+# inverted index.
+#
+do_execsql_test 11.3 {
+  PRAGMA integrity_check(t2);
+} {ok}
+do_execsql_test 11.4 {
+  DROP TABLE t1;
+  PRAGMA integrity_check(t2);
+} {ok}
+
 finish_test

ext/fts5/test/fts5misc.test

@@ -44,12 +44,12 @@ do_catchsql_test 1.2.2 {
 
 do_catchsql_test 1.3.1 { 
   SELECT highlight(t1, 4, '<b>', '</b>') FROM t1('*reads'); 
-} {1 {no such cursor: 1}}
+} {1 {no such cursor: 2}}
 
 do_catchsql_test 1.3.2 {
   SELECT a FROM t1
     WHERE rank = (SELECT highlight(t1, 4, '<b>', '</b>') FROM t1('*reads'));
-} {1 {no such cursor: 1}}
+} {1 {no such cursor: 2}}
 
 db close
 sqlite3 db test.db

ext/fts5/test/fts5optimize2.test

@@ -9,7 +9,7 @@
 #
 #***********************************************************************
 #
-# TESTRUNNER: slow
+# TESTRUNNER: superslow
 #
 
 source [file join [file dirname [info script]] fts5_common.tcl]
@@ -42,23 +42,4 @@ do_execsql_test 1.2 {
   SELECT count(*) FROM t1('mno')
 } $nLoop
 
-do_execsql_test 2.0 {
-  CREATE VIRTUAL TABLE t2 USING fts5(x);
-  INSERT INTO t2(t2, rank) VALUES('pgsz', 32);
-}
-
-do_test 2.1 {
-  for {set ii 0} {$ii < $nLoop} {incr ii} {
-    execsql {
-      INSERT INTO t2 VALUES('abc def ghi');
-      INSERT INTO t2 VALUES('jkl mno pqr');
-      INSERT INTO t2(t2, rank) VALUES('merge', -1);
-    }
-  }
-} {}
-
-do_execsql_test 2.2 {
-  SELECT count(*) FROM t2('mno')
-} $nLoop
-
 finish_test

ext/fts5/test/fts5optimize3.test

@@ -0,0 +1,45 @@
+# 2023 Aug 27
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# TESTRUNNER: superslow
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5optimize2
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+set nLoop 2500
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t2 USING fts5(x);
+  INSERT INTO t2(t2, rank) VALUES('pgsz', 32);
+}
+
+do_test 1.1 {
+  for {set ii 0} {$ii < $nLoop} {incr ii} {
+    execsql {
+      INSERT INTO t2 VALUES('abc def ghi');
+      INSERT INTO t2 VALUES('jkl mno pqr');
+      INSERT INTO t2(t2, rank) VALUES('merge', -1);
+    }
+  }
+} {}
+
+do_execsql_test 1.2 {
+  SELECT count(*) FROM t2('mno')
+} $nLoop
+
+finish_test

ext/fts5/test/fts5rank.test

@@ -180,4 +180,28 @@ do_execsql_test 6.1 {
   {table table table} {the table names.} {rank on an fts5 table}
 }
 
+
+#-------------------------------------------------------------------------
+# forum post: https://sqlite.org/forum/forumpost/a2dd636330
+#
+reset_db
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t USING fts5 (a, b);  
+  INSERT INTO t (a, b) VALUES ('data1', 'sentence1'), ('data2', 'sentence2'); 
+  INSERT INTO t(t, rank) VALUES ('rank', 'bm25(10.0,1.0)');
+}
+
+sqlite3 db2 test.db 
+do_execsql_test -db db2 1.1 {
+  SELECT *, rank<0.0 FROM t('data*') ORDER BY RANK;
+} {data1 sentence1 1 data2 sentence2 1}
+
+do_execsql_test 1.2 {
+  INSERT INTO t(t, rank) VALUES ('rank', 'bm25(10.0,1.0)');
+}
+do_execsql_test -db db2 1.3 {
+  SELECT *, rank<0.0 FROM t('data*') ORDER BY RANK;
+} {data1 sentence1 1 data2 sentence2 1}
+db2 close
+
 finish_test

ext/fts5/test/fts5savepoint.test

@@ -71,7 +71,7 @@ ifcapable fts3 {
 
   do_catchsql_test 3.2 {
     DROP TABLE vt1;
-  } {1 {SQL logic error}}
+  } {0 {}}
 
   do_execsql_test 3.3 {
     SAVEPOINT x;

ext/fts5/test/fts5secure.test

@@ -273,6 +273,76 @@ do_execsql_test 5.3 {
 do_execsql_test 5.4 { SELECT rowid FROM t1('abc'); } 2
 do_execsql_test 5.5 { SELECT rowid FROM t1('aa'); } 2
 
+#-------------------------------------------------------------------------
+# Tests for the bug fixed by https://sqlite.org/src/info/4b60a1c3
+#
+reset_db
+do_execsql_test 6.0 {
+  CREATE VIRTUAL TABLE fts USING fts5(content);
+  INSERT INTO fts(fts, rank) VALUES ('secure-delete', 1);
+  INSERT INTO fts(rowid, content) VALUES
+    (3407, 'profile profile profile profile profile profile profile profile pull pulling pulling really');
+  DELETE FROM fts WHERE rowid IS 3407;
+  INSERT INTO fts(fts) VALUES ('integrity-check');
+}
+
+foreach {tn detail} {
+  1 full
+  2 column
+  3 none
+} {
+  do_execsql_test 6.1.$detail "
+    DROP TABLE IF EXISTS t1;
+    CREATE VIRTUAL TABLE t1 USING fts5(x, detail=$detail);
+  "
+
+  do_execsql_test 6.2.$detail {
+    INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
+  }
+
+  for {set ii 1} {$ii < 100} {incr ii} {
+    do_execsql_test 6.3.$detail.$ii.1 {
+      BEGIN;
+        INSERT INTO t1(rowid, x) VALUES(10, 'word1');
+        WITH s(i) AS (
+          SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<CAST($ii AS integer)
+        )
+        INSERT INTO t1(x) SELECT 'word3' FROM s;
+      COMMIT;
+      INSERT INTO t1(t1) VALUES('optimize');
+    }
+
+    do_execsql_test 6.3.$detail.$ii.2 {
+      DELETE FROM t1 WHERE rowid=10;
+      INSERT INTO t1(t1) VALUES ('integrity-check');
+    }
+
+    do_execsql_test 6.3.$detail.$ii.3 {
+      DELETE FROM t1;
+    }
+
+    do_execsql_test 6.3.$detail.$ii.4 {
+      BEGIN;
+        INSERT INTO t1(rowid, x) VALUES(10, 'tokenA');
+        WITH s(i) AS (
+          SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<CAST($ii AS integer)
+        )
+        INSERT INTO t1(x) SELECT group_concat('tokenB ') FROM s;
+      COMMIT;
+      INSERT INTO t1(t1) VALUES('optimize');
+    }
+
+    do_execsql_test 6.3.$detail.$ii.5 {
+      DELETE FROM t1 WHERE rowid=10;
+      INSERT INTO t1(t1) VALUES ('integrity-check');
+    }
+
+    do_execsql_test 6.3.$detail.$ii.6 {
+      DELETE FROM t1;
+    }
+  }
+}
+
 
 finish_test
 

ext/fts5/test/fts5secure6.test

@@ -18,7 +18,7 @@ db progress 1 progress_handler
 set ::PHC 0
 proc progress_handler {args} {
   incr ::PHC
-  if {($::PHC % 100000)==0}  breakpoint
+  # if {($::PHC % 100000)==0}  breakpoint
   return 0
 }
 
@@ -70,5 +70,72 @@ do_execsql_test 2.2 {
   SELECT rowid FROM t1('def')
 } {-100000 -99999 9223372036854775800}
 
+#-------------------------------------------------------------------------
+reset_db
+
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x);
+  INSERT INTO t1(t1, rank) VALUES('secure-delete', $sd) 
+}
+
+do_execsql_test 3.1 {
+  BEGIN;
+    INSERT INTO t1(rowid, x) 
+      VALUES(51869, 'when whenever where weress what turn'), 
+            (51871, 'to were');
+  COMMIT;
+}
+
+do_execsql_test 3.2 {
+  DELETE FROM t1 WHERE rowid=51871;
+  INSERT INTO t1(t1) VALUES('integrity-check');
+}
+
+#-------------------------------------------------------------------------
+reset_db
+
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x);
+  INSERT INTO t1(rowid, x) VALUES(10, 'one two');
+}
+do_execsql_test 4.1 {
+  UPDATE t1 SET x = 'one three' WHERE rowid=10;
+  INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
+}
+do_execsql_test 4.2 {
+  DELETE FROM t1 WHERE rowid=10;
+}
+do_execsql_test 4.3 {
+  INSERT INTO t1(t1) VALUES('integrity-check');
+}
+
+#-------------------------------------------------------------------------
+reset_db
+
+do_execsql_test 5.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(content);
+
+  INSERT INTO t1(t1,rank) VALUES('secure-delete',1);
+  INSERT INTO t1 VALUES('active'),('boomer'),('atom'),('atomic'),
+         ('alpha channel backup abandon test aback boomer atom alpha active');
+  DELETE FROM t1 WHERE t1 MATCH 'abandon';
+} 
+
+do_execsql_test 5.1 {
+  INSERT INTO t1(t1) VALUES('rebuild');
+}
+
+do_execsql_test 5.2 {
+  DELETE FROM t1 WHERE rowid NOTNULL<5;
+}
+
+db close
+sqlite3 db test.db
+
+do_execsql_test 5.3 {
+  PRAGMA integrity_check;
+} {ok}
+
+
 finish_test
 

ext/fts5/test/fts5secure7.test

@@ -0,0 +1,116 @@
+# 2023 Feb 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+# TESTRUNNER: slow
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+ifcapable !fts5 { finish_test ; return }
+set ::testprefix fts5secure7
+
+
+set NVOCAB 500
+set NDOC [expr 1000]
+
+set NREP 100
+set nDeletePerRep [expr 5]
+
+set VOCAB [list]
+
+proc select_one {list} {
+  set n [llength $list]
+  lindex $list [expr {abs(int(rand()*$n))}]
+}
+
+proc init_vocab {} {
+  set L [split "abcdefghijklmnopqrstuvwxyz" {}]
+  set nL [llength $L]
+  for {set i 0} {$i < $::NVOCAB} {incr i} {
+    set n [expr {6 + int(rand()*8)}]
+    set word ""
+    for {set j 0} {$j < $n} {incr j} {
+      append word [select_one $L]
+    }
+    lappend ::VOCAB $word
+  }
+}
+
+proc get_word {} {
+  select_one $::VOCAB
+}
+
+proc get_document {nWord} {
+  set ret [list]
+  for {set i 0} {$i < $nWord} {incr i} {
+    lappend ret [get_word]
+  }
+  return $ret
+}
+
+init_vocab
+
+db func document [list get_document 12]
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(body);
+  INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
+}
+do_execsql_test 1.1 {
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<$NDOC
+  )
+  INSERT INTO t1 SELECT document() FROM s;
+}
+
+for {set iRep 0} {$iRep < $NREP} {incr iRep} {
+  set lRowid [db eval {SELECT rowid FROM t1}]
+  for {set iDel 0} {$iDel < $nDeletePerRep} {incr iDel} {
+    set idx [select_one $lRowid]
+    db eval {
+      DELETE FROM t1 WHERE rowid=$idx
+    }
+  }
+  db eval {
+    WITH s(i) AS (
+      SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<$nDeletePerRep
+    )
+    INSERT INTO t1 SELECT document() FROM s;
+  }
+  do_execsql_test 1.2.$iRep {
+    INSERT INTO t1(t1) VALUES('integrity-check');
+  }
+}
+
+reset_db
+db func document [list get_document 12]
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(body);
+  INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
+  INSERT INTO t1(t1, rank) VALUES('pgsz', 128);
+}
+do_execsql_test 2.1 {
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<$NDOC
+  )
+  INSERT INTO t1 SELECT document() FROM s;
+}
+for {set ii 0} {$ii < $NDOC} {incr ii} {
+  set lRowid [db eval {SELECT rowid FROM t1}]
+  set idx [select_one $lRowid]
+  db eval { DELETE FROM t1 WHERE rowid=$idx }
+  do_execsql_test 2.2.$ii {
+    INSERT INTO t1(t1) VALUES('integrity-check');
+  }
+}
+
+finish_test
+
+

ext/jni/GNUmakefile

@@ -6,9 +6,10 @@ JAVA_HOME ?= $(HOME)/jdk/current
 # e.g. /usr/lib/jvm/default-javajava-19-openjdk-amd64
 JDK_HOME ?= $(JAVA_HOME)
 # ^^^ JDK_HOME is not as widely used as JAVA_HOME
-bin.javac := $(JDK_HOME)/bin/javac
-bin.java  := $(JDK_HOME)/bin/java
 bin.jar     := $(JDK_HOME)/bin/jar
+bin.java    := $(JDK_HOME)/bin/java
+bin.javac   := $(JDK_HOME)/bin/javac
+bin.javadoc := $(JDK_HOME)/bin/javadoc
 ifeq (,$(wildcard $(JDK_HOME)))
 $(error set JDK_HOME to the top-most dir of your JDK installation.)
 endif
@@ -20,15 +21,24 @@ package.jar := sqlite3-jni.jar
 dir.top     := ../..
 dir.tool    := ../../tool
 dir.jni     := $(patsubst %/,%,$(dir $(MAKEFILE)))
-
 dir.src     := $(dir.jni)/src
 dir.src.c   := $(dir.src)/c
 dir.bld     := $(dir.jni)/bld
 dir.bld.c   := $(dir.bld)
 dir.src.jni := $(dir.src)/org/sqlite/jni
-dir.src.jni.tester := $(dir.src.jni)/tester
+dir.src.capi := $(dir.src.jni)/capi
+dir.src.fts5 := $(dir.src.jni)/fts5
+dir.tests   := $(dir.src)/tests
+mkdir       ?= mkdir -p
 $(dir.bld.c):
-	mkdir -p $@
+	$(mkdir) $@
+
+javac.flags ?= -Xlint:unchecked -Xlint:deprecation
+java.flags ?=
+jnicheck ?= 1
+ifeq (1,$(jnicheck))
+  java.flags += -Xcheck:jni
+endif
 
 classpath := $(dir.src)
 CLEAN_FILES := $(package.jar)
@@ -36,17 +46,28 @@ DISTCLEAN_FILES := $(dir.jni)/*~ $(dir.src.c)/*~ $(dir.src.jni)/*~
 
 sqlite3-jni.h := $(dir.src.c)/sqlite3-jni.h
 .NOTPARALLEL: $(sqlite3-jni.h)
-SQLite3Jni.java := src/org/sqlite/jni/SQLite3Jni.java
+CApi.java := $(dir.src.capi)/CApi.java
-SQLTester.java := src/org/sqlite/jni/tester/SQLTester.java
+SQLTester.java := $(dir.src.capi)/SQLTester.java
-SQLite3Jni.class := $(SQLite3Jni.java:.java=.class)
+CApi.class := $(CApi.java:.java=.class)
 SQLTester.class := $(SQLTester.java:.java=.class)
 
 ########################################################################
 # The future of FTS5 customization in this API is as yet unclear.
-# It would be a real doozy to bind to JNI.
+# The pieces are all in place, and are all thin proxies so not much
+# complexity, but some semantic changes were required in porting
+# which are largely untested.
+#
+# Reminder: this flag influences the contents of $(sqlite3-jni.h),
+# which is checked in. Please do not check in changes to that file in
+# which the fts5 APIs have been stripped unless that feature is
+# intended to be stripped for good.
 enable.fts5 ?= 1
-# If enable.tester is 0, the org/sqlite/jni/tester/* bits are elided.
+
-enable.tester ?= 1
+ifeq (,$(wildcard $(dir.tests)/*))
+  enable.tester := 0
+else
+  enable.tester := 1
+endif
 
 # bin.version-info = binary to output various sqlite3 version info
 # building the distribution zip file.
@@ -57,68 +78,95 @@ $(bin.version-info): $(dir.tool)/version-info.c $(sqlite3.h) $(dir.top)/Makefile
 
 # Be explicit about which Java files to compile so that we can work on
 # in-progress files without requiring them to be in a compilable statae.
-JAVA_FILES.main := $(patsubst %,$(dir.src.jni)/%,\
+JAVA_FILES.main := $(patsubst %,$(dir.src.jni)/annotation/%,\
-  BusyHandler.java \
+  NotNull.java \
-  Collation.java \
+  Nullable.java \
-  CollationNeeded.java \
+) $(patsubst %,$(dir.src.capi)/%,\
-  CommitHook.java \
+  AbstractCollationCallback.java \
+  AggregateFunction.java \
+  AuthorizerCallback.java \
+  AutoExtensionCallback.java \
+  BusyHandlerCallback.java \
+  CollationCallback.java \
+  CollationNeededCallback.java \
+  CommitHookCallback.java \
+  ConfigLogCallback.java \
+  ConfigSqllogCallback.java \
   NativePointerHolder.java \
   OutputPointer.java \
-  ProgressHandler.java \
+  PrepareMultiCallback.java \
+  PreupdateHookCallback.java \
+  ProgressHandlerCallback.java \
   ResultCode.java \
-  RollbackHook.java \
+  RollbackHookCallback.java \
+  ScalarFunction.java \
   SQLFunction.java \
-  sqlite3_context.java \
+  CallbackProxy.java \
+  CApi.java \
+  TableColumnMetadata.java \
+  TraceV2Callback.java \
+  UpdateHookCallback.java \
+  ValueHolder.java \
+  WindowFunction.java \
+  XDestroyCallback.java \
   sqlite3.java \
-  SQLite3Jni.java \
+  sqlite3_context.java \
   sqlite3_stmt.java \
   sqlite3_value.java \
-  Tester1.java \
+) $(patsubst %,$(dir.src.jni)/wrapper1/%,\
-  Tracer.java \
+  AggregateFunction.java \
-  UpdateHook.java \
+  ScalarFunction.java \
+  SqlFunction.java \
+  Sqlite.java \
+  SqliteException.java \
   ValueHolder.java \
 )
+
+JAVA_FILES.unittest := $(patsubst %,$(dir.src.jni)/%,\
+  capi/Tester1.java \
+  wrapper1/Tester2.java \
+)
 ifeq (1,$(enable.fts5))
-  JAVA_FILES.main += $(patsubst %,$(dir.src.jni)/%,\
+  JAVA_FILES.unittest += $(patsubst %,$(dir.src.fts5)/%,\
+    TesterFts5.java \
+  )
+  JAVA_FILES.main += $(patsubst %,$(dir.src.fts5)/%,\
     fts5_api.java \
     fts5_extension_function.java \
     fts5_tokenizer.java \
     Fts5.java \
     Fts5Context.java \
     Fts5ExtensionApi.java \
-    Fts5Function.java \
     Fts5PhraseIter.java \
     Fts5Tokenizer.java \
-    TesterFts5.java \
+    XTokenizeCallback.java \
   )
 endif
-JAVA_FILES.tester := $(dir.src.jni.tester)/SQLTester.java
+JAVA_FILES.tester := $(SQLTester.java)
+JAVA_FILES.package.info := \
+  $(dir.src.jni)/package-info.java \
+  $(dir.src.jni)/annotation/package-info.java
 
 CLASS_FILES.main := $(JAVA_FILES.main:.java=.class)
+CLASS_FILES.unittest := $(JAVA_FILES.unittest:.java=.class)
 CLASS_FILES.tester := $(JAVA_FILES.tester:.java=.class)
 
-JAVA_FILES += $(JAVA_FILES.main)
+JAVA_FILES += $(JAVA_FILES.main) $(JAVA_FILES.unittest)
 ifeq (1,$(enable.tester))
   JAVA_FILES += $(JAVA_FILES.tester)
 endif
 
 CLASS_FILES :=
-define DOTCLASS_DEPS
+define CLASSFILE_DEPS
-$(1).class: $(1).java $(MAKEFILE)
 all: $(1).class
 CLASS_FILES += $(1).class
 endef
-$(foreach B,$(basename $(JAVA_FILES)),$(eval $(call DOTCLASS_DEPS,$(B))))
+$(foreach B,$(basename \
-$(CLASS_FILES.tester): $(CLASS_FILES.main)
+  $(JAVA_FILES.main) $(JAVA_FILES.unittest) $(JAVA_FILES.tester)),\
-javac.flags ?= -Xlint:unchecked -Xlint:deprecation
+  $(eval $(call CLASSFILE_DEPS,$(B))))
-java.flags ?=
+$(CLASS_FILES): $(JAVA_FILES) $(MAKEFILE)
-jnicheck ?= 1
-ifeq (1,$(jnicheck))
-  java.flags += -Xcheck:jni
-endif
-$(SQLite3Jni.class): $(JAVA_FILES)
 	$(bin.javac) $(javac.flags) -h $(dir.bld.c) -cp $(classpath) $(JAVA_FILES)
-all: $(SQLite3Jni.class)
+
 #.PHONY: classfiles
 
 ########################################################################
@@ -152,27 +200,40 @@ $(sqlite3.h):
 	$(MAKE) -C $(dir.top) sqlite3.c
 $(sqlite3.c): $(sqlite3.h)
 
+opt.threadsafe ?= 1
+opt.fatal-oom ?= 1
+opt.debug ?= 1
+opt.metrics ?= 1
 SQLITE_OPT = \
-  -DSQLITE_ENABLE_RTREE \
+  -DSQLITE_THREADSAFE=$(opt.threadsafe) \
+  -DSQLITE_TEMP_STORE=2 \
+  -DSQLITE_USE_URI=1 \
+  -DSQLITE_OMIT_LOAD_EXTENSION \
+  -DSQLITE_OMIT_DEPRECATED \
+  -DSQLITE_OMIT_SHARED_CACHE \
+  -DSQLITE_C=$(sqlite3.c) \
+  -DSQLITE_JNI_FATAL_OOM=$(opt.fatal-oom) \
+  -DSQLITE_JNI_ENABLE_METRICS=$(opt.metrics)
+
+opt.extras ?= 1
+ifeq (1,$(opt.extras))
+SQLITE_OPT += -DSQLITE_ENABLE_RTREE \
   -DSQLITE_ENABLE_EXPLAIN_COMMENTS \
   -DSQLITE_ENABLE_STMTVTAB \
   -DSQLITE_ENABLE_DBPAGE_VTAB \
   -DSQLITE_ENABLE_DBSTAT_VTAB \
   -DSQLITE_ENABLE_BYTECODE_VTAB \
   -DSQLITE_ENABLE_OFFSET_SQL_FUNC \
-  -DSQLITE_OMIT_LOAD_EXTENSION \
+  -DSQLITE_ENABLE_PREUPDATE_HOOK \
-  -DSQLITE_OMIT_DEPRECATED \
+  -DSQLITE_ENABLE_NORMALIZE \
-  -DSQLITE_OMIT_SHARED_CACHE \
+  -DSQLITE_ENABLE_SQLLOG
-  -DSQLITE_THREADSAFE=0 \
+endif
-  -DSQLITE_TEMP_STORE=2 \
-  -DSQLITE_USE_URI=1 \
-  -DSQLITE_C=$(sqlite3.c)
-#  -DSQLITE_DEBUG
-# -DSQLITE_DEBUG is just to work around a -Wall warning
-# for a var which gets set in all builds but only read
-# via assert().
 
-SQLITE_OPT += -g -DDEBUG -UNDEBUG
+ifeq (1,$(opt.debug))
+  SQLITE_OPT += -DSQLITE_DEBUG -g -DDEBUG -UNDEBUG
+else
+  SQLITE_OPT += -Os
+endif
 
 ifeq (1,$(enable.fts5))
   SQLITE_OPT += -DSQLITE_ENABLE_FTS5
@@ -181,25 +242,30 @@ endif
 sqlite3-jni.c := $(dir.src.c)/sqlite3-jni.c
 sqlite3-jni.o := $(dir.bld.c)/sqlite3-jni.o
 sqlite3-jni.h   := $(dir.src.c)/sqlite3-jni.h
-sqlite3-jni.dll := $(dir.bld.c)/libsqlite3-jni.so
+package.dll := $(dir.bld.c)/libsqlite3-jni.so
 # All javac-generated .h files must be listed in $(sqlite3-jni.h.in):
 sqlite3-jni.h.in :=
+# $(java.with.jni) lists all Java files which contain JNI decls:
+java.with.jni :=
 define ADD_JNI_H
-sqlite3-jni.h.in += $$(dir.bld.c)/org_sqlite_jni_$(1).h
+sqlite3-jni.h.in += $$(dir.bld.c)/org_sqlite_jni$(3)_$(2).h
-$$(dir.bld.c)/org_sqlite_jni_$(1).h: $$(dir.src.jni)/$(1).java
+java.with.jni += $(1)/$(2).java
+$$(dir.bld.c)/org_sqlite_jni$(3)_$(2).h: $(1)/$(2).java
 endef
-$(eval $(call ADD_JNI_H,SQLite3Jni))
+# Invoke ADD_JNI_H once for each Java file which includes JNI
+# declarations:
+$(eval $(call ADD_JNI_H,$(dir.src.capi),CApi,_capi))
+$(eval $(call ADD_JNI_H,$(dir.src.capi),SQLTester,_capi))
 ifeq (1,$(enable.fts5))
-  $(eval $(call ADD_JNI_H,Fts5ExtensionApi))
+ $(eval $(call ADD_JNI_H,$(dir.src.fts5),Fts5ExtensionApi,_fts5))
-  $(eval $(call ADD_JNI_H,fts5_api))
+ $(eval $(call ADD_JNI_H,$(dir.src.fts5),fts5_api,_fts5))
-  $(eval $(call ADD_JNI_H,fts5_tokenizer))
+ $(eval $(call ADD_JNI_H,$(dir.src.fts5),fts5_tokenizer,_fts5))
 endif
-ifeq (1,$(enable.tester))
+$(sqlite3-jni.h.in): $(dir.bld.c)
-  sqlite3-jni.h.in += $(dir.bld.c)/org_sqlite_jni_tester_SQLTester.h
+
-  $(dir.bld.c)/org_sqlite_jni_tester_SQLTester.h: $(dir.src.jni.tester)/SQLTester.java
+#package.dll.cfiles :=
-endif
+package.dll.cflags = \
-#sqlite3-jni.dll.cfiles := $(dir.src.c)
+  -std=c99 \
-sqlite3-jni.dll.cflags = \
   -fPIC \
   -I. \
   -I$(dir $(sqlite3.h)) \
@@ -207,54 +273,82 @@ sqlite3-jni.dll.cflags = \
   -I$(JDK_HOME)/include \
   $(patsubst %,-I%,$(patsubst %.h,,$(wildcard $(JDK_HOME)/include/*))) \
   -Wall
-# Using (-Wall -Wextra) triggers an untennable number of
-# gcc warnings from sqlite3.c for mundane things like
-# unused parameters.
-#
 # The gross $(patsubst...) above is to include the platform-specific
 # subdir which lives under $(JDK_HOME)/include and is a required
 # include path for client-level code.
+#
+# Using (-Wall -Wextra) triggers an untennable number of
+# gcc warnings from sqlite3.c for mundane things like
+# unused parameters.
 ########################################################################
 ifeq (1,$(enable.tester))
-  sqlite3-jni.dll.cflags += -DS3JNI_ENABLE_SQLTester
+  package.dll.cflags += -DSQLITE_JNI_ENABLE_SQLTester
 endif
-$(sqlite3-jni.h): $(sqlite3-jni.h.in) $(MAKEFILE)
-	cat $(sqlite3-jni.h.in) > $@
-$(sqlite3-jni.dll): $(sqlite3-jni.h) $(sqlite3.c) $(sqlite3.h)
-$(sqlite3-jni.dll): $(dir.bld.c) $(sqlite3-jni.c) $(SQLite3Jni.java) $(MAKEFILE)
-	$(CC) $(sqlite3-jni.dll.cflags) $(SQLITE_OPT) \
-			$(sqlite3-jni.c) -shared -o $@
-all: $(sqlite3-jni.dll)
 
-.PHONY: test
+$(sqlite3-jni.h): $(sqlite3-jni.h.in) $(MAKEFILE)
-test.flags ?= -v
+	@cat $(sqlite3-jni.h.in) > $@.tmp
-test: $(SQLite3Jni.class) $(sqlite3-jni.dll)
+	@if cmp $@ $@.tmp >/dev/null; then \
-	$(bin.java) -ea -Djava.library.path=$(dir.bld.c) \
+		rm -f $@.tmp; \
-		$(java.flags) -cp $(classpath) \
+		echo "$@ not modified"; \
-		org.sqlite.jni.Tester1 $(if $(test.flags),-- $(test.flags),)
+	else \
+		mv $@.tmp $@; \
+		echo "Updated $@"; \
+	fi
+	@if [ x1 != x$(enable.fts5) ]; then \
+		echo "*** REMINDER:"; \
+		echo "*** enable.fts5=0, so please do not check in changes to $@."; \
+	fi
+
+$(package.dll): $(sqlite3-jni.h) $(sqlite3.c) $(sqlite3.h)
+$(package.dll): $(sqlite3-jni.c) $(MAKEFILE)
+	$(CC) $(package.dll.cflags) $(SQLITE_OPT) \
+			$(sqlite3-jni.c) -shared -o $@
+all: $(package.dll)
+
+.PHONY: test test-one
+Tester1.flags ?=
+Tester2.flags ?=
+test.flags.jvm = -ea -Djava.library.path=$(dir.bld.c) \
+                  $(java.flags) -cp $(classpath)
+test.deps := $(CLASS_FILES) $(package.dll)
+test-one: $(test.deps)
+	$(bin.java) $(test.flags.jvm) org.sqlite.jni.capi.Tester1 $(Tester1.flags)
+	$(bin.java) $(test.flags.jvm) org.sqlite.jni.wrapper1.Tester2 $(Tester2.flags)
+test-sqllog: $(test.deps)
+	@echo "Testing with -sqllog..."
+	$(bin.java) $(test.flags.jvm) -sqllog
+test-mt: $(test.deps)
+	@echo "Testing in multi-threaded mode:";
+	$(bin.java) $(test.flags.jvm) org.sqlite.jni.capi.Tester1 \
+    -t 7 -r 50 -shuffle $(Tester1.flags)
+	$(bin.java) $(test.flags.jvm) org.sqlite.jni.wrapper1.Tester2 \
+    -t 7 -r 50 -shuffle $(Tester2.flags)
+
+test: test-one test-mt
+tests: test test-sqllog
 
 tester.scripts := $(sort $(wildcard $(dir.src)/tests/*.test))
 tester.flags ?= # --verbose
 .PHONY: tester tester-local tester-ext
 ifeq (1,$(enable.tester))
-tester-local: $(CLASS_FILES.tester) $(sqlite3-jni.dll)
+tester-local: $(CLASS_FILES.tester) $(package.dll)
 	$(bin.java) -ea -Djava.library.path=$(dir.bld.c) \
 		$(java.flags) -cp $(classpath) \
-		org.sqlite.jni.tester.SQLTester $(tester.flags) $(tester.scripts)
+		org.sqlite.jni.capi.SQLTester $(tester.flags) $(tester.scripts)
 tester: tester-local
 else
 tester:
-	@echo "SQLTester support is disabled. Build with enable.tester=1 to enable it."
+	@echo "SQLTester support is disabled."
 endif
 
-tester.extdir.default := src/tests/ext
+tester.extdir.default := $(dir.tests)/ext
 tester.extdir ?= $(tester.extdir.default)
 tester.extern-scripts := $(wildcard $(tester.extdir)/*.test)
 ifneq (,$(tester.extern-scripts))
 tester-ext:
 	$(bin.java) -ea -Djava.library.path=$(dir.bld.c) \
 		$(java.flags) -cp $(classpath) \
-		org.sqlite.jni.tester.SQLTester $(tester.flags) $(tester.extern-scripts)
+		org.sqlite.jni.capi.SQLTester $(tester.flags) $(tester.extern-scripts)
 else
 tester-ext:
 	@echo "******************************************************"; \
@@ -267,25 +361,79 @@ endif
 
 tester-ext: tester-local
 tester: tester-ext
-tests: test tester
+tests: tester
+########################################################################
+# Build each SQLITE_THREADMODE variant and run all tests against them.
+multitest: clean
+define MULTIOPT
+multitest: multitest-$(1)
+multitest-$(1):
+	$$(MAKE) opt.debug=$$(opt.debug) $(patsubst %,opt.%,$(2)) \
+		tests clean enable.fts5=1
+endef
+
+$(eval $(call MULTIOPT,01,threadsafe=0 oom=1))
+$(eval $(call MULTIOPT,00,threadsafe=0 oom=0))
+$(eval $(call MULTIOPT,11,threadsafe=1 oom=1))
+$(eval $(call MULTIOPT,10,threadsafe=1 oom=0))
+$(eval $(call MULTIOPT,21,threadsafe=2 oom=1))
+$(eval $(call MULTIOPT,20,threadsafe=2 oom=0))
+
+
+########################################################################
+# jar bundle...
 package.jar.in := $(abspath $(dir.src)/jar.in)
 CLEAN_FILES += $(package.jar.in)
-$(package.jar.in): $(MAKEFILE) $(CLASS_FILES.main)
+JAVA_FILES.jar := $(JAVA_FILES.main) $(JAVA_FILES.unittest) $(JAVA_FILES.package.info)
-	cd $(dir.src); ls -1 org/sqlite/jni/*.java org/sqlite/jni/*.class > $@
+CLASS_FILES.jar := $(filter-out %/package-info.class,$(JAVA_FILES.jar:.java=.class))
-	@ls -la $@
+$(package.jar.in): $(package.dll) $(MAKEFILE)
-	@echo "To use this jar you will need the -Djava.library.path=DIR/WITH/libsqlite3-jni.so flag."
+	ls -1 \
-	@echo "e.g. java -jar $@ -Djava.library.path=bld"
+		$(dir.src.jni)/*/*.java $(dir.src.jni)/*/*.class \
+	| sed -e 's,^$(dir.src)/,,' | sort > $@
 
-$(package.jar): $(CLASS_FILES) $(MAKEFILE) $(package.jar.in)
+$(package.jar): $(CLASS_FILES.jar) $(MAKEFILE) $(package.jar.in)
-	rm -f $(dir.src)/c/*~ $(dir.src.jni)/*~
+	@rm -f $(dir.src)/c/*~ $(dir.src.jni)/*~
-	cd $(dir.src); $(bin.jar) -cfe ../$@ org.sqlite.jni.Tester1 @$(package.jar.in)
+	cd $(dir.src); $(bin.jar) -cfe ../$@ org.sqlite.jni.capi.Tester1 @$(package.jar.in)
+	@ls -la $@
+	@echo "To use this jar you will need the -Djava.library.path=DIR/CONTAINING/libsqlite3-jni.so flag."
+	@echo "e.g. java -Djava.library.path=bld -jar $@"
 
 jar: $(package.jar)
+run-jar: $(package.jar) $(package.dll)
+	$(bin.java) -Djava.library.path=$(dir.bld) -jar $(package.jar) $(run-jar.flags)
 
+########################################################################
+# javadoc...
+dir.doc   := $(dir.jni)/javadoc
+doc.index := $(dir.doc)/index.html
+javadoc.exclude := -exclude org.sqlite.jni.fts5
+# ^^^^ 2023-09-13: elide the fts5 parts from the public docs for
+# the time being, as it's not clear where the Java bindings for
+# those bits are going.
+# javadoc.exclude += -exclude org.sqlite.jni.capi
+# ^^^^ exclude the capi API only for certain builds (TBD)
+$(doc.index): $(JAVA_FILES.main) $(MAKEFILE)
+	@if [ -d $(dir.doc) ]; then rm -fr $(dir.doc)/*; fi
+	$(bin.javadoc) -cp $(classpath) -d $(dir.doc) -quiet \
+		-subpackages org.sqlite.jni $(javadoc.exclude)
+	@echo "javadoc output is in $@"
+
+.PHONY: doc javadoc docserve
+.FORCE: doc
+doc: $(doc.index)
+javadoc: $(doc.index)
+# Force rebild of docs
+redoc:
+	@rm -f $(doc.index)
+	@$(MAKE) doc
+docserve: $(doc.index)
+	cd $(dir.doc) && althttpd -max-age 1 -page index.html
+########################################################################
+# Clean up...
 CLEAN_FILES += $(dir.bld.c)/* \
   $(dir.src.jni)/*.class \
-  $(dir.src.jni.tester)/*.class \
+  $(dir.src.jni)/*/*.class \
-  $(sqlite3-jni.dll) \
+  $(package.dll) \
   hs_err_pid*.log
 
 .PHONY: clean distclean
@@ -293,7 +441,7 @@ clean:
 	-rm -f $(CLEAN_FILES)
 distclean: clean
 	-rm -f $(DISTCLEAN_FILES)
-	-rm -fr $(dir.bld.c)
+	-rm -fr $(dir.bld.c) $(dir.doc)
 
 ########################################################################
 # disttribution bundle rules...
@@ -317,6 +465,10 @@ dist: \
     $(bin.version-info) $(sqlite3.canonical.c) \
     $(package.jar) $(MAKEFILE)
 	@echo "Making end-user deliverables..."
+	@echo "****************************************************************************"; \
+	echo  "*** WARNING: be sure to build this with JDK8 (javac 1.8) for compatibility."; \
+	echo  "*** reasons!"; $$($(bin.javac) -version); \
+	echo  "****************************************************************************"
 	@rm -fr $(dist-dir.top)
 	@mkdir -p $(dist-dir.src)
 	@cp -p $(dist.top.extras) $(dist-dir.top)/.

ext/jni/README.md

@@ -15,7 +15,10 @@ Technical support is available in the forum:
 
 > **FOREWARNING:** this subproject is very much in development and
   subject to any number of changes. Please do not rely on any
-  information about its API until this disclaimer is removed.
+  information about its API until this disclaimer is removed.  The JNI
+  bindings released with version 3.43 are a "tech preview" and 3.44
+  will be "final," at which point strong backward compatibility
+  guarantees will apply.
 
 Project goals/requirements:
 
@@ -40,13 +43,41 @@ Non-goals:
 - Creation of high-level OO wrapper APIs. Clients are free to create
   them off of the C-style API.
 
+- Support for mixed-mode operation, where client code accesses SQLite
+  both via the Java-side API and the C API via their own native
+  code. In such cases, proxy functionalities (primarily callback
+  handler wrappers of all sorts) may fail because the C-side use of
+  the SQLite APIs will bypass those proxies.
 
-Significant TODOs
-========================================================================
 
-- The initial beta release with version 3.43 has severe threading
+Hello World
-  limitations.  Namely, two threads cannot call into the JNI-bound API
+-----------------------------------------------------------------------
-  at once. This limitation will be remove in a subsequent release.
+
+```java
+import org.sqlite.jni.*;
+import static org.sqlite.jni.CApi.*;
+
+...
+
+final sqlite3 db = sqlite3_open(":memory:");
+try {
+  final int rc = sqlite3_errcode(db);
+  if( 0 != rc ){
+    if( null != db ){
+      System.out.print("Error opening db: "+sqlite3_errmsg(db));
+    }else{
+      System.out.print("Error opening db: rc="+rc);
+    }
+    ... handle error ...
+  }
+  // ... else use the db ...
+}finally{
+  // ALWAYS close databases using sqlite3_close() or sqlite3_close_v2()
+  // when done with them. All of their active statement handles must
+  // first have been passed to sqlite3_finalize().
+  sqlite3_close_v2(db);
+}
+```
 
 
 Building
@@ -60,55 +91,97 @@ The canonical builds assumes a Linux-like environment and requires:
 
 Put simply:
 
-```
+```console
 $ export JAVA_HOME=/path/to/jdk/root
 $ make
 $ make test
 $ make clean
 ```
 
+The jar distribution can be created with `make jar`, but note that it
+does not contain the binary DLL file. A different DLL is needed for
+each target platform.
+
+
 <a id='1to1ish'></a>
 One-to-One(-ish) Mapping to C
 ========================================================================
 
 This JNI binding aims to provide as close to a 1-to-1 experience with
-the C API as cross-language semantics allow. Exceptions are
+the C API as cross-language semantics allow. Interface changes are
 necessarily made where cross-language semantics do not allow a 1-to-1,
 and judiciously made where a 1-to-1 mapping would be unduly cumbersome
-to use in Java.
+to use in Java. In all cases, this binding makes every effort to
+provide semantics compatible with the C API documentation even if the
+interface to those semantics is slightly different.  Any cases which
+deviate from those semantics (either removing or adding semantics) are
+clearly documented.
 
-Golden Rule: _Never_ Throw from Callbacks
+Where it makes sense to do so for usability, Java-side overloads are
+provided which accept or return data in alternative forms or provide
+sensible default argument values. In all such cases they are thin
+proxies around the corresponding C APIs and do not introduce new
+semantics.
+
+In some very few cases, Java-specific capabilities have been added in
+new APIs, all of which have "_java" somewhere in their names.
+Examples include:
+
+- `sqlite3_result_java_object()`
+- `sqlite3_column_java_object()`
+- `sqlite3_column_java_casted()`
+- `sqlite3_value_java_object()`
+- `sqlite3_value_java_casted()`
+
+which, as one might surmise, collectively enable the passing of
+arbitrary Java objects from user-defined SQL functions through to the
+caller.
+
+
+Golden Rule: Garbage Collection Cannot Free SQLite Resources
 ------------------------------------------------------------------------
 
-JNI bindings which accept client-defined functions _must never throw
+It is important that all databases and prepared statement handles get
-exceptions_ unless _very explicitly documented_ as being
+cleaned up by client code. A database cannot be closed if it has open
-throw-safe. Exceptions are generally reserved for higher-level
+statement handles. `sqlite3_close()` fails if the db cannot be closed
-bindings which are constructed to specifically deal with them and
+whereas `sqlite3_close_v2()` recognizes that case and marks the db as
-ensure that they do not leak C-level resources. Some of the JNI
+a "zombie," pending finalization when the library detects that all
-bindings are provided as Java functions which expect this rule to
+pending statements have been closed. Be aware that Java garbage
-always hold.
+collection _cannot_ close a database or finalize a prepared statement.
+Those things require explicit API calls.
 
-UTF-8(-ish)
+
+Golden Rule #2: _Never_ Throw from Callbacks (Unless...)
 ------------------------------------------------------------------------
 
-SQLite internally uses UTF-8 encoding, whereas Java natively uses
+All routines in this API, barring explicitly documented exceptions,
-UTF-16.  Java JNI has routines for converting to and from UTF-8, _but_
+retain C-like semantics. For example, they are not permitted to throw
-Java uses what its docs call "[modified UTF-8][modutf8]." Care must be
+or propagate exceptions and must return error information (if any) via
-taken when converting Java strings to UTF-8 to ensure that the proper
+result codes or `null`. The only cases where the C-style APIs may
-conversion is performed. In short,
+throw is through client-side misuse, e.g. passing in a null where it
-`String.getBytes(StandardCharsets.UTF_8)` performs the proper
+shouldn't be used. The APIs clearly mark function parameters which
-conversion in Java, and there is no JNI C API for that conversion
+should not be null, but does not actively defend itself against such
-(JNI's `NewStringUTF()` returns MUTF-8).
+misuse. Some C-style APIs explicitly accept `null` as a no-op for
+usability's sake, and some of the JNI APIs deliberately return an
+error code, instead of segfaulting, when passed a `null`.
 
-Known consequences and limitations of this discrepancy include:
+Client-defined callbacks _must never throw exceptions_ unless _very
+explicitly documented_ as being throw-safe. Exceptions are generally
+reserved for higher-level bindings which are constructed to
+specifically deal with them and ensure that they do not leak C-level
+resources. In some cases, callback handlers are permitted to throw, in
+which cases they get translated to C-level result codes and/or
+messages. If a callback which is not permitted to throw throws, its
+exception may trigger debug output but will otherwise be suppressed.
 
-- Names of databases, tables, and collations must not contain
+The reason some callbacks are permitted to throw and others not is
-  characters which differ in MUTF-8 and UTF-8, or certain APIs will
+because all such callbacks act as proxies for C function callback
-  mis-translate them on their way between languages. APIs which
+interfaces and some of those interfaces have no error-reporting
-  transfer other client-side data to Java take extra care to
+mechanism. Those which are capable of propagating errors back through
-  convert the data at the cost of performance.
+the library convert exceptions from callbacks into corresponding
-
+C-level error information. Those which cannot propagate errors
-[modutf8]: https://docs.oracle.com/javase/8/docs/api/java/io/DataInput.html#modified-utf-8
+necessarily suppress any exceptions in order to maintain the C-style
+semantics of the APIs.
 
 
 Unwieldy Constructs are Re-mapped
@@ -126,7 +199,7 @@ A prime example of where interface changes for Java are necessary for
 usability is [registration of a custom
 collation](https://sqlite.org/c3ref/create_collation.html):
 
-```
+```c
 // C:
 int sqlite3_create_collation(sqlite3 * db, const char * name, int eTextRep,
                              void *pUserData,
@@ -145,7 +218,7 @@ passed that object as their first argument. That data is passed around
 bind that part as-is to Java, the result would be awkward to use (^Yes,
 we tried this.):
 
-```
+```java
 // Java:
 int sqlite3_create_collation(sqlite3 db, String name, int eTextRep,
                              Object pUserData, xCompareType xCompare);
@@ -160,20 +233,20 @@ for callbacks and (B) having their internal state provided separately,
 which is ill-fitting in Java. For the sake of usability, C APIs which
 follow that pattern use a slightly different Java interface:
 
-```
+```java
 int sqlite3_create_collation(sqlite3 db, String name, int eTextRep,
-                             Collation collation);
+                             SomeCallbackType collation);
 ```
 
-Where the `Collation` class has an abstract `xCompare()` method and
+Where the `Collation` class has an abstract `call()` method and
 no-op `xDestroy()` method which can be overridden if needed, leading to
 a much more Java-esque usage:
 
-```
+```java
-int rc = sqlite3_create_collation(db, "mycollation", SQLITE_UTF8, new Collation(){
+int rc = sqlite3_create_collation(db, "mycollation", SQLITE_UTF8, new SomeCallbackType(){
 
   // Required comparison function:
-  @Override public int xCompare(byte[] lhs, byte[] rhs){ ... }
+  @Override public int call(byte[] lhs, byte[] rhs){ ... }
 
   // Optional finalizer function:
   @Override public void xDestroy(){ ... }
@@ -188,8 +261,8 @@ int rc = sqlite3_create_collation(db, "mycollation", SQLITE_UTF8, new Collation(
 
 Noting that:
 
-- It is still possible to bind in call-scope-local state via closures,
+- It is possible to bind in call-scope-local state via closures, if
-  if desired.
+  desired, as opposed to packing it into the Collation object.
 
 - No capabilities of the C API are lost or unduly obscured via the
   above API reshaping, so power users need not make any compromises.
@@ -200,6 +273,7 @@ Noting that:
   overriding the `xDestroy()` method effectively gives it v2
   semantics.
 
+
 ### User-defined SQL Functions (a.k.a. UDFs)
 
 The [`sqlite3_create_function()`](https://sqlite.org/c3ref/create_function.html)
@@ -207,12 +281,13 @@ family of APIs make heavy use of function pointers to provide
 client-defined callbacks, necessitating interface changes in the JNI
 binding. The Java API has only one core function-registration function:
 
-```
+```java
 int sqlite3_create_function(sqlite3 db, String funcName, int nArgs,
                             int encoding, SQLFunction func);
 ```
 
-> Design question: does the encoding argument serve any purpose in JS?
+> Design question: does the encoding argument serve any purpose in
+  Java? That's as-yet undetermined. If not, it will be removed.
 
 `SQLFunction` is not used directly, but is instead instantiated via
 one of its three subclasses:
@@ -230,5 +305,9 @@ Search [`Tester1.java`](/file/ext/jni/src/org/sqlite/jni/Tester1.java) for
 Reminder: see the disclaimer at the top of this document regarding the
 in-flux nature of this API.
 
-[jsrc]: /file/
+### And so on...
-[www]: https://sqlite.org
+
+Various APIs which accept callbacks, e.g. `sqlite3_trace_v2()` and
+`sqlite3_update_hook()`, use interfaces similar to those shown above.
+Despite the changes in signature, the JNI layer makes every effort to
+provide the same semantics as the C API documentation suggests.

ext/jni/jar-dist.make

@@ -6,7 +6,9 @@
 # proper top-level JDK directory and, depending on the platform, add a
 # platform-specific -I directory. It should build as-is with any
 # 2020s-era version of gcc or clang. It requires JDK version 8 or
-# higher.
+# higher and that JAVA_HOME points to the top-most installation
+# directory of that JDK. On Ubuntu-style systems the JDK is typically
+# installed under /usr/lib/jvm/java-VERSION-PLATFORM.
 
 default: all
 
@@ -31,10 +33,11 @@ SQLITE_OPT = \
   -DSQLITE_OMIT_LOAD_EXTENSION \
   -DSQLITE_OMIT_DEPRECATED \
   -DSQLITE_OMIT_SHARED_CACHE \
-  -DSQLITE_THREADSAFE=0 \
+  -DSQLITE_THREADSAFE=1 \
   -DSQLITE_TEMP_STORE=2 \
   -DSQLITE_USE_URI=1 \
-  -DSQLITE_ENABLE_FTS5
+  -DSQLITE_ENABLE_FTS5 \
+  -DSQLITE_DEBUG
 
 sqlite3-jni.dll = libsqlite3-jni.so
 $(sqlite3-jni.dll):
@@ -46,8 +49,10 @@ $(sqlite3-jni.dll):
 		src/sqlite3-jni.c -shared -o $@
 	@echo "Now try running it with: make test"
 
+test.flags = -Djava.library.path=. sqlite3-jni-*.jar
 test: $(sqlite3-jni.dll)
-	java -jar -Djava.library.path=. sqlite3-jni-*.jar
+	java -jar $(test.flags)
+	java -jar $(test.flags) -t 7 -r 10 -shuffle
 
 clean:
 	-rm -f $(sqlite3-jni.dll)

ext/jni/src/org/sqlite/jni/Authorizer.java

@@ -1,32 +0,0 @@
-/*
-** 2023-08-05
-**
-** The author disclaims copyright to this source code.  In place of
-** a legal notice, here is a blessing:
-**
-**    May you do good and not evil.
-**    May you find forgiveness for yourself and forgive others.
-**    May you share freely, never taking more than you give.
-**
-*************************************************************************
-** This file is part of the JNI bindings for the sqlite3 C API.
-*/
-package org.sqlite.jni;
-
-/**
-   A callback for use with sqlite3_set_authorizer().
-*/
-public interface Authorizer {
-  /**
-     Must functions as described for the sqlite3_set_authorizer()
-     callback, with one caveat: the string values passed here were
-     initially (at the C level) encoded in standard UTF-8. If they
-     contained any constructs which are not compatible with MUTF-8,
-     these strings will not have the expected values. For further
-     details, see the documentation for the SQLite3Jni class.
-
-     Must not throw.
-  */
-  int xAuth(int opId, @Nullable String s1, @Nullable String s2,
-            @Nullable String s3, @Nullable String s4);
-}

ext/jni/src/org/sqlite/jni/AutoExtension.java

@@ -1,31 +0,0 @@
-/*
-** 2023-08-05
-**
-** The author disclaims copyright to this source code.  In place of
-** a legal notice, here is a blessing:
-**
-**    May you do good and not evil.
-**    May you find forgiveness for yourself and forgive others.
-**    May you share freely, never taking more than you give.
-**
-*************************************************************************
-** This file is part of the JNI bindings for the sqlite3 C API.
-*/
-package org.sqlite.jni;
-
-/**
-   A callback for use with sqlite3_auto_extension().
-*/
-public interface AutoExtension {
-  /**
-     Must function as described for the sqlite3_auto_extension(),
-     with the caveat that the signature is more limited.
-
-     As an exception (as it were) to the callbacks-must-not-throw
-     rule, AutoExtensions may do so and the exception's error message
-     will be set as the db's error string.
-
-     Results are undefined if db is closed by an auto-extension.
-  */
-  int xEntryPoint(sqlite3 db);
-}

ext/jni/src/org/sqlite/jni/BusyHandler.java

@@ -1,45 +0,0 @@
-/*
-** 2023-07-22
-**
-** The author disclaims copyright to this source code.  In place of
-** a legal notice, here is a blessing:
-**
-**    May you do good and not evil.
-**    May you find forgiveness for yourself and forgive others.
-**    May you share freely, never taking more than you give.
-**
-*************************************************************************
-** This file is part of the JNI bindings for the sqlite3 C API.
-*/
-package org.sqlite.jni;
-
-/**
-   Callback proxy for use with sqlite3_busy_handler().
-*/
-public abstract class BusyHandler {
-  /**
-     Must function as documented for the sqlite3_busy_handler()
-     callback argument, minus the (void*) argument the C-level
-     function requires.
-
-     Any exceptions thrown by this callback are suppressed in order to
-     retain the C-style API semantics of the JNI bindings.
-  */
-  public abstract int xCallback(int n);
-
-  /**
-     Optionally override to perform any cleanup when this busy
-     handler is destroyed. It is destroyed when:
-
-     - The associated db is passed to sqlite3_close() or
-       sqlite3_close_v2().
-
-     - sqlite3_busy_handler() is called to replace the handler,
-       whether it's passed a null handler or any other instance of
-       this class.
-
-     - sqlite3_busy_timeout() is called, which implicitly installs
-       a busy handler.
-  */
-  public void xDestroy(){}
-}

ext/jni/src/org/sqlite/jni/Fts5Function.java

@@ -1,27 +0,0 @@
-/*
-** 2023-08-04
-**
-** The author disclaims copyright to this source code.  In place of
-** a legal notice, here is a blessing:
-**
-**    May you do good and not evil.
-**    May you find forgiveness for yourself and forgive others.
-**    May you share freely, never taking more than you give.
-**
-*************************************************************************
-** This file is part of the JNI bindings for the sqlite3 C API.
-*/
-package org.sqlite.jni;
-
-/**
-   Fts5Function is used in conjunction with the
-   sqlite3_create_fts_function() JNI-bound API to give that native code
-   access to the callback functions needed in order to implement
-   FTS5 auxiliary functions in Java.
-*/
-public abstract class Fts5Function {
-
-  public abstract void xFunction(Fts5ExtensionApi pApi, Fts5Context pFts,
-                                 sqlite3_context pCtx, sqlite3_value argv[]);
-  public void xDestroy() {}
-}

ext/jni/src/org/sqlite/jni/OutputPointer.java

@@ -1,165 +0,0 @@
-/*
-** 2023-07-21
-**
-** The author disclaims copyright to this source code.  In place of
-** a legal notice, here is a blessing:
-**
-**    May you do good and not evil.
-**    May you find forgiveness for yourself and forgive others.
-**    May you share freely, never taking more than you give.
-**
-*************************************************************************
-** This file is part of the JNI bindings for the sqlite3 C API.
-*/
-package org.sqlite.jni;
-
-/**
-   Helper classes for handling JNI output pointers.
-
-   We do not use a generic OutputPointer<T> because working with those
-   from the native JNI code is unduly quirky due to a lack of
-   autoboxing at that level.
-
-   The usage is similar for all of thes types:
-
-   ```
-   OutputPointer.sqlite3 out = new OutputPointer.sqlite3();
-   assert( null==out.get() );
-   int rc = sqlite3_open(":memory:", out);
-   if( 0!=rc ) ... error;
-   assert( null!=out.get() );
-   sqlite3 db = out.take();
-   assert( null==out.get() );
-   ```
-
-   With the minor exception that the primitive types permit direct
-   access to the object's value via the `value` property, whereas the
-   JNI-level opaque types do not permit client-level code to set that
-   property.
-*/
-public final class OutputPointer {
-
-  /**
-     Output pointer for use with routines, such as sqlite3_open(),
-     which return a database handle via an output pointer. These
-     pointers can only be set by the JNI layer, not by client-level
-     code.
-  */
-  public static final class sqlite3 {
-    private org.sqlite.jni.sqlite3 value;
-    //! Initializes with a null value.
-    public sqlite3(){value = null;}
-    //! Sets the current value to null.
-    public void clear(){value = null;}
-    //! Returns the current value.
-    public final org.sqlite.jni.sqlite3 get(){return value;}
-    //! Equivalent to calling get() then clear().
-    public final org.sqlite.jni.sqlite3 take(){
-      final org.sqlite.jni.sqlite3 v = value;
-      value = null;
-      return v;
-    }
-  }
-
-  /**
-     Output pointer for use with routines, such as sqlite3_prepare(),
-     which return a statement handle via an output pointer. These
-     pointers can only be set by the JNI layer, not by client-level
-     code.
-  */
-  public static final class sqlite3_stmt {
-    private org.sqlite.jni.sqlite3_stmt value;
-    //! Initializes with a null value.
-    public sqlite3_stmt(){value = null;}
-    //! Sets the current value to null.
-    public void clear(){value = null;}
-    //! Returns the current value.
-    public final org.sqlite.jni.sqlite3_stmt get(){return value;}
-    //! Equivalent to calling get() then clear().
-    public final org.sqlite.jni.sqlite3_stmt take(){
-      final org.sqlite.jni.sqlite3_stmt v = value;
-      value = null;
-      return v;
-    }
-  }
-
-  /**
-     Output pointer for use with native routines which return integers via
-     output pointers.
-  */
-  public static final class Int32 {
-    /**
-       This is public for ease of use. Accessors are provided for
-       consistency with the higher-level types.
-    */
-    public int value;
-    //! Initializes with the value 0.
-    public Int32(){this(0);}
-    //! Initializes with the value v.
-    public Int32(int v){value = v;}
-    //! Returns the current value.
-    public final int get(){return value;}
-    //! Sets the current value to v.
-    public final void set(int v){value = v;}
-  }
-
-  /**
-     Output pointer for use with native routines which return 64-bit integers
-     via output pointers.
-  */
-  public static final class Int64 {
-    /**
-       This is public for ease of use. Accessors are provided for
-       consistency with the higher-level types.
-    */
-    public long value;
-    //! Initializes with the value 0.
-    public Int64(){this(0);}
-    //! Initializes with the value v.
-    public Int64(long v){value = v;}
-    //! Returns the current value.
-    public final long get(){return value;}
-    //! Sets the current value.
-    public final void set(long v){value = v;}
-  }
-
-  /**
-     Output pointer for use with native routines which return strings via
-     output pointers.
-  */
-  public static final class String {
-    /**
-       This is public for ease of use. Accessors are provided for
-       consistency with the higher-level types.
-    */
-    public java.lang.String value;
-    //! Initializes with a null value.
-    public String(){this(null);}
-    //! Initializes with the value v.
-    public String(java.lang.String v){value = v;}
-    //! Returns the current value.
-    public final java.lang.String get(){return value;}
-    //! Sets the current value.
-    public final void set(java.lang.String v){value = v;}
-  }
-
-  /**
-     Output pointer for use with native routines which return byte
-     arrays via output pointers.
-  */
-  public static final class ByteArray {
-    /**
-       This is public for ease of use. Accessors are provided for
-       consistency with the higher-level types.
-    */
-    public byte[] value;
-    //! Initializes with the value null.
-    public ByteArray(){this(null);}
-    //! Initializes with the value v.
-    public ByteArray(byte[] v){value = v;}
-    //! Returns the current value.
-    public final byte[] get(){return value;}
-    //! Sets the current value.
-    public final void set(byte[] v){value = v;}
-  }
-}

ext/jni/src/org/sqlite/jni/ResultCode.java

@@ -1,155 +0,0 @@
-/*
-** 2023-07-21
-**
-** The author disclaims copyright to this source code.  In place of
-** a legal notice, here is a blessing:
-**
-**    May you do good and not evil.
-**    May you find forgiveness for yourself and forgive others.
-**    May you share freely, never taking more than you give.
-**
-*************************************************************************
-** This file is part of the JNI bindings for the sqlite3 C API.
-*/
-package org.sqlite.jni;
-
-/**
-   This enum contains all of the core and "extended" result codes used
-   by the sqlite3 library. It is provided not for use with the C-style
-   API (with which it won't work) but for higher-level code which may
-   find it useful to map SQLite result codes to human-readable names.
-*/
-public enum ResultCode {
-  SQLITE_OK(SQLite3Jni.SQLITE_OK),
-  SQLITE_ERROR(SQLite3Jni.SQLITE_ERROR),
-  SQLITE_INTERNAL(SQLite3Jni.SQLITE_INTERNAL),
-  SQLITE_PERM(SQLite3Jni.SQLITE_PERM),
-  SQLITE_ABORT(SQLite3Jni.SQLITE_ABORT),
-  SQLITE_BUSY(SQLite3Jni.SQLITE_BUSY),
-  SQLITE_LOCKED(SQLite3Jni.SQLITE_LOCKED),
-  SQLITE_NOMEM(SQLite3Jni.SQLITE_NOMEM),
-  SQLITE_READONLY(SQLite3Jni.SQLITE_READONLY),
-  SQLITE_INTERRUPT(SQLite3Jni.SQLITE_INTERRUPT),
-  SQLITE_IOERR(SQLite3Jni.SQLITE_IOERR),
-  SQLITE_CORRUPT(SQLite3Jni.SQLITE_CORRUPT),
-  SQLITE_NOTFOUND(SQLite3Jni.SQLITE_NOTFOUND),
-  SQLITE_FULL(SQLite3Jni.SQLITE_FULL),
-  SQLITE_CANTOPEN(SQLite3Jni.SQLITE_CANTOPEN),
-  SQLITE_PROTOCOL(SQLite3Jni.SQLITE_PROTOCOL),
-  SQLITE_EMPTY(SQLite3Jni.SQLITE_EMPTY),
-  SQLITE_SCHEMA(SQLite3Jni.SQLITE_SCHEMA),
-  SQLITE_TOOBIG(SQLite3Jni.SQLITE_TOOBIG),
-  SQLITE_CONSTRAINT(SQLite3Jni.SQLITE_CONSTRAINT),
-  SQLITE_MISMATCH(SQLite3Jni.SQLITE_MISMATCH),
-  SQLITE_MISUSE(SQLite3Jni.SQLITE_MISUSE),
-  SQLITE_NOLFS(SQLite3Jni.SQLITE_NOLFS),
-  SQLITE_AUTH(SQLite3Jni.SQLITE_AUTH),
-  SQLITE_FORMAT(SQLite3Jni.SQLITE_FORMAT),
-  SQLITE_RANGE(SQLite3Jni.SQLITE_RANGE),
-  SQLITE_NOTADB(SQLite3Jni.SQLITE_NOTADB),
-  SQLITE_NOTICE(SQLite3Jni.SQLITE_NOTICE),
-  SQLITE_WARNING(SQLite3Jni.SQLITE_WARNING),
-  SQLITE_ROW(SQLite3Jni.SQLITE_ROW),
-  SQLITE_DONE(SQLite3Jni.SQLITE_DONE),
-  SQLITE_ERROR_MISSING_COLLSEQ(SQLite3Jni.SQLITE_ERROR_MISSING_COLLSEQ),
-  SQLITE_ERROR_RETRY(SQLite3Jni.SQLITE_ERROR_RETRY),
-  SQLITE_ERROR_SNAPSHOT(SQLite3Jni.SQLITE_ERROR_SNAPSHOT),
-  SQLITE_IOERR_READ(SQLite3Jni.SQLITE_IOERR_READ),
-  SQLITE_IOERR_SHORT_READ(SQLite3Jni.SQLITE_IOERR_SHORT_READ),
-  SQLITE_IOERR_WRITE(SQLite3Jni.SQLITE_IOERR_WRITE),
-  SQLITE_IOERR_FSYNC(SQLite3Jni.SQLITE_IOERR_FSYNC),
-  SQLITE_IOERR_DIR_FSYNC(SQLite3Jni.SQLITE_IOERR_DIR_FSYNC),
-  SQLITE_IOERR_TRUNCATE(SQLite3Jni.SQLITE_IOERR_TRUNCATE),
-  SQLITE_IOERR_FSTAT(SQLite3Jni.SQLITE_IOERR_FSTAT),
-  SQLITE_IOERR_UNLOCK(SQLite3Jni.SQLITE_IOERR_UNLOCK),
-  SQLITE_IOERR_RDLOCK(SQLite3Jni.SQLITE_IOERR_RDLOCK),
-  SQLITE_IOERR_DELETE(SQLite3Jni.SQLITE_IOERR_DELETE),
-  SQLITE_IOERR_BLOCKED(SQLite3Jni.SQLITE_IOERR_BLOCKED),
-  SQLITE_IOERR_NOMEM(SQLite3Jni.SQLITE_IOERR_NOMEM),
-  SQLITE_IOERR_ACCESS(SQLite3Jni.SQLITE_IOERR_ACCESS),
-  SQLITE_IOERR_CHECKRESERVEDLOCK(SQLite3Jni.SQLITE_IOERR_CHECKRESERVEDLOCK),
-  SQLITE_IOERR_LOCK(SQLite3Jni.SQLITE_IOERR_LOCK),
-  SQLITE_IOERR_CLOSE(SQLite3Jni.SQLITE_IOERR_CLOSE),
-  SQLITE_IOERR_DIR_CLOSE(SQLite3Jni.SQLITE_IOERR_DIR_CLOSE),
-  SQLITE_IOERR_SHMOPEN(SQLite3Jni.SQLITE_IOERR_SHMOPEN),
-  SQLITE_IOERR_SHMSIZE(SQLite3Jni.SQLITE_IOERR_SHMSIZE),
-  SQLITE_IOERR_SHMLOCK(SQLite3Jni.SQLITE_IOERR_SHMLOCK),
-  SQLITE_IOERR_SHMMAP(SQLite3Jni.SQLITE_IOERR_SHMMAP),
-  SQLITE_IOERR_SEEK(SQLite3Jni.SQLITE_IOERR_SEEK),
-  SQLITE_IOERR_DELETE_NOENT(SQLite3Jni.SQLITE_IOERR_DELETE_NOENT),
-  SQLITE_IOERR_MMAP(SQLite3Jni.SQLITE_IOERR_MMAP),
-  SQLITE_IOERR_GETTEMPPATH(SQLite3Jni.SQLITE_IOERR_GETTEMPPATH),
-  SQLITE_IOERR_CONVPATH(SQLite3Jni.SQLITE_IOERR_CONVPATH),
-  SQLITE_IOERR_VNODE(SQLite3Jni.SQLITE_IOERR_VNODE),
-  SQLITE_IOERR_AUTH(SQLite3Jni.SQLITE_IOERR_AUTH),
-  SQLITE_IOERR_BEGIN_ATOMIC(SQLite3Jni.SQLITE_IOERR_BEGIN_ATOMIC),
-  SQLITE_IOERR_COMMIT_ATOMIC(SQLite3Jni.SQLITE_IOERR_COMMIT_ATOMIC),
-  SQLITE_IOERR_ROLLBACK_ATOMIC(SQLite3Jni.SQLITE_IOERR_ROLLBACK_ATOMIC),
-  SQLITE_IOERR_DATA(SQLite3Jni.SQLITE_IOERR_DATA),
-  SQLITE_IOERR_CORRUPTFS(SQLite3Jni.SQLITE_IOERR_CORRUPTFS),
-  SQLITE_LOCKED_SHAREDCACHE(SQLite3Jni.SQLITE_LOCKED_SHAREDCACHE),
-  SQLITE_LOCKED_VTAB(SQLite3Jni.SQLITE_LOCKED_VTAB),
-  SQLITE_BUSY_RECOVERY(SQLite3Jni.SQLITE_BUSY_RECOVERY),
-  SQLITE_BUSY_SNAPSHOT(SQLite3Jni.SQLITE_BUSY_SNAPSHOT),
-  SQLITE_BUSY_TIMEOUT(SQLite3Jni.SQLITE_BUSY_TIMEOUT),
-  SQLITE_CANTOPEN_NOTEMPDIR(SQLite3Jni.SQLITE_CANTOPEN_NOTEMPDIR),
-  SQLITE_CANTOPEN_ISDIR(SQLite3Jni.SQLITE_CANTOPEN_ISDIR),
-  SQLITE_CANTOPEN_FULLPATH(SQLite3Jni.SQLITE_CANTOPEN_FULLPATH),
-  SQLITE_CANTOPEN_CONVPATH(SQLite3Jni.SQLITE_CANTOPEN_CONVPATH),
-  SQLITE_CANTOPEN_SYMLINK(SQLite3Jni.SQLITE_CANTOPEN_SYMLINK),
-  SQLITE_CORRUPT_VTAB(SQLite3Jni.SQLITE_CORRUPT_VTAB),
-  SQLITE_CORRUPT_SEQUENCE(SQLite3Jni.SQLITE_CORRUPT_SEQUENCE),
-  SQLITE_CORRUPT_INDEX(SQLite3Jni.SQLITE_CORRUPT_INDEX),
-  SQLITE_READONLY_RECOVERY(SQLite3Jni.SQLITE_READONLY_RECOVERY),
-  SQLITE_READONLY_CANTLOCK(SQLite3Jni.SQLITE_READONLY_CANTLOCK),
-  SQLITE_READONLY_ROLLBACK(SQLite3Jni.SQLITE_READONLY_ROLLBACK),
-  SQLITE_READONLY_DBMOVED(SQLite3Jni.SQLITE_READONLY_DBMOVED),
-  SQLITE_READONLY_CANTINIT(SQLite3Jni.SQLITE_READONLY_CANTINIT),
-  SQLITE_READONLY_DIRECTORY(SQLite3Jni.SQLITE_READONLY_DIRECTORY),
-  SQLITE_ABORT_ROLLBACK(SQLite3Jni.SQLITE_ABORT_ROLLBACK),
-  SQLITE_CONSTRAINT_CHECK(SQLite3Jni.SQLITE_CONSTRAINT_CHECK),
-  SQLITE_CONSTRAINT_COMMITHOOK(SQLite3Jni.SQLITE_CONSTRAINT_COMMITHOOK),
-  SQLITE_CONSTRAINT_FOREIGNKEY(SQLite3Jni.SQLITE_CONSTRAINT_FOREIGNKEY),
-  SQLITE_CONSTRAINT_FUNCTION(SQLite3Jni.SQLITE_CONSTRAINT_FUNCTION),
-  SQLITE_CONSTRAINT_NOTNULL(SQLite3Jni.SQLITE_CONSTRAINT_NOTNULL),
-  SQLITE_CONSTRAINT_PRIMARYKEY(SQLite3Jni.SQLITE_CONSTRAINT_PRIMARYKEY),
-  SQLITE_CONSTRAINT_TRIGGER(SQLite3Jni.SQLITE_CONSTRAINT_TRIGGER),
-  SQLITE_CONSTRAINT_UNIQUE(SQLite3Jni.SQLITE_CONSTRAINT_UNIQUE),
-  SQLITE_CONSTRAINT_VTAB(SQLite3Jni.SQLITE_CONSTRAINT_VTAB),
-  SQLITE_CONSTRAINT_ROWID(SQLite3Jni.SQLITE_CONSTRAINT_ROWID),
-  SQLITE_CONSTRAINT_PINNED(SQLite3Jni.SQLITE_CONSTRAINT_PINNED),
-  SQLITE_CONSTRAINT_DATATYPE(SQLite3Jni.SQLITE_CONSTRAINT_DATATYPE),
-  SQLITE_NOTICE_RECOVER_WAL(SQLite3Jni.SQLITE_NOTICE_RECOVER_WAL),
-  SQLITE_NOTICE_RECOVER_ROLLBACK(SQLite3Jni.SQLITE_NOTICE_RECOVER_ROLLBACK),
-  SQLITE_WARNING_AUTOINDEX(SQLite3Jni.SQLITE_WARNING_AUTOINDEX),
-  SQLITE_AUTH_USER(SQLite3Jni.SQLITE_AUTH_USER),
-  SQLITE_OK_LOAD_PERMANENTLY(SQLite3Jni.SQLITE_OK_LOAD_PERMANENTLY);
-
-  public final int value;
-
-  ResultCode(int rc){
-    value = rc;
-    ResultCodeMap.set(rc, this);
-  }
-
-  /**
-     Returns the entry from this enum for the given result code, or
-     null if no match is found.
-  */
-  public static ResultCode getEntryForInt(int rc){
-    return ResultCodeMap.get(rc);
-  }
-
-  /**
-     Internal level of indirection required because we cannot initialize
-     static enum members in an enum before the enum constructor is
-     invoked.
-  */
-  private static final class ResultCodeMap {
-    private static final java.util.Map<Integer,ResultCode> i2e
-      = new java.util.HashMap<>();
-    private static void set(int rc, ResultCode e){ i2e.put(rc, e); }
-    private static ResultCode get(int rc){ return i2e.get(rc); }
-  }
-
-}

ext/jni/src/org/sqlite/jni/SQLFunction.java

@@ -1,172 +0,0 @@
-/*
-** 2023-07-22
-**
-** The author disclaims copyright to this source code.  In place of
-** a legal notice, here is a blessing:
-**
-**    May you do good and not evil.
-**    May you find forgiveness for yourself and forgive others.
-**    May you share freely, never taking more than you give.
-**
-*************************************************************************
-** This file is part of the JNI bindings for the sqlite3 C API.
-*/
-package org.sqlite.jni;
-
-/**
-   SQLFunction is used in conjunction with the
-   sqlite3_create_function() JNI-bound API to give that native code
-   access to the callback functions needed in order to implement SQL
-   functions in Java.
-
-   This class is not used by itself, but is a marker base class. The
-   three UDF types are modelled by the inner classes Scalar,
-   Aggregate<T>, and Window<T>. Most simply, clients may create
-   anonymous classes from those to implement UDFs. Clients are free to
-   create their own classes for use with UDFs, so long as they conform
-   to the public interfaces defined by those three classes. The JNI
-   layer only actively relies on the SQLFunction base class.
-*/
-public abstract class SQLFunction {
-
-  /**
-     PerContextState assists aggregate and window functions in
-     managinga their accumulator state across calls to the UDF's
-     callbacks.
-
-     If a given aggregate or window function is called multiple times
-     in a single SQL statement, e.g. SELECT MYFUNC(A), MYFUNC(B)...,
-     then the clients need some way of knowing which call is which so
-     that they can map their state between their various UDF callbacks
-     and reset it via xFinal(). This class takes care of such
-     mappings.
-
-     This class works by mapping
-     sqlite3_context.getAggregateContext() to a single piece of
-     state, of a client-defined type (the T part of this class), which
-     persists across a "matching set" of the UDF's callbacks.
-
-     This class is a helper providing commonly-needed functionality -
-     it is not required for use with aggregate or window functions.
-     Client UDFs are free to perform such mappings using custom
-     approaches. The provided Aggregate<T> and Window<T> classes
-     use this.
-  */
-  public static final class PerContextState<T> {
-    private final java.util.Map<Long,ValueHolder<T>> map
-      = new java.util.HashMap<>();
-
-    /**
-       Should be called from a UDF's xStep(), xValue(), and xInverse()
-       methods, passing it that method's first argument and an initial
-       value for the persistent state. If there is currently no
-       mapping for cx.getAggregateContext() within the map, one is
-       created using the given initial value, else the existing one is
-       used and the 2nd argument is ignored.  It returns a
-       ValueHolder<T> which can be used to modify that state directly
-       without requiring that the client update the underlying map's
-       entry.
-
-       T must be of a type which can be legally stored as a value in
-       java.util.HashMap<KeyType,T>.
-    */
-    public ValueHolder<T> getAggregateState(sqlite3_context cx, T initialValue){
-      ValueHolder<T> rc = map.get(cx.getAggregateContext());
-      if(null == rc){
-        map.put(cx.getAggregateContext(), rc = new ValueHolder<>(initialValue));
-      }
-      return rc;
-    }
-
-    /**
-       Should be called from a UDF's xFinal() method and passed that
-       method's first argument. This function removes the value
-       associated with cx.getAggregateContext() from the map and
-       returns it, returning null if no other UDF method has been
-       called to set up such a mapping. The latter condition will be
-       the case if a UDF is used in a statement which has no result
-       rows.
-    */
-    public T takeAggregateState(sqlite3_context cx){
-      final ValueHolder<T> h = map.remove(cx.getAggregateContext());
-      return null==h ? null : h.value;
-    }
-  }
-
-  //! Subclass for creating scalar functions.
-  public static abstract class Scalar extends SQLFunction {
-
-    //! As for the xFunc() argument of the C API's sqlite3_create_function()
-    public abstract void xFunc(sqlite3_context cx, sqlite3_value[] args);
-
-    /**
-       Optionally override to be notified when the UDF is finalized by
-       SQLite.
-    */
-    public void xDestroy() {}
-  }
-
-  /**
-     SQLFunction Subclass for creating aggregate functions.  Its T is
-     the data type of its "accumulator" state, an instance of which is
-     intended to be be managed using the getAggregateState() and
-     takeAggregateState() methods.
-  */
-  public static abstract class Aggregate<T> extends SQLFunction {
-
-    //! As for the xStep() argument of the C API's sqlite3_create_function()
-    public abstract void xStep(sqlite3_context cx, sqlite3_value[] args);
-
-    //! As for the xFinal() argument of the C API's sqlite3_create_function()
-    public abstract void xFinal(sqlite3_context cx);
-
-    /**
-       Optionally override to be notified when the UDF is finalized by
-       SQLite.
-    */
-    public void xDestroy() {}
-
-    //! Per-invocation state for the UDF.
-    private final PerContextState<T> map = new PerContextState<>();
-
-    /**
-       To be called from the implementation's xStep() method, as well
-       as the xValue() and xInverse() methods of the Window<T>
-       subclass, to fetch the current per-call UDF state. On the
-       first call to this method for any given sqlite3_context
-       argument, the context is set to the given initial value. On all other
-       calls, the 2nd argument is ignored.
-
-       @see PerContextState<T>#takeAggregateState()
-    */
-    protected final ValueHolder<T> getAggregateState(sqlite3_context cx, T initialValue){
-      return map.getAggregateState(cx, initialValue);
-    }
-
-    /**
-       To be called from the implementation's xFinal() method to fetch
-       the final state of the UDF and remove its mapping.
-
-       @see PerContextState<T>#takeAggregateState()
-    */
-    protected final T takeAggregateState(sqlite3_context cx){
-      return map.takeAggregateState(cx);
-    }
-  }
-
-  /**
-     An SQLFunction subclass for creating window functions.  Note that
-     Window<T> inherits from Aggregate<T> and each instance is
-     required to implement the inherited abstract methods from that
-     class. See Aggregate<T> for information on managing the UDF's
-     invocation-specific state.
-  */
-  public static abstract class Window<T> extends Aggregate<T> {
-
-    //! As for the xInverse() argument of the C API's sqlite3_create_window_function()
-    public abstract void xInverse(sqlite3_context cx, sqlite3_value[] args);
-
-    //! As for the xValue() argument of the C API's sqlite3_create_window_function()
-    public abstract void xValue(sqlite3_context cx);
-  }
-}

ext/jni/src/org/sqlite/jni/TesterFts5.java

@@ -1,87 +0,0 @@
-/*
-** 2023-08-04
-**
-** The author disclaims copyright to this source code.  In place of
-** a legal notice, here is a blessing:
-**
-**    May you do good and not evil.
-**    May you find forgiveness for yourself and forgive others.
-**    May you share freely, never taking more than you give.
-**
-*************************************************************************
-** This file contains a set of tests for the sqlite3 JNI bindings.
-*/
-package org.sqlite.jni;
-import static org.sqlite.jni.SQLite3Jni.*;
-import static org.sqlite.jni.Tester1.*;
-
-public class TesterFts5 {
-
-  private static void test1(){
-    Fts5ExtensionApi fea = Fts5ExtensionApi.getInstance();
-    affirm( null != fea );
-    affirm( fea.getNativePointer() != 0 );
-    affirm( fea == Fts5ExtensionApi.getInstance() )/*singleton*/;
-
-    sqlite3 db = createNewDb();
-    fts5_api fApi = fts5_api.getInstanceForDb(db);
-    affirm( fApi != null );
-    affirm( fApi == fts5_api.getInstanceForDb(db) /* singleton per db */ );
-
-    execSql(db, new String[] {
-        "CREATE VIRTUAL TABLE ft USING fts5(a, b);",
-        "INSERT INTO ft(rowid, a, b) VALUES(1, 'X Y', 'Y Z');",
-        "INSERT INTO ft(rowid, a, b) VALUES(2, 'A Z', 'Y Y');"
-      });
-
-    final String pUserData = "This is pUserData";
-    ValueHolder<Boolean> xDestroyCalled = new ValueHolder<>(false);
-    ValueHolder<Integer> xFuncCount = new ValueHolder<>(0);
-    final fts5_extension_function func = new fts5_extension_function(){
-        public void xFunction(Fts5ExtensionApi ext, Fts5Context fCx,
-                              sqlite3_context pCx, sqlite3_value argv[]){
-          int nCols = ext.xColumnCount(fCx);
-          affirm( 2 == nCols );
-          affirm( nCols == argv.length );
-          affirm( ext.xUserData(fCx) == pUserData );
-          if(true){
-            OutputPointer.String op = new OutputPointer.String();
-            for(int i = 0; i < nCols; ++i ){
-              int rc = ext.xColumnText(fCx, i, op);
-              affirm( 0 == rc );
-              final String val = op.value;
-              affirm( val.equals(sqlite3_value_text(argv[i])) );
-              //outln("xFunction col "+i+": "+val);
-            }
-          }
-          ++xFuncCount.value;
-        }
-        public void xDestroy(){
-          xDestroyCalled.value = true;
-        }
-      };
-
-    int rc = fApi.xCreateFunction("myaux", pUserData, func);
-    affirm( 0==rc );
-
-    affirm( 0==xFuncCount.value );
-    execSql(db, "select myaux(ft,a,b) from ft;");
-    affirm( 2==xFuncCount.value );
-    affirm( !xDestroyCalled.value );
-    sqlite3_close_v2(db);
-    affirm( xDestroyCalled.value );
-  }
-
-  public TesterFts5(){
-    int oldAffirmCount = Tester1.affirmCount;
-    Tester1.affirmCount = 0;
-    final long timeStart = System.nanoTime();
-    test1();
-    final long timeEnd = System.nanoTime();
-    outln("FTS5 Tests done. Metrics:");
-    outln("\tAssertions checked: "+Tester1.affirmCount);
-    outln("\tTotal time = "
-          +((timeEnd - timeStart)/1000000.0)+"ms");
-    Tester1.affirmCount = oldAffirmCount;
-  }
-}

ext/jni/src/org/sqlite/jni/Tracer.java

@@ -1,62 +0,0 @@
-/*
-** 2023-07-22
-**
-** The author disclaims copyright to this source code.  In place of
-** a legal notice, here is a blessing:
-**
-**    May you do good and not evil.
-**    May you find forgiveness for yourself and forgive others.
-**    May you share freely, never taking more than you give.
-**
-*************************************************************************
-** This file is part of the JNI bindings for the sqlite3 C API.
-*/
-package org.sqlite.jni;
-
-/**
-   Callback proxy for use with sqlite3_trace_v2().
-*/
-public interface Tracer {
-  /**
-     Achtung: this interface is subject to change because the current
-     approach to mapping the passed-in natives back to Java is
-     uncomfortably quirky.
-
-     Called by sqlite3 for various tracing operations, as per
-     sqlite3_trace_v2(). Note that this interface elides the 2nd
-     argument to the native trace callback, as that role is better
-     filled by instance-local state.
-
-     The 2nd argument to this function, if non-0, will be a native
-     pointer to either an sqlite3 or sqlite3_stmt object, depending on
-     the first argument (see below). Client code can pass it to the
-     sqlite3 resp. sqlite3_stmt constructor to create a wrapping
-     object, if necessary. This API does not do so by default because
-     tracing can be called frequently, creating such a wrapper for
-     each call is comparatively expensive, and the objects are
-     probably only seldom useful.
-
-     The final argument to this function is the "X" argument
-     documented for sqlite3_trace() and sqlite3_trace_v2(). Its type
-     depends on value of the first argument:
-
-     - SQLITE_TRACE_STMT: pNative is a sqlite3_stmt. pX is a string
-       containing the prepared SQL, with one caveat: JNI only provides
-       us with the ability to convert that string to MUTF-8, as
-       opposed to standard UTF-8, and is cannot be ruled out that that
-       difference may be significant for certain inputs. The
-       alternative would be that we first convert it to UTF-16 before
-       passing it on, but there's no readily-available way to do that
-       without calling back into the db to peform the conversion
-       (which would lead to further tracing).
-
-     - SQLITE_TRACE_PROFILE: pNative is a sqlite3_stmt. pX is a Long
-       holding an approximate number of nanoseconds the statement took
-       to run.
-
-     - SQLITE_TRACE_ROW: pNative is a sqlite3_stmt. pX is null.
-
-     - SQLITE_TRACE_CLOSE: pNative is a sqlite3. pX is null.
-  */
-  int xCallback(int traceFlag, Object pNative, Object pX);
-}

ext/jni/src/org/sqlite/jni/annotation/NotNull.java

@@ -0,0 +1,59 @@
+/*
+** 2023-09-27
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file houses the NotNull annotaion for the sqlite3 C API.
+*/
+package org.sqlite.jni.annotation;
+
+/**
+   This annotation is for flagging parameters which may not legally be
+   null or point to closed/finalized C-side resources.
+
+   <p>In the case of Java types which map directly to C struct types
+   (e.g. {@link org.sqlite.jni.sqlite3}, {@link
+   org.sqlite.jni.sqlite3_stmt}, and {@link
+   org.sqlite.jni.sqlite3_context}), a closed/finalized resource is
+   also considered to be null for purposes this annotation because the
+   C-side effect of passing such a handle is the same as if null is
+   passed.</p>
+
+   <p>When used in the context of Java interfaces which are called
+   from the C APIs, this annotation communicates that the C API will
+   never pass a null value to the callback for that parameter.</p>
+
+   <p>Passing a null, for this annotation's definition of null, for
+   any parameter marked with this annoation specifically invokes
+   undefined behavior.</p>
+
+   <p>Passing 0 (i.e. C NULL) or a negative value for any long-type
+   parameter marked with this annoation specifically invokes undefined
+   behavior. Such values are treated as C pointers in the JNI
+   layer.</p>
+
+   <p>Note that the C-style API does not throw any exceptions on its
+   own because it has a no-throw policy in order to retain its C-style
+   semantics, but it may trigger NullPointerExceptions (or similar) if
+   passed a null for a parameter flagged with this annotation.</p>
+
+   <p>This annotation is informational only. No policy is in place to
+   programmatically ensure that NotNull is conformed to in client
+   code.</p>
+
+   <p>This annotation is solely for the use by the classes in the
+   org.sqlite package and subpackages, but is made public so that
+   javadoc will link to it from the annotated functions. It is not
+   part of the public API and client-level code must not rely on
+   it.</p>
+*/
+@java.lang.annotation.Documented
+@java.lang.annotation.Retention(java.lang.annotation.RetentionPolicy.SOURCE)
+@java.lang.annotation.Target(java.lang.annotation.ElementType.PARAMETER)
+public @interface NotNull{}

ext/jni/src/org/sqlite/jni/annotation/Nullable.java

@@ -0,0 +1,32 @@
+/*
+** 2023-09-27
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file houses the Nullable annotaion for the sqlite3 C API.
+*/
+package org.sqlite.jni.annotation;
+
+/**
+   This annotation is for flagging parameters which may legally be
+   null, noting that they may behave differently if passed null but
+   are prepared to expect null as a value. When used in the context of
+   callback methods which are called into from the C APIs, this
+   annotation communicates that the C API may pass a null value to the
+   callback.
+
+   <p>This annotation is solely for the use by the classes in this
+   package but is made public so that javadoc will link to it from the
+   annotated functions. It is not part of the public API and
+   client-level code must not rely on it.
+*/
+@java.lang.annotation.Documented
+@java.lang.annotation.Retention(java.lang.annotation.RetentionPolicy.SOURCE)
+@java.lang.annotation.Target(java.lang.annotation.ElementType.PARAMETER)
+public @interface Nullable{}

ext/jni/src/org/sqlite/jni/annotation/package-info.java

@@ -0,0 +1,17 @@
+/*
+** 2023-09-27
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+*/
+/**
+   This package houses annotations specific to the JNI bindings of the
+   SQLite3 C API.
+*/
+package org.sqlite.jni.annotation;

ext/jni/src/org/sqlite/jni/capi/AbstractCollationCallback.java

@@ -0,0 +1,34 @@
+/*
+** 2023-08-25
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file is part of the JNI bindings for the sqlite3 C API.
+*/
+package org.sqlite.jni.capi;
+import org.sqlite.jni.annotation.NotNull;
+
+/**
+   An implementation of {@link CollationCallback} which provides a
+   no-op xDestroy() method.
+*/
+public abstract class AbstractCollationCallback
+  implements CollationCallback, XDestroyCallback {
+  /**
+     Must compare the given byte arrays and return the result using
+     {@code memcmp()} semantics.
+  */
+  public abstract int call(@NotNull byte[] lhs, @NotNull byte[] rhs);
+
+  /**
+     Optionally override to be notified when the UDF is finalized by
+     SQLite. This implementation does nothing.
+  */
+  public void xDestroy(){}
+}

ext/jni/src/org/sqlite/jni/capi/AggregateFunction.java

@@ -0,0 +1,72 @@
+/*
+** 2023-08-25
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file is part of the JNI bindings for the sqlite3 C API.
+*/
+package org.sqlite.jni.capi;
+
+
+/**
+   A SQLFunction implementation for aggregate functions.  Its T is the
+   data type of its "accumulator" state, an instance of which is
+   intended to be be managed using the getAggregateState() and
+   takeAggregateState() methods.
+*/
+public abstract class AggregateFunction<T> implements SQLFunction {
+
+  /**
+     As for the xStep() argument of the C API's
+     sqlite3_create_function().  If this function throws, the
+     exception is not propagated and a warning might be emitted to a
+     debugging channel.
+  */
+  public abstract void xStep(sqlite3_context cx, sqlite3_value[] args);
+
+  /**
+     As for the xFinal() argument of the C API's sqlite3_create_function().
+     If this function throws, it is translated into an sqlite3_result_error().
+  */
+  public abstract void xFinal(sqlite3_context cx);
+
+  /**
+     Optionally override to be notified when the UDF is finalized by
+     SQLite.
+  */
+  public void xDestroy() {}
+
+  /** Per-invocation state for the UDF. */
+  private final SQLFunction.PerContextState<T> map =
+    new SQLFunction.PerContextState<>();
+
+  /**
+     To be called from the implementation's xStep() method, as well
+     as the xValue() and xInverse() methods of the {@link WindowFunction}
+     subclass, to fetch the current per-call UDF state. On the
+     first call to this method for any given sqlite3_context
+     argument, the context is set to the given initial value. On all other
+     calls, the 2nd argument is ignored.
+
+     @see SQLFunction.PerContextState#getAggregateState
+  */
+  protected final ValueHolder<T> getAggregateState(sqlite3_context cx, T initialValue){
+    return map.getAggregateState(cx, initialValue);
+  }
+
+  /**
+     To be called from the implementation's xFinal() method to fetch
+     the final state of the UDF and remove its mapping.
+
+     see SQLFunction.PerContextState#takeAggregateState
+  */
+  protected final T takeAggregateState(sqlite3_context cx){
+    return map.takeAggregateState(cx);
+  }
+}

ext/jni/src/org/sqlite/jni/capi/AuthorizerCallback.java

@@ -0,0 +1,28 @@
+/*
+** 2023-08-25
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file is part of the JNI bindings for the sqlite3 C API.
+*/
+package org.sqlite.jni.capi;
+import org.sqlite.jni.annotation.*;
+
+/**
+   Callback for use with {@link CApi#sqlite3_set_authorizer}.
+*/
+public interface AuthorizerCallback extends CallbackProxy {
+  /**
+     Must function as described for the C-level
+     sqlite3_set_authorizer() callback.
+  */
+  int call(int opId, @Nullable String s1, @Nullable String s2,
+           @Nullable String s3, @Nullable String s4);
+
+}

ext/jni/src/org/sqlite/jni/capi/AutoExtensionCallback.java

@@ -0,0 +1,40 @@
+/*
+** 2023-08-25
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file is part of the JNI bindings for the sqlite3 C API.
+*/
+package org.sqlite.jni.capi;
+
+/**
+   Callback for use with the {@link CApi#sqlite3_auto_extension}
+   family of APIs.
+*/
+public interface AutoExtensionCallback extends CallbackProxy {
+  /**
+     Must function as described for a C-level
+     sqlite3_auto_extension() callback.
+
+     <p>This callback may throw and the exception's error message will
+     be set as the db's error string.
+
+     <p>Tips for implementations:
+
+     <p>- Opening a database from an auto-extension handler will lead to
+     an endless recursion of the auto-handler triggering itself
+     indirectly for each newly-opened database.
+
+     <p>- If this routine is stateful, it may be useful to make the
+     overridden method synchronized.
+
+     <p>- Results are undefined if the given db is closed by an auto-extension.
+  */
+  int call(sqlite3 db);
+}

ext/jni/src/org/sqlite/jni/capi/BusyHandlerCallback.java

@@ -0,0 +1,26 @@
+/*
+** 2023-08-25
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file is part of the JNI bindings for the sqlite3 C API.
+*/
+package org.sqlite.jni.capi;
+
+/**
+   Callback for use with {@link CApi#sqlite3_busy_handler}.
+*/
+public interface BusyHandlerCallback extends CallbackProxy {
+  /**
+     Must function as documented for the C-level
+     sqlite3_busy_handler() callback argument, minus the (void*)
+     argument the C-level function requires.
+  */
+  int call(int n);
+}

ext/jni/src/org/sqlite/jni/capi/CallbackProxy.java

@@ -0,0 +1,44 @@
+/*
+** 2023-08-25
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file is part of the JNI bindings for the sqlite3 C API.
+*/
+package org.sqlite.jni.capi;
+/**
+   This marker interface exists soley for use as a documentation and
+   class-grouping tool. It should be applied to interfaces or
+   classes which have a call() method implementing some specific
+   callback interface on behalf of the C library.
+
+   <p>Unless very explicitely documented otherwise, callbacks must
+   never throw. Any which do throw but should not might trigger debug
+   output regarding the error, but the exception will not be
+   propagated.  For callback interfaces which support returning error
+   info to the core, the JNI binding will convert any exceptions to
+   C-level error information. For callback interfaces which do not
+   support, all exceptions will necessarily be suppressed in order to
+   retain the C-style no-throw semantics.
+
+   <p>Callbacks of this style follow a common naming convention:
+
+   <p>1) They use the UpperCamelCase form of the C function they're
+   proxying for, minus the {@code sqlite3_} prefix, plus a {@code
+   Callback} suffix. e.g. {@code sqlite3_busy_handler()}'s callback is
+   named {@code BusyHandlerCallback}. Exceptions are made where that
+   would potentially be ambiguous, e.g. {@link ConfigSqllogCallback}
+   instead of {@code ConfigCallback} because the {@code
+   sqlite3_config()} interface may need to support more callback types
+   in the future.
+
+   <p>2) They all have a {@code call()} method but its signature is
+   callback-specific.
+*/
+public interface CallbackProxy {}

ext/jni/src/org/sqlite/jni/capi/CollationCallback.java

@@ -0,0 +1,35 @@
+/*
+** 2023-08-25
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file is part of the JNI bindings for the sqlite3 C API.
+*/
+package org.sqlite.jni.capi;
+import org.sqlite.jni.annotation.NotNull;
+
+/**
+   Callback for use with {@link CApi#sqlite3_create_collation}.
+
+   @see AbstractCollationCallback
+*/
+public interface CollationCallback
+  extends CallbackProxy, XDestroyCallback {
+  /**
+     Must compare the given byte arrays and return the result using
+     {@code memcmp()} semantics.
+  */
+  int call(@NotNull byte[] lhs, @NotNull byte[] rhs);
+
+  /**
+     Called by SQLite when the collation is destroyed. If a collation
+     requires custom cleanup, override this method.
+  */
+  void xDestroy();
+}

ext/jni/src/org/sqlite/jni/capi/CollationNeededCallback.java

@@ -1,5 +1,5 @@
 /*
-** 2023-07-30
+** 2023-08-25
 **
 ** The author disclaims copyright to this source code.  In place of
 ** a legal notice, here is a blessing:
@@ -11,18 +11,18 @@
 *************************************************************************
 ** This file is part of the JNI bindings for the sqlite3 C API.
 */
-package org.sqlite.jni;
+package org.sqlite.jni.capi;
 
 /**
-   Callback proxy for use with sqlite3_collation_needed().
+   Callback for use with {@link CApi#sqlite3_collation_needed}.
 */
-public interface CollationNeeded {
+public interface CollationNeededCallback extends CallbackProxy {
   /**
      Has the same semantics as the C-level sqlite3_create_collation()
      callback.
 
-     If it throws, the exception message is passed on to the db and
+     <p>If it throws, the exception message is passed on to the db and
      the exception is suppressed.
   */
-  int xCollationNeeded(sqlite3 db, int eTextRep, String collationName);
+  int call(sqlite3 db, int eTextRep, String collationName);
 }

ext/jni/src/org/sqlite/jni/capi/CommitHookCallback.java

@@ -1,5 +1,5 @@
 /*
-** 2023-07-22
+** 2023-08-25
 **
 ** The author disclaims copyright to this source code.  In place of
 ** a legal notice, here is a blessing:
@@ -11,15 +11,15 @@
 *************************************************************************
 ** This file is part of the JNI bindings for the sqlite3 C API.
 */
-package org.sqlite.jni;
+package org.sqlite.jni.capi;
 
 /**
-   Callback proxy for use with sqlite3_update_hook().
+   Callback for use with {@link CApi#sqlite3_commit_hook}.
 */
-public interface UpdateHook {
+public interface CommitHookCallback extends CallbackProxy {
   /**
-     Works as documented for the sqlite3_update_hook() callback.
+     Works as documented for the C-level sqlite3_commit_hook()
-     Must not throw.
+     callback.  Must not throw.
   */
-  void xUpdateHook(int opId, String dbName, String tableName, long rowId);
+  int call();
 }

ext/jni/src/org/sqlite/jni/capi/ConfigLogCallback.java

@@ -0,0 +1,25 @@
+/*
+** 2023-08-23
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file is part of the JNI bindings for the sqlite3 C API.
+*/
+package org.sqlite.jni.capi;
+
+/**
+   A callback for use with sqlite3_config().
+*/
+public interface ConfigLogCallback {
+  /**
+     Must function as described for a C-level callback for
+     {@link CApi#sqlite3_config(ConfigLogCallback)}, with the slight signature change.
+  */
+  void call(int errCode, String msg);
+}

ext/jni/src/org/sqlite/jni/capi/ConfigSqllogCallback.java

@@ -0,0 +1,25 @@
+/*
+** 2023-08-23
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file is part of the JNI bindings for the sqlite3 C API.
+*/
+package org.sqlite.jni.capi;
+
+/**
+   A callback for use with sqlite3_config().
+*/
+public interface ConfigSqllogCallback {
+  /**
+     Must function as described for a C-level callback for
+     {@link CApi#sqlite3_config(ConfigSqllogCallback)}, with the slight signature change.
+  */
+  void call(sqlite3 db, String msg, int msgType );
+}

ext/jni/src/org/sqlite/jni/capi/NativePointerHolder.java

@@ -11,7 +11,7 @@
 *************************************************************************
 ** This file is part of the JNI bindings for the sqlite3 C API.
 */
-package org.sqlite.jni;
+package org.sqlite.jni.capi;
 
 /**
    A helper for passing pointers between JNI C code and Java, in
@@ -23,11 +23,24 @@ package org.sqlite.jni;
    NativePointerHolder is not inadvertently passed to an incompatible
    function signature.
 
-   These objects do not _own_ the pointer they refer to.  They are
+   These objects do not own the pointer they refer to.  They are
    intended simply to communicate that pointer between C and Java.
 */
 public class NativePointerHolder<ContextType> {
   //! Only set from JNI, where access permissions don't matter.
-  private long nativePointer = 0;
+  private volatile long nativePointer = 0;
+  /**
+     For use ONLY by package-level APIs which act as proxies for
+     close/finalize operations. Such ops must call this to zero out
+     the pointer so that this object is not carrying a stale
+     pointer. This function returns the prior value of the pointer and
+     sets it to 0.
+  */
+  final long clearNativePointer() {
+    final long rv = nativePointer;
+    nativePointer= 0;
+    return rv;
+  }
+
   public final long getNativePointer(){ return nativePointer; }
 }

ext/jni/src/org/sqlite/jni/capi/OutputPointer.java

@@ -0,0 +1,231 @@
+/*
+** 2023-07-21
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file is part of the JNI bindings for the sqlite3 C API.
+*/
+package org.sqlite.jni.capi;
+
+/**
+   Helper classes for handling JNI output pointers.
+
+   <p>We do not use a generic OutputPointer<T> because working with those
+   from the native JNI code is unduly quirky due to a lack of
+   autoboxing at that level.
+
+   <p>The usage is similar for all of thes types:
+
+   <pre>{@code
+   OutputPointer.sqlite3 out = new OutputPointer.sqlite3();
+   assert( null==out.get() );
+   int rc = sqlite3_open(":memory:", out);
+   if( 0!=rc ) ... error;
+   assert( null!=out.get() );
+   sqlite3 db = out.take();
+   assert( null==out.get() );
+   }</pre>
+
+   <p>With the minor exception that the primitive types permit direct
+   access to the object's value via the `value` property, whereas the
+   JNI-level opaque types do not permit client-level code to set that
+   property.
+
+   <p>Warning: do not share instances of these classes across
+   threads. Doing so may lead to corrupting sqlite3-internal state.
+*/
+public final class OutputPointer {
+
+  /**
+     Output pointer for use with routines, such as sqlite3_open(),
+     which return a database handle via an output pointer. These
+     pointers can only be set by the JNI layer, not by client-level
+     code.
+  */
+  public static final class sqlite3 {
+    private org.sqlite.jni.capi.sqlite3 value;
+    /** Initializes with a null value. */
+    public sqlite3(){value = null;}
+    /** Sets the current value to null. */
+    public void clear(){value = null;}
+    /** Returns the current value. */
+    public final org.sqlite.jni.capi.sqlite3 get(){return value;}
+    /** Equivalent to calling get() then clear(). */
+    public final org.sqlite.jni.capi.sqlite3 take(){
+      final org.sqlite.jni.capi.sqlite3 v = value;
+      value = null;
+      return v;
+    }
+  }
+
+  /**
+     Output pointer for sqlite3_blob_open(). These
+     pointers can only be set by the JNI layer, not by client-level
+     code.
+  */
+  public static final class sqlite3_blob {
+    private org.sqlite.jni.capi.sqlite3_blob value;
+    /** Initializes with a null value. */
+    public sqlite3_blob(){value = null;}
+    /** Sets the current value to null. */
+    public void clear(){value = null;}
+    /** Returns the current value. */
+    public final org.sqlite.jni.capi.sqlite3_blob get(){return value;}
+    /** Equivalent to calling get() then clear(). */
+    public final org.sqlite.jni.capi.sqlite3_blob take(){
+      final org.sqlite.jni.capi.sqlite3_blob v = value;
+      value = null;
+      return v;
+    }
+  }
+
+  /**
+     Output pointer for use with routines, such as sqlite3_prepare(),
+     which return a statement handle via an output pointer. These
+     pointers can only be set by the JNI layer, not by client-level
+     code.
+  */
+  public static final class sqlite3_stmt {
+    private org.sqlite.jni.capi.sqlite3_stmt value;
+    /** Initializes with a null value. */
+    public sqlite3_stmt(){value = null;}
+    /** Sets the current value to null. */
+    public void clear(){value = null;}
+    /** Returns the current value. */
+    public final org.sqlite.jni.capi.sqlite3_stmt get(){return value;}
+    /** Equivalent to calling get() then clear(). */
+    public final org.sqlite.jni.capi.sqlite3_stmt take(){
+      final org.sqlite.jni.capi.sqlite3_stmt v = value;
+      value = null;
+      return v;
+    }
+  }
+
+  /**
+     Output pointer for use with routines, such as sqlite3_prepupdate_new(),
+     which return a sqlite3_value handle via an output pointer. These
+     pointers can only be set by the JNI layer, not by client-level
+     code.
+  */
+  public static final class sqlite3_value {
+    private org.sqlite.jni.capi.sqlite3_value value;
+    /** Initializes with a null value. */
+    public sqlite3_value(){value = null;}
+    /** Sets the current value to null. */
+    public void clear(){value = null;}
+    /** Returns the current value. */
+    public final org.sqlite.jni.capi.sqlite3_value get(){return value;}
+    /** Equivalent to calling get() then clear(). */
+    public final org.sqlite.jni.capi.sqlite3_value take(){
+      final org.sqlite.jni.capi.sqlite3_value v = value;
+      value = null;
+      return v;
+    }
+  }
+
+  /**
+     Output pointer for use with native routines which return booleans
+     via integer output pointers.
+  */
+  public static final class Bool {
+    /**
+       This is public for ease of use. Accessors are provided for
+       consistency with the higher-level types.
+    */
+    public boolean value;
+    /** Initializes with the value 0. */
+    public Bool(){this(false);}
+    /** Initializes with the value v. */
+    public Bool(boolean v){value = v;}
+    /** Returns the current value. */
+    public final boolean get(){return value;}
+    /** Sets the current value to v. */
+    public final void set(boolean v){value = v;}
+  }
+
+  /**
+     Output pointer for use with native routines which return integers via
+     output pointers.
+  */
+  public static final class Int32 {
+    /**
+       This is public for ease of use. Accessors are provided for
+       consistency with the higher-level types.
+    */
+    public int value;
+    /** Initializes with the value 0. */
+    public Int32(){this(0);}
+    /** Initializes with the value v. */
+    public Int32(int v){value = v;}
+    /** Returns the current value. */
+    public final int get(){return value;}
+    /** Sets the current value to v. */
+    public final void set(int v){value = v;}
+  }
+
+  /**
+     Output pointer for use with native routines which return 64-bit integers
+     via output pointers.
+  */
+  public static final class Int64 {
+    /**
+       This is public for ease of use. Accessors are provided for
+       consistency with the higher-level types.
+    */
+    public long value;
+    /** Initializes with the value 0. */
+    public Int64(){this(0);}
+    /** Initializes with the value v. */
+    public Int64(long v){value = v;}
+    /** Returns the current value. */
+    public final long get(){return value;}
+    /** Sets the current value. */
+    public final void set(long v){value = v;}
+  }
+
+  /**
+     Output pointer for use with native routines which return strings via
+     output pointers.
+  */
+  public static final class String {
+    /**
+       This is public for ease of use. Accessors are provided for
+       consistency with the higher-level types.
+    */
+    public java.lang.String value;
+    /** Initializes with a null value. */
+    public String(){this(null);}
+    /** Initializes with the value v. */
+    public String(java.lang.String v){value = v;}
+    /** Returns the current value. */
+    public final java.lang.String get(){return value;}
+    /** Sets the current value. */
+    public final void set(java.lang.String v){value = v;}
+  }
+
+  /**
+     Output pointer for use with native routines which return byte
+     arrays via output pointers.
+  */
+  public static final class ByteArray {
+    /**
+       This is public for ease of use. Accessors are provided for
+       consistency with the higher-level types.
+    */
+    public byte[] value;
+    /** Initializes with the value null. */
+    public ByteArray(){this(null);}
+    /** Initializes with the value v. */
+    public ByteArray(byte[] v){value = v;}
+    /** Returns the current value. */
+    public final byte[] get(){return value;}
+    /** Sets the current value. */
+    public final void set(byte[] v){value = v;}
+  }
+}

ext/jni/src/org/sqlite/jni/capi/PrepareMultiCallback.java

@@ -0,0 +1,78 @@
+/*
+** 2023-09-13
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file is part of the JNI bindings for the sqlite3 C API.
+*/
+package org.sqlite.jni.capi;
+
+/**
+   Callback for use with {@link CApi#sqlite3_prepare_multi}.
+*/
+public interface PrepareMultiCallback extends CallbackProxy {
+
+  /**
+     Gets passed a sqlite3_stmt which it may handle in arbitrary ways,
+     transfering ownership of it to this function.
+
+     sqlite3_prepare_multi() will _not_ finalize st - it is up
+     to the call() implementation how st is handled.
+
+     Must return 0 on success or an SQLITE_... code on error.
+
+     See the {@link Finalize} class for a wrapper which finalizes the
+     statement after calling a proxy PrepareMultiCallback.
+  */
+  int call(sqlite3_stmt st);
+
+  /**
+     A PrepareMultiCallback impl which wraps a separate impl and finalizes
+     any sqlite3_stmt passed to its callback.
+  */
+  public static final class Finalize implements PrepareMultiCallback {
+    private PrepareMultiCallback p;
+    /**
+       p is the proxy to call() when this.call() is called.
+    */
+    public Finalize( PrepareMultiCallback p ){
+      this.p = p;
+    }
+    /**
+       Calls the call() method of the proxied callback and either returns its
+       result or propagates an exception. Either way, it passes its argument to
+       sqlite3_finalize() before returning.
+    */
+    @Override public int call(sqlite3_stmt st){
+      try {
+        return this.p.call(st);
+      }finally{
+        CApi.sqlite3_finalize(st);
+      }
+    }
+  }
+
+  /**
+     A PrepareMultiCallback impl which steps entirely through a result set,
+     ignoring all non-error results.
+  */
+  public static final class StepAll implements PrepareMultiCallback {
+    public StepAll(){}
+    /**
+       Calls sqlite3_step() on st until it returns something other than
+       SQLITE_ROW. If the final result is SQLITE_DONE then 0 is returned,
+       else the result of the final step is returned.
+    */
+    @Override public int call(sqlite3_stmt st){
+      int rc = CApi.SQLITE_DONE;
+      while( CApi.SQLITE_ROW == (rc = CApi.sqlite3_step(st)) ){}
+      return CApi.SQLITE_DONE==rc ? 0 : rc;
+    }
+  }
+}

ext/jni/src/org/sqlite/jni/capi/PreupdateHookCallback.java

@@ -1,5 +1,5 @@
 /*
-** 2023-07-22
+** 2023-08-25
 **
 ** The author disclaims copyright to this source code.  In place of
 ** a legal notice, here is a blessing:
@@ -11,18 +11,16 @@
 *************************************************************************
 ** This file is part of the JNI bindings for the sqlite3 C API.
 */
-package org.sqlite.jni;
+package org.sqlite.jni.capi;
 
 /**
+   Callback for use with {@link CApi#sqlite3_preupdate_hook}.
 */
-public abstract class Collation {
+public interface PreupdateHookCallback extends CallbackProxy {
   /**
-     Must compare the given byte arrays using memcmp() semantics.
+     Must function as described for the C-level sqlite3_preupdate_hook()
+     callback.
   */
-  public abstract int xCompare(byte[] lhs, byte[] rhs);
+  void call(sqlite3 db, int op, String dbName, String dbTable,
-  /**
+            long iKey1, long iKey2 );
-     Called by SQLite when the collation is destroyed. If a Collation
-     requires custom cleanup, override this method.
-  */
-  public void xDestroy() {}
 }

ext/jni/src/org/sqlite/jni/capi/ProgressHandlerCallback.java

@@ -1,5 +1,5 @@
 /*
-** 2023-07-22
+** 2023-08-25
 **
 ** The author disclaims copyright to this source code.  In place of
 ** a legal notice, here is a blessing:
@@ -11,17 +11,17 @@
 *************************************************************************
 ** This file is part of the JNI bindings for the sqlite3 C API.
 */
-package org.sqlite.jni;
+package org.sqlite.jni.capi;
 
 /**
-   Callback proxy for use with sqlite3_progress_handler().
+   Callback for use with {@link CApi#sqlite3_progress_handler}.
 */
-public interface ProgressHandler {
+public interface ProgressHandlerCallback extends CallbackProxy {
   /**
-     Works as documented for the sqlite3_progress_handler() callback.
+     Works as documented for the C-level sqlite3_progress_handler() callback.
 
-     If it throws, the exception message is passed on to the db and
+     <p>If it throws, the exception message is passed on to the db and
      the exception is suppressed.
   */
-  int xCallback();
+  int call();
 }

ext/jni/src/org/sqlite/jni/capi/ResultCode.java

@@ -0,0 +1,155 @@
+/*
+** 2023-07-21
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file is part of the JNI bindings for the sqlite3 C API.
+*/
+package org.sqlite.jni.capi;
+
+/**
+   This enum contains all of the core and "extended" result codes used
+   by the sqlite3 library. It is provided not for use with the C-style
+   API (with which it won't work) but for higher-level code which may
+   find it useful to map SQLite result codes to human-readable names.
+*/
+public enum ResultCode {
+  SQLITE_OK(CApi.SQLITE_OK),
+  SQLITE_ERROR(CApi.SQLITE_ERROR),
+  SQLITE_INTERNAL(CApi.SQLITE_INTERNAL),
+  SQLITE_PERM(CApi.SQLITE_PERM),
+  SQLITE_ABORT(CApi.SQLITE_ABORT),
+  SQLITE_BUSY(CApi.SQLITE_BUSY),
+  SQLITE_LOCKED(CApi.SQLITE_LOCKED),
+  SQLITE_NOMEM(CApi.SQLITE_NOMEM),
+  SQLITE_READONLY(CApi.SQLITE_READONLY),
+  SQLITE_INTERRUPT(CApi.SQLITE_INTERRUPT),
+  SQLITE_IOERR(CApi.SQLITE_IOERR),
+  SQLITE_CORRUPT(CApi.SQLITE_CORRUPT),
+  SQLITE_NOTFOUND(CApi.SQLITE_NOTFOUND),
+  SQLITE_FULL(CApi.SQLITE_FULL),
+  SQLITE_CANTOPEN(CApi.SQLITE_CANTOPEN),
+  SQLITE_PROTOCOL(CApi.SQLITE_PROTOCOL),
+  SQLITE_EMPTY(CApi.SQLITE_EMPTY),
+  SQLITE_SCHEMA(CApi.SQLITE_SCHEMA),
+  SQLITE_TOOBIG(CApi.SQLITE_TOOBIG),
+  SQLITE_CONSTRAINT(CApi.SQLITE_CONSTRAINT),
+  SQLITE_MISMATCH(CApi.SQLITE_MISMATCH),
+  SQLITE_MISUSE(CApi.SQLITE_MISUSE),
+  SQLITE_NOLFS(CApi.SQLITE_NOLFS),
+  SQLITE_AUTH(CApi.SQLITE_AUTH),
+  SQLITE_FORMAT(CApi.SQLITE_FORMAT),
+  SQLITE_RANGE(CApi.SQLITE_RANGE),
+  SQLITE_NOTADB(CApi.SQLITE_NOTADB),
+  SQLITE_NOTICE(CApi.SQLITE_NOTICE),
+  SQLITE_WARNING(CApi.SQLITE_WARNING),
+  SQLITE_ROW(CApi.SQLITE_ROW),
+  SQLITE_DONE(CApi.SQLITE_DONE),
+  SQLITE_ERROR_MISSING_COLLSEQ(CApi.SQLITE_ERROR_MISSING_COLLSEQ),
+  SQLITE_ERROR_RETRY(CApi.SQLITE_ERROR_RETRY),
+  SQLITE_ERROR_SNAPSHOT(CApi.SQLITE_ERROR_SNAPSHOT),
+  SQLITE_IOERR_READ(CApi.SQLITE_IOERR_READ),
+  SQLITE_IOERR_SHORT_READ(CApi.SQLITE_IOERR_SHORT_READ),
+  SQLITE_IOERR_WRITE(CApi.SQLITE_IOERR_WRITE),
+  SQLITE_IOERR_FSYNC(CApi.SQLITE_IOERR_FSYNC),
+  SQLITE_IOERR_DIR_FSYNC(CApi.SQLITE_IOERR_DIR_FSYNC),
+  SQLITE_IOERR_TRUNCATE(CApi.SQLITE_IOERR_TRUNCATE),
+  SQLITE_IOERR_FSTAT(CApi.SQLITE_IOERR_FSTAT),
+  SQLITE_IOERR_UNLOCK(CApi.SQLITE_IOERR_UNLOCK),
+  SQLITE_IOERR_RDLOCK(CApi.SQLITE_IOERR_RDLOCK),
+  SQLITE_IOERR_DELETE(CApi.SQLITE_IOERR_DELETE),
+  SQLITE_IOERR_BLOCKED(CApi.SQLITE_IOERR_BLOCKED),
+  SQLITE_IOERR_NOMEM(CApi.SQLITE_IOERR_NOMEM),
+  SQLITE_IOERR_ACCESS(CApi.SQLITE_IOERR_ACCESS),
+  SQLITE_IOERR_CHECKRESERVEDLOCK(CApi.SQLITE_IOERR_CHECKRESERVEDLOCK),
+  SQLITE_IOERR_LOCK(CApi.SQLITE_IOERR_LOCK),
+  SQLITE_IOERR_CLOSE(CApi.SQLITE_IOERR_CLOSE),
+  SQLITE_IOERR_DIR_CLOSE(CApi.SQLITE_IOERR_DIR_CLOSE),
+  SQLITE_IOERR_SHMOPEN(CApi.SQLITE_IOERR_SHMOPEN),
+  SQLITE_IOERR_SHMSIZE(CApi.SQLITE_IOERR_SHMSIZE),
+  SQLITE_IOERR_SHMLOCK(CApi.SQLITE_IOERR_SHMLOCK),
+  SQLITE_IOERR_SHMMAP(CApi.SQLITE_IOERR_SHMMAP),
+  SQLITE_IOERR_SEEK(CApi.SQLITE_IOERR_SEEK),
+  SQLITE_IOERR_DELETE_NOENT(CApi.SQLITE_IOERR_DELETE_NOENT),
+  SQLITE_IOERR_MMAP(CApi.SQLITE_IOERR_MMAP),
+  SQLITE_IOERR_GETTEMPPATH(CApi.SQLITE_IOERR_GETTEMPPATH),
+  SQLITE_IOERR_CONVPATH(CApi.SQLITE_IOERR_CONVPATH),
+  SQLITE_IOERR_VNODE(CApi.SQLITE_IOERR_VNODE),
+  SQLITE_IOERR_AUTH(CApi.SQLITE_IOERR_AUTH),
+  SQLITE_IOERR_BEGIN_ATOMIC(CApi.SQLITE_IOERR_BEGIN_ATOMIC),
+  SQLITE_IOERR_COMMIT_ATOMIC(CApi.SQLITE_IOERR_COMMIT_ATOMIC),
+  SQLITE_IOERR_ROLLBACK_ATOMIC(CApi.SQLITE_IOERR_ROLLBACK_ATOMIC),
+  SQLITE_IOERR_DATA(CApi.SQLITE_IOERR_DATA),
+  SQLITE_IOERR_CORRUPTFS(CApi.SQLITE_IOERR_CORRUPTFS),
+  SQLITE_LOCKED_SHAREDCACHE(CApi.SQLITE_LOCKED_SHAREDCACHE),
+  SQLITE_LOCKED_VTAB(CApi.SQLITE_LOCKED_VTAB),
+  SQLITE_BUSY_RECOVERY(CApi.SQLITE_BUSY_RECOVERY),
+  SQLITE_BUSY_SNAPSHOT(CApi.SQLITE_BUSY_SNAPSHOT),
+  SQLITE_BUSY_TIMEOUT(CApi.SQLITE_BUSY_TIMEOUT),
+  SQLITE_CANTOPEN_NOTEMPDIR(CApi.SQLITE_CANTOPEN_NOTEMPDIR),
+  SQLITE_CANTOPEN_ISDIR(CApi.SQLITE_CANTOPEN_ISDIR),
+  SQLITE_CANTOPEN_FULLPATH(CApi.SQLITE_CANTOPEN_FULLPATH),
+  SQLITE_CANTOPEN_CONVPATH(CApi.SQLITE_CANTOPEN_CONVPATH),
+  SQLITE_CANTOPEN_SYMLINK(CApi.SQLITE_CANTOPEN_SYMLINK),
+  SQLITE_CORRUPT_VTAB(CApi.SQLITE_CORRUPT_VTAB),
+  SQLITE_CORRUPT_SEQUENCE(CApi.SQLITE_CORRUPT_SEQUENCE),
+  SQLITE_CORRUPT_INDEX(CApi.SQLITE_CORRUPT_INDEX),
+  SQLITE_READONLY_RECOVERY(CApi.SQLITE_READONLY_RECOVERY),
+  SQLITE_READONLY_CANTLOCK(CApi.SQLITE_READONLY_CANTLOCK),
+  SQLITE_READONLY_ROLLBACK(CApi.SQLITE_READONLY_ROLLBACK),
+  SQLITE_READONLY_DBMOVED(CApi.SQLITE_READONLY_DBMOVED),
+  SQLITE_READONLY_CANTINIT(CApi.SQLITE_READONLY_CANTINIT),
+  SQLITE_READONLY_DIRECTORY(CApi.SQLITE_READONLY_DIRECTORY),
+  SQLITE_ABORT_ROLLBACK(CApi.SQLITE_ABORT_ROLLBACK),
+  SQLITE_CONSTRAINT_CHECK(CApi.SQLITE_CONSTRAINT_CHECK),
+  SQLITE_CONSTRAINT_COMMITHOOK(CApi.SQLITE_CONSTRAINT_COMMITHOOK),
+  SQLITE_CONSTRAINT_FOREIGNKEY(CApi.SQLITE_CONSTRAINT_FOREIGNKEY),
+  SQLITE_CONSTRAINT_FUNCTION(CApi.SQLITE_CONSTRAINT_FUNCTION),
+  SQLITE_CONSTRAINT_NOTNULL(CApi.SQLITE_CONSTRAINT_NOTNULL),
+  SQLITE_CONSTRAINT_PRIMARYKEY(CApi.SQLITE_CONSTRAINT_PRIMARYKEY),
+  SQLITE_CONSTRAINT_TRIGGER(CApi.SQLITE_CONSTRAINT_TRIGGER),
+  SQLITE_CONSTRAINT_UNIQUE(CApi.SQLITE_CONSTRAINT_UNIQUE),
+  SQLITE_CONSTRAINT_VTAB(CApi.SQLITE_CONSTRAINT_VTAB),
+  SQLITE_CONSTRAINT_ROWID(CApi.SQLITE_CONSTRAINT_ROWID),
+  SQLITE_CONSTRAINT_PINNED(CApi.SQLITE_CONSTRAINT_PINNED),
+  SQLITE_CONSTRAINT_DATATYPE(CApi.SQLITE_CONSTRAINT_DATATYPE),
+  SQLITE_NOTICE_RECOVER_WAL(CApi.SQLITE_NOTICE_RECOVER_WAL),
+  SQLITE_NOTICE_RECOVER_ROLLBACK(CApi.SQLITE_NOTICE_RECOVER_ROLLBACK),
+  SQLITE_WARNING_AUTOINDEX(CApi.SQLITE_WARNING_AUTOINDEX),
+  SQLITE_AUTH_USER(CApi.SQLITE_AUTH_USER),
+  SQLITE_OK_LOAD_PERMANENTLY(CApi.SQLITE_OK_LOAD_PERMANENTLY);
+
+  public final int value;
+
+  ResultCode(int rc){
+    value = rc;
+    ResultCodeMap.set(rc, this);
+  }
+
+  /**
+     Returns the entry from this enum for the given result code, or
+     null if no match is found.
+  */
+  public static ResultCode getEntryForInt(int rc){
+    return ResultCodeMap.get(rc);
+  }
+
+  /**
+     Internal level of indirection required because we cannot initialize
+     static enum members in an enum before the enum constructor is
+     invoked.
+  */
+  private static final class ResultCodeMap {
+    private static final java.util.Map<Integer,ResultCode> i2e
+      = new java.util.HashMap<>();
+    private static void set(int rc, ResultCode e){ i2e.put(rc, e); }
+    private static ResultCode get(int rc){ return i2e.get(rc); }
+  }
+
+}

ext/jni/src/org/sqlite/jni/capi/RollbackHookCallback.java

@@ -1,5 +1,5 @@
 /*
-** 2023-07-22
+** 2023-08-25
 **
 ** The author disclaims copyright to this source code.  In place of
 ** a legal notice, here is a blessing:
@@ -11,15 +11,15 @@
 *************************************************************************
 ** This file is part of the JNI bindings for the sqlite3 C API.
 */
-package org.sqlite.jni;
+package org.sqlite.jni.capi;
 
 /**
-   Callback proxy for use with sqlite3_rollback_hook().
+   Callback for use with {@link CApi#sqlite3_rollback_hook}.
 */
-public interface RollbackHook {
+public interface RollbackHookCallback extends CallbackProxy {
   /**
-     Works as documented for the sqlite3_rollback_hook() callback.
+     Works as documented for the C-level sqlite3_rollback_hook()
-     Must not throw.
+     callback.
   */
-  void xRollbackHook();
+  void call();
 }

ext/jni/src/org/sqlite/jni/capi/SQLFunction.java

@@ -0,0 +1,103 @@
+/*
+** 2023-07-22
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file is part of the JNI bindings for the sqlite3 C API.
+*/
+package org.sqlite.jni.capi;
+
+/**
+   SQLFunction is used in conjunction with the
+   sqlite3_create_function() JNI-bound API to give that native code
+   access to the callback functions needed in order to implement SQL
+   functions in Java.
+
+   <p>
+
+   This class is not used by itself, but is a marker base class. The
+   three UDF types are modelled by the inner classes Scalar,
+   Aggregate<T>, and Window<T>. Most simply, clients may subclass
+   those, or create anonymous classes from them, to implement
+   UDFs. Clients are free to create their own classes for use with
+   UDFs, so long as they conform to the public interfaces defined by
+   those three classes. The JNI layer only actively relies on the
+   SQLFunction base class and the method names and signatures used by
+   the UDF callback interfaces.
+*/
+public interface SQLFunction {
+
+  /**
+     PerContextState assists aggregate and window functions in
+     managing their accumulator state across calls to the UDF's
+     callbacks.
+
+     <p>T must be of a type which can be legally stored as a value in
+     java.util.HashMap<KeyType,T>.
+
+     <p>If a given aggregate or window function is called multiple times
+     in a single SQL statement, e.g. SELECT MYFUNC(A), MYFUNC(B)...,
+     then the clients need some way of knowing which call is which so
+     that they can map their state between their various UDF callbacks
+     and reset it via xFinal(). This class takes care of such
+     mappings.
+
+     <p>This class works by mapping
+     sqlite3_context.getAggregateContext() to a single piece of
+     state, of a client-defined type (the T part of this class), which
+     persists across a "matching set" of the UDF's callbacks.
+
+     <p>This class is a helper providing commonly-needed functionality
+     - it is not required for use with aggregate or window functions.
+     Client UDFs are free to perform such mappings using custom
+     approaches. The provided {@link AggregateFunction} and {@link
+     WindowFunction} classes use this.
+  */
+  public static final class PerContextState<T> {
+    private final java.util.Map<Long,ValueHolder<T>> map
+      = new java.util.HashMap<>();
+
+    /**
+       Should be called from a UDF's xStep(), xValue(), and xInverse()
+       methods, passing it that method's first argument and an initial
+       value for the persistent state. If there is currently no
+       mapping for the given context within the map, one is created
+       using the given initial value, else the existing one is used
+       and the 2nd argument is ignored.  It returns a ValueHolder<T>
+       which can be used to modify that state directly without
+       requiring that the client update the underlying map's entry.
+
+       <p>The caller is obligated to eventually call
+       takeAggregateState() to clear the mapping.
+    */
+    public ValueHolder<T> getAggregateState(sqlite3_context cx, T initialValue){
+      final Long key = cx.getAggregateContext(true);
+      ValueHolder<T> rc = null==key ? null : map.get(key);
+      if( null==rc ){
+        map.put(key, rc = new ValueHolder<>(initialValue));
+      }
+      return rc;
+    }
+
+    /**
+       Should be called from a UDF's xFinal() method and passed that
+       method's first argument. This function removes the value
+       associated with cx.getAggregateContext() from the map and
+       returns it, returning null if no other UDF method has been
+       called to set up such a mapping. The latter condition will be
+       the case if a UDF is used in a statement which has no result
+       rows.
+    */
+    public T takeAggregateState(sqlite3_context cx){
+      final ValueHolder<T> h = map.remove(cx.getAggregateContext(false));
+      return null==h ? null : h.value;
+    }
+  }
+
+}

ext/jni/src/org/sqlite/jni/capi/SQLTester.java

@@ -12,16 +12,13 @@
 ** This file contains the main application entry pointer for the
 ** SQLTester framework.
 */
-package org.sqlite.jni.tester;
+package org.sqlite.jni.capi;
 import java.util.List;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.nio.charset.StandardCharsets;
 import java.util.regex.*;
-import org.sqlite.jni.*;
+import static org.sqlite.jni.capi.CApi.*;
-import static org.sqlite.jni.SQLite3Jni.*;
-import org.sqlite.jni.sqlite3;
-
 
 /**
    Modes for how to escape (or not) column values and names from
@@ -150,14 +147,17 @@ class Outer {
 }
 
 /**
-   This class provides an application which aims to implement the
+   <p>This class provides an application which aims to implement the
    rudimentary SQL-driven test tool described in the accompanying
-   test-script-interpreter.md.
+   {@code test-script-interpreter.md}.
 
-   This is a work in progress.
+   <p>This class is an internal testing tool, not part of the public
+   interface but is (A) in the same package as the library because
+   access permissions require it to be so and (B) the JDK8 javadoc
+   offers no way to filter individual classes out of the doc
+   generation process (it can only exclude packages, but see (A)).
 
-
+   <p>An instance of this application provides a core set of services
-   An instance of this application provides a core set of services
    which TestScript instances use for processing testing logic.
    TestScripts, in turn, delegate the concrete test work to Command
    objects, which the TestScript parses on their behalf.
@@ -181,7 +181,7 @@ public class SQLTester {
   private int nTestFile = 0;
   //! Number of scripts which were aborted.
   private int nAbortedScript = 0;
-  //! Per-script test counter.
+  //! Incremented by test case handlers
   private int nTest = 0;
   //! True to enable column name output from execSql()
   private boolean emitColNames;
@@ -250,14 +250,14 @@ public class SQLTester {
   }
 
   public void runTests() throws Exception {
-    final long tStart = System.nanoTime();
+    final long tStart = System.currentTimeMillis();
     for(String f : listInFiles){
       reset();
       ++nTestFile;
       final TestScript ts = new TestScript(f);
       outln(nextStartEmoji(), " starting [",f,"]");
       boolean threw = false;
-      final long timeStart = System.nanoTime();
+      final long timeStart = System.currentTimeMillis();
       try{
         ts.run(this);
       }catch(SQLTesterException e){
@@ -267,14 +267,13 @@ public class SQLTester {
         if( keepGoing ) outln("Continuing anyway becaure of the keep-going option.");
         else if( e.isFatal() ) throw e;
       }finally{
-        final long timeEnd = System.nanoTime();
+        final long timeEnd = System.currentTimeMillis();
         outln("🏁",(threw ? "❌" : "✅")," ",nTest," test(s) in ",
-              ((timeEnd-timeStart)/1000000.0),"ms.");
+              (timeEnd-timeStart),"ms.");
-        //ts.getFilename());
       }
     }
-    final long tEnd = System.nanoTime();
+    final long tEnd = System.currentTimeMillis();
-    outln("Total run-time: ",((tEnd-tStart)/1000000.0),"ms");
+    outln("Total run-time: ",(tEnd-tStart),"ms");
     Util.unlink(initialDbName);
   }
 
@@ -336,7 +335,9 @@ public class SQLTester {
   }
 
   sqlite3 setCurrentDb(int n) throws Exception{
-    return affirmDbId(n).aDb[n];
+    affirmDbId(n);
+    iCurrentDb = n;
+    return this.aDb[n];
   }
 
   sqlite3 getCurrentDb(){ return aDb[iCurrentDb]; }
@@ -399,7 +400,7 @@ public class SQLTester {
     nullView = "nil";
     emitColNames = false;
     iCurrentDb = 0;
-    dbInitSql.append("SELECT 1;");
+    //dbInitSql.append("SELECT 1;");
   }
 
   void setNullValue(String v){nullView = v;}
@@ -456,7 +457,7 @@ public class SQLTester {
   }
 
   private void appendDbErr(sqlite3 db, StringBuilder sb, int rc){
-    sb.append(org.sqlite.jni.ResultCode.getEntryForInt(rc)).append(' ');
+    sb.append(org.sqlite.jni.capi.ResultCode.getEntryForInt(rc)).append(' ');
     final String msg = escapeSqlValue(sqlite3_errmsg(db));
     if( '{' == msg.charAt(0) ){
       sb.append(msg);
@@ -474,12 +475,12 @@ public class SQLTester {
      the db's result code.
 
      appendMode specifies how/whether to append results to the result
-     buffer. lineMode specifies whether to output all results in a
+     buffer. rowMode specifies whether to output all results in a
      single line or one line per row. If appendMode is
-     ResultBufferMode.NONE then lineMode is ignored and may be null.
+     ResultBufferMode.NONE then rowMode is ignored and may be null.
   */
   public int execSql(sqlite3 db, boolean throwOnError,
-                     ResultBufferMode appendMode, ResultRowMode lineMode,
+                     ResultBufferMode appendMode, ResultRowMode rowMode,
                      String sql) throws SQLTesterException {
     if( null==db && null==aDb[0] ){
       // Delay opening of the initial db to enable tests to change its
@@ -561,7 +562,7 @@ public class SQLTester {
                   throw new SQLTesterException("Unhandled ResultBufferMode: "+appendMode);
               }
             }
-            if( ResultRowMode.NEWLINE == lineMode ){
+            if( ResultRowMode.NEWLINE == rowMode ){
               spacing = 0;
               sb.append('\n');
             }
@@ -580,6 +581,10 @@ public class SQLTester {
         }
       }
     }finally{
+      sqlite3_reset(stmt
+        /* In order to trigger an exception in the
+           INSERT...RETURNING locking scenario:
+           https://sqlite.org/forum/forumpost/36f7a2e7494897df */);
       sqlite3_finalize(stmt);
     }
     if( 0!=rc && throwOnError ){
@@ -609,9 +614,9 @@ public class SQLTester {
       }
       t.addTestScript(a);
     }
-    final AutoExtension ax = new AutoExtension() {
+    final AutoExtensionCallback ax = new AutoExtensionCallback() {
         private final SQLTester tester = t;
-        public int xEntryPoint(sqlite3 db){
+        @Override public int call(sqlite3 db){
           final String init = tester.getDbInitSql();
           if( !init.isEmpty() ){
             tester.execSql(db, true, ResultBufferMode.NONE, null, init);
@@ -629,7 +634,7 @@ public class SQLTester {
         t.outln("Aborted ",t.nAbortedScript," script(s).");
       }
       if( dumpInternals ){
-        sqlite3_do_something_for_developer();
+        sqlite3_jni_internal_details();
       }
     }
   }
@@ -663,7 +668,7 @@ public class SQLTester {
   static {
     System.loadLibrary("sqlite3-jni")
       /* Interestingly, when SQLTester is the main app, we have to
-         load that lib from here. The same load from SQLite3Jni does
+         load that lib from here. The same load from CApi does
          not happen early enough. Without this,
          installCustomExtensions() is an unresolved symbol. */;
   }
@@ -924,7 +929,7 @@ class RunCommand extends Command {
     final sqlite3 db = (1==argv.length)
       ? t.getCurrentDb() : t.getDbById( Integer.parseInt(argv[1]) );
     final String sql = t.takeInputBuffer();
-    int rc = t.execSql(db, false, ResultBufferMode.NONE,
+    final int rc = t.execSql(db, false, ResultBufferMode.NONE,
                              ResultRowMode.ONELINE, sql);
     if( 0!=rc && t.isVerbose() ){
       String msg = sqlite3_errmsg(db);
@@ -948,8 +953,7 @@ class TableResultCommand extends Command {
     if( !body.endsWith("\n--end") ){
       ts.toss(argv[0], " must be terminated with --end.");
     }else{
-      int n = body.length();
+      body = body.substring(0, body.length()-6);
-      body = body.substring(0, n-6);
     }
     final String[] globs = body.split("\\s*\\n\\s*");
     if( globs.length < 1 ){
@@ -1124,8 +1128,9 @@ class TestScript {
   }
 
   public String getOutputPrefix(){
-    String rc =  "["+(moduleName==null ? filename : moduleName)+"]";
+    String rc = "["+(moduleName==null ? "<unnamed>" : moduleName)+"]";
     if( null!=testCaseName ) rc += "["+testCaseName+"]";
+    if( null!=filename ) rc += "["+filename+"]";
     return rc + " line "+ cur.lineNo;
   }
 
@@ -1238,14 +1243,15 @@ class TestScript {
     final int oldPB = cur.putbackPos;
     final int oldPBL = cur.putbackLineNo;
     final int oldLine = cur.lineNo;
-    final String rc = getLine();
+    try{ return getLine(); }
+    finally{
       cur.peekedPos = cur.pos;
       cur.peekedLineNo = cur.lineNo;
       cur.pos = oldPos;
       cur.lineNo = oldLine;
       cur.putbackPos = oldPB;
       cur.putbackLineNo = oldPBL;
-    return rc;
+    }
   }
 
   /**
@@ -1268,6 +1274,7 @@ class TestScript {
   }
 
   private boolean checkRequiredProperties(SQLTester t, String[] props) throws SQLTesterException{
+    if( true ) return false;
     int nOk = 0;
     for(String rp : props){
       verbose1("REQUIRED_PROPERTIES: ",rp);
@@ -1288,6 +1295,12 @@ class TestScript {
           t.appendDbInitSql("pragma temp_store=0;");
           ++nOk;
           break;
+        case "AUTOVACUUM":
+          t.appendDbInitSql("pragma auto_vacuum=full;");
+          ++nOk;
+        case "INCRVACUUM":
+          t.appendDbInitSql("pragma auto_vacuum=incremental;");
+          ++nOk;
         default:
           break;
       }
@@ -1326,9 +1339,9 @@ class TestScript {
     m = patternRequiredProperties.matcher(line);
     if( m.find() ){
       final String rp = m.group(1);
-      //if( ! checkRequiredProperties( tester, rp.split("\\s+") ) ){
+      if( ! checkRequiredProperties( tester, rp.split("\\s+") ) ){
         throw new IncompatibleDirective(this, "REQUIRED_PROPERTIES: "+rp);
-      //}
+      }
     }
     m = patternMixedModuleName.matcher(line);
     if( m.find() ){
@@ -1372,11 +1385,10 @@ class TestScript {
     String line;
     while( (null != (line = peekLine())) ){
       checkForDirective(tester, line);
-      if( !isCommandLine(line, true) ){
+      if( isCommandLine(line, true) ) break;
+      else {
         sb.append(line).append("\n");
         consumePeeked();
-      }else{
-        break;
       }
     }
     line = sb.toString();

ext/jni/src/org/sqlite/jni/capi/ScalarFunction.java

@@ -0,0 +1,33 @@
+/*
+** 2023-08-25
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file is part of the JNI bindings for the sqlite3 C API.
+*/
+package org.sqlite.jni.capi;
+
+
+/**
+   A SQLFunction implementation for scalar functions.
+*/
+public abstract class ScalarFunction implements SQLFunction {
+  /**
+     As for the xFunc() argument of the C API's
+     sqlite3_create_function(). If this function throws, it is
+     translated into an sqlite3_result_error().
+  */
+  public abstract void xFunc(sqlite3_context cx, sqlite3_value[] args);
+
+  /**
+     Optionally override to be notified when the UDF is finalized by
+     SQLite. This default implementation does nothing.
+  */
+  public void xDestroy() {}
+}

ext/jni/src/org/sqlite/jni/capi/TableColumnMetadata.java

@@ -0,0 +1,35 @@
+/*
+** 2023-07-21
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file is part of the JNI bindings for the sqlite3 C API.
+*/
+package org.sqlite.jni.capi;
+
+/**
+   A wrapper object for use with sqlite3_table_column_metadata().
+   They are populated only via that interface.
+*/
+public final class TableColumnMetadata {
+  OutputPointer.Bool pNotNull = new OutputPointer.Bool();
+  OutputPointer.Bool pPrimaryKey = new OutputPointer.Bool();
+  OutputPointer.Bool pAutoinc = new OutputPointer.Bool();
+  OutputPointer.String pzCollSeq = new OutputPointer.String();
+  OutputPointer.String pzDataType = new OutputPointer.String();
+
+  public TableColumnMetadata(){
+  }
+
+  public String getDataType(){ return pzDataType.value; }
+  public String getCollation(){ return pzCollSeq.value; }
+  public boolean isNotNull(){ return pNotNull.value; }
+  public boolean isPrimaryKey(){ return pPrimaryKey.value; }
+  public boolean isAutoincrement(){ return pAutoinc.value; }
+}

ext/jni/src/org/sqlite/jni/capi/TraceV2Callback.java

@@ -0,0 +1,50 @@
+/*
+** 2023-08-25
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file is part of the JNI bindings for the sqlite3 C API.
+*/
+package org.sqlite.jni.capi;
+import org.sqlite.jni.annotation.Nullable;
+
+/**
+   Callback for use with {@link CApi#sqlite3_trace_v2}.
+*/
+public interface TraceV2Callback extends CallbackProxy {
+  /**
+     Called by sqlite3 for various tracing operations, as per
+     sqlite3_trace_v2(). Note that this interface elides the 2nd
+     argument to the native trace callback, as that role is better
+     filled by instance-local state.
+
+     <p>These callbacks may throw, in which case their exceptions are
+     converted to C-level error information.
+
+     <p>The 2nd argument to this function, if non-null, will be a an
+     sqlite3 or sqlite3_stmt object, depending on the first argument
+     (see below).
+
+     <p>The final argument to this function is the "X" argument
+     documented for sqlite3_trace() and sqlite3_trace_v2(). Its type
+     depends on value of the first argument:
+
+     <p>- SQLITE_TRACE_STMT: pNative is a sqlite3_stmt. pX is a String
+     containing the prepared SQL.
+
+     <p>- SQLITE_TRACE_PROFILE: pNative is a sqlite3_stmt. pX is a Long
+     holding an approximate number of nanoseconds the statement took
+     to run.
+
+     <p>- SQLITE_TRACE_ROW: pNative is a sqlite3_stmt. pX is null.
+
+     <p>- SQLITE_TRACE_CLOSE: pNative is a sqlite3. pX is null.
+  */
+  int call(int traceFlag, Object pNative, @Nullable Object pX);
+}

ext/jni/src/org/sqlite/jni/capi/UpdateHookCallback.java

@@ -0,0 +1,25 @@
+/*
+** 2023-08-25
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file is part of the JNI bindings for the sqlite3 C API.
+*/
+package org.sqlite.jni.capi;
+
+/**
+   Callback for use with {@link CApi#sqlite3_update_hook}.
+*/
+public interface UpdateHookCallback extends CallbackProxy {
+  /**
+     Must function as described for the C-level sqlite3_update_hook()
+     callback.
+  */
+  void call(int opId, String dbName, String tableName, long rowId);
+}

ext/jni/src/org/sqlite/jni/capi/ValueHolder.java

@@ -1,5 +1,5 @@
 /*
-** 2023-07-21
+** 2023-10-16
 **
 ** The author disclaims copyright to this source code.  In place of
 ** a legal notice, here is a blessing:
@@ -9,12 +9,12 @@
 **    May you share freely, never taking more than you give.
 **
 *************************************************************************
-** This file is part of the JNI bindings for the sqlite3 C API.
+** This file contains a set of tests for the sqlite3 JNI bindings.
 */
-package org.sqlite.jni;
+package org.sqlite.jni.capi;
 
 /**
-   A helper class which simply holds a single value. Its current use
+   A helper class which simply holds a single value. Its primary use
    is for communicating values out of anonymous classes, as doing so
    requires a "final" reference.
 */

ext/jni/src/org/sqlite/jni/capi/WindowFunction.java

@@ -0,0 +1,39 @@
+/*
+** 2023-08-25
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file is part of the JNI bindings for the sqlite3 C API.
+*/
+package org.sqlite.jni.capi;
+
+
+/**
+   A SQLFunction implementation for window functions.  Note that
+   WindowFunction inherits from {@link AggregateFunction} and each
+   instance is required to implement the inherited abstract methods
+   from that class. See {@link AggregateFunction} for information on
+   managing the UDF's invocation-specific state.
+*/
+public abstract class WindowFunction<T> extends AggregateFunction<T> {
+
+  /**
+     As for the xInverse() argument of the C API's
+     sqlite3_create_window_function(). If this function throws, the
+     exception is not propagated and a warning might be emitted
+     to a debugging channel.
+  */
+  public abstract void xInverse(sqlite3_context cx, sqlite3_value[] args);
+
+  /**
+     As for the xValue() argument of the C API's sqlite3_create_window_function().
+     See xInverse() for the fate of any exceptions this throws.
+  */
+  public abstract void xValue(sqlite3_context cx);
+}

ext/jni/src/org/sqlite/jni/capi/XDestroyCallback.java

@@ -0,0 +1,37 @@
+/*
+** 2023-07-21
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file declares JNI bindings for the sqlite3 C API.
+*/
+package org.sqlite.jni.capi;
+
+/**
+   Callback for a hook called by SQLite when certain client-provided
+   state are destroyed. It gets its name from the pervasive use of
+   the symbol name xDestroy() for this purpose in the C API
+   documentation.
+*/
+public interface XDestroyCallback {
+  /**
+     Must perform any cleanup required by this object. Must not
+     throw. Must not call back into the sqlite3 API, else it might
+     invoke a deadlock.
+
+     WARNING: as a rule, it is never safe to register individual
+     instances with this interface multiple times in the
+     library. e.g., do not register the same CollationCallback with
+     multiple arities or names using sqlite3_create_collation().  If
+     this rule is violated, the library will eventually try to free
+     each individual reference, leading to memory corruption or a
+     crash via duplicate free().
+  */
+  public void xDestroy();
+}

ext/jni/src/org/sqlite/jni/capi/package-info.java

@@ -0,0 +1,89 @@
+/**
+   This package houses a JNI binding to the SQLite3 C API.
+
+   <p>The primary interfaces are in {@link
+   org.sqlite.jni.capi.CApi}.</p>
+
+   <h1>API Goals and Requirements</h1>
+
+   <ul>
+
+     <li>A 1-to-1(-ish) mapping of the C API to Java via JNI, insofar
+     as cross-language semantics allow for. A closely-related goal is
+     that <a href='https://sqlite.org/c3ref/intro.html'>the C
+     documentation</a> should be usable as-is, insofar as possible,
+     for most of the JNI binding. As a rule, undocumented symbols in
+     the Java interface behave as documented for their C API
+     counterpart. Only semantic differences and Java-specific features
+     are documented here.</li>
+
+     <li>Support Java as far back as version 8 (2014).</li>
+
+     <li>Environment-independent. Should work everywhere both Java and
+     SQLite3 do.</li>
+
+     <li>No 3rd-party dependencies beyond the JDK. That includes no
+     build-level dependencies for specific IDEs and toolchains.  We
+     welcome the addition of build files for arbitrary environments
+     insofar as they neither interfere with each other nor become a
+     maintenance burden for the sqlite developers.</li>
+
+  </ul>
+
+  <h2>Non-Goals</h2>
+
+  <ul>
+
+    <li>Creation of high-level OO wrapper APIs. Clients are free to
+    create them off of the C-style API.</li>
+
+    <li>Support for mixed-mode operation, where client code accesses
+    SQLite both via the Java-side API and the C API via their own
+    native code. In such cases, proxy functionalities (primarily
+    callback handler wrappers of all sorts) may fail because the
+    C-side use of the SQLite APIs will bypass those proxies.</li>
+
+  </ul>
+
+   <h1>State of this API</h1>
+
+   <p>As of version 3.43, this software is in "tech preview" form. We
+   tentatively plan to stamp it as stable with the 3.44 release.</p>
+
+   <h1>Threading Considerations</h1>
+
+   <p>This API is, if built with SQLITE_THREADSAFE set to 1 or 2,
+   thread-safe, insofar as the C API guarantees, with some addenda:</p>
+
+   <ul>
+
+     <li>It is not legal to use Java-facing SQLite3 resource handles
+     (sqlite3, sqlite3_stmt, etc) from multiple threads concurrently,
+     nor to use any database-specific resources concurrently in a
+     thread separate from the one the database is currently in use
+     in. i.e. do not use a sqlite3_stmt in thread #2 when thread #1 is
+     using the database which prepared that handle.
+
+     <br>Violating this will eventually corrupt the JNI-level bindings
+     between Java's and C's view of the database. This is a limitation
+     of the JNI bindings, not the lower-level library.
+     </li>
+
+     <li>It is legal to use a given handle, and database-specific
+     resources, across threads, so long as no two threads pass
+     resources owned by the same database into the library
+     concurrently.
+     </li>
+
+   </ul>
+
+   <p>Any number of threads may, of course, create and use any number
+   of database handles they wish. Care only needs to be taken when
+   those handles or their associated resources cross threads, or...</p>
+
+   <p>When built with SQLITE_THREADSAFE=0 then no threading guarantees
+   are provided and multi-threaded use of the library will provoke
+   undefined behavior.</p>
+
+*/
+package org.sqlite.jni.capi;

ext/jni/src/org/sqlite/jni/capi/sqlite3.java

@@ -11,7 +11,7 @@
 *************************************************************************
 ** This file is part of the JNI bindings for the sqlite3 C API.
 */
-package org.sqlite.jni;
+package org.sqlite.jni.capi;
 
 /**
    A wrapper for communicating C-level (sqlite3*) instances with
@@ -19,19 +19,25 @@ package org.sqlite.jni;
    simply provide a type-safe way to communicate it between Java
    and C via JNI.
 */
-public final class sqlite3 extends NativePointerHolder<sqlite3> {
+public final class sqlite3 extends NativePointerHolder<sqlite3>
+ implements AutoCloseable {
+
   // Only invoked from JNI
   private sqlite3(){}
 
   public String toString(){
-    long ptr = getNativePointer();
+    final long ptr = getNativePointer();
     if( 0==ptr ){
       return sqlite3.class.getSimpleName()+"@null";
     }
-    String fn = SQLite3Jni.sqlite3_db_filename(this, "main");
+    final String fn = CApi.sqlite3_db_filename(this, "main");
     return sqlite3.class.getSimpleName()
       +"@"+String.format("0x%08x",ptr)
       +"["+((null == fn) ? "<unnamed>" : fn)+"]"
       ;
   }
+
+  @Override public void close(){
+    CApi.sqlite3_close_v2(this.clearNativePointer());
+  }
 }

ext/jni/src/org/sqlite/jni/capi/sqlite3_backup.java

@@ -0,0 +1,31 @@
+/*
+** 2023-09-03
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file is part of the JNI bindings for the sqlite3 C API.
+*/
+package org.sqlite.jni.capi;
+
+/**
+   A wrapper for passing C-level (sqlite3_backup*) instances around in
+   Java. These wrappers do not own their associated pointer, they
+   simply provide a type-safe way to communicate it between Java and C
+   via JNI.
+*/
+public final class sqlite3_backup extends NativePointerHolder<sqlite3_backup>
+  implements AutoCloseable {
+  // Only invoked from JNI.
+  private sqlite3_backup(){}
+
+  @Override public void close(){
+    CApi.sqlite3_backup_finish(this);
+  }
+
+}

ext/jni/src/org/sqlite/jni/capi/sqlite3_blob.java

@@ -0,0 +1,31 @@
+/*
+** 2023-09-03
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file is part of the JNI bindings for the sqlite3 C API.
+*/
+package org.sqlite.jni.capi;
+
+/**
+   A wrapper for passing C-level (sqlite3_blob*) instances around in
+   Java. These wrappers do not own their associated pointer, they
+   simply provide a type-safe way to communicate it between Java and C
+   via JNI.
+*/
+public final class sqlite3_blob extends NativePointerHolder<sqlite3_blob>
+  implements AutoCloseable {
+  // Only invoked from JNI.
+  private sqlite3_blob(){}
+
+  @Override public void close(){
+    CApi.sqlite3_blob_close(this.clearNativePointer());
+  }
+
+}

ext/jni/src/org/sqlite/jni/capi/sqlite3_context.java

@@ -0,0 +1,79 @@
+/*
+** 2023-07-21
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file is part of the JNI bindings for the sqlite3 C API.
+*/
+package org.sqlite.jni.capi;
+
+/**
+   sqlite3_context instances are used in conjunction with user-defined
+   SQL functions (a.k.a. UDFs).
+*/
+public final class sqlite3_context extends NativePointerHolder<sqlite3_context> {
+  private Long aggregateContext = null;
+
+  /**
+     getAggregateContext() corresponds to C's
+     sqlite3_aggregate_context(), with a slightly different interface
+     to account for cross-language differences. It serves the same
+     purposes in a slightly different way: it provides a key which is
+     stable across invocations of a UDF's callbacks, such that all
+     calls into those callbacks can determine which "set" of those
+     calls they belong to.
+
+     <p>Note that use of this method is not a requirement for proper use
+     of this class. sqlite3_aggregate_context() can also be used.
+
+     <p>If the argument is true and the aggregate context has not yet
+     been set up, it will be initialized and fetched on demand, else it
+     won't. The intent is that xStep(), xValue(), and xInverse()
+     methods pass true and xFinal() methods pass false.
+
+     <p>This function treats numeric 0 as null, always returning null instead
+     of 0.
+
+     <p>If this object is being used in the context of an aggregate or
+     window UDF, this function returns a non-0 value which is distinct
+     for each set of UDF callbacks from a single invocation of the
+     UDF, otherwise it returns 0. The returned value is only only
+     valid within the context of execution of a single SQL statement,
+     and must not be re-used by future invocations of the UDF in
+     different SQL statements.
+
+     <p>Consider this SQL, where MYFUNC is a user-defined aggregate function:
+
+     <pre>{@code
+     SELECT MYFUNC(A), MYFUNC(B) FROM T;
+     }</pre>
+
+     <p>The xStep() and xFinal() methods of the callback need to be able
+     to differentiate between those two invocations in order to
+     perform their work properly. The value returned by
+     getAggregateContext() will be distinct for each of those
+     invocations of MYFUNC() and is intended to be used as a lookup
+     key for mapping callback invocations to whatever client-defined
+     state is needed by the UDF.
+
+     <p>There is one case where this will return null in the context
+     of an aggregate or window function: if the result set has no
+     rows, the UDF's xFinal() will be called without any other x...()
+     members having been called. In that one case, no aggregate
+     context key will have been generated. xFinal() implementations
+     need to be prepared to accept that condition as legal.
+  */
+  public synchronized Long getAggregateContext(boolean initIfNeeded){
+      if( aggregateContext==null ){
+        aggregateContext = CApi.sqlite3_aggregate_context(this, initIfNeeded);
+        if( !initIfNeeded && null==aggregateContext ) aggregateContext = 0L;
+      }
+      return (null==aggregateContext || 0!=aggregateContext) ? aggregateContext : null;
+  }
+}

ext/jni/src/org/sqlite/jni/capi/sqlite3_stmt.java

@@ -11,7 +11,7 @@
 *************************************************************************
 ** This file is part of the JNI bindings for the sqlite3 C API.
 */
-package org.sqlite.jni;
+package org.sqlite.jni.capi;
 
 /**
    A wrapper for communicating C-level (sqlite3_stmt*) instances with
@@ -19,7 +19,12 @@ package org.sqlite.jni;
    simply provide a type-safe way to communicate it between Java and C
    via JNI.
 */
-public final class sqlite3_stmt extends NativePointerHolder<sqlite3_stmt> {
+public final class sqlite3_stmt extends NativePointerHolder<sqlite3_stmt>
+  implements AutoCloseable {
   // Only invoked from JNI.
   private sqlite3_stmt(){}
+
+  @Override public void close(){
+    CApi.sqlite3_finalize(this.clearNativePointer());
+  }
 }

ext/jni/src/org/sqlite/jni/capi/sqlite3_value.java

@@ -11,7 +11,7 @@
 *************************************************************************
 ** This file is part of the JNI bindings for the sqlite3 C API.
 */
-package org.sqlite.jni;
+package org.sqlite.jni.capi;
 
 public final class sqlite3_value extends NativePointerHolder<sqlite3_value> {
   //! Invoked only from JNI.

ext/jni/src/org/sqlite/jni/fts5/Fts5.java

@@ -11,24 +11,18 @@
 *************************************************************************
 ** This file is part of the JNI bindings for the sqlite3 C API.
 */
-package org.sqlite.jni;
+package org.sqlite.jni.fts5;
 
 /**
    INCOMPLETE AND COMPLETELY UNTESTED.
 
-   A wrapper for communicating C-level (fts5_api*) instances with
+   A utility object for holding FTS5-specific types and constants
-   Java. These wrappers do not own their associated pointer, they
+   which are used by multiple FTS5 classes.
-   simply provide a type-safe way to communicate it between Java and C
-   via JNI.
 */
 public final class Fts5 {
   /* Not used */
   private Fts5(){}
 
-  //! Callback type for use with xTokenize() variants
-  public static interface xTokenizeCallback {
-    int xToken(int tFlags, byte txt[], int iStart, int iEnd);
-  }
 
   public static final int FTS5_TOKENIZE_QUERY    = 0x0001;
   public static final int FTS5_TOKENIZE_PREFIX   = 0x0002;

ext/jni/src/org/sqlite/jni/fts5/Fts5Context.java

@@ -11,7 +11,8 @@
 *************************************************************************
 ** This file is part of the JNI bindings for the sqlite3 C API.
 */
-package org.sqlite.jni;
+package org.sqlite.jni.fts5;
+import org.sqlite.jni.capi.*;
 
 /**
    A wrapper for communicating C-level (Fts5Context*) instances with

ext/jni/src/org/sqlite/jni/fts5/Fts5ExtensionApi.java

@@ -11,76 +11,87 @@
 *************************************************************************
 ** This file is part of the JNI bindings for the sqlite3 C API.
 */
-package org.sqlite.jni;
+package org.sqlite.jni.fts5;
 import java.nio.charset.StandardCharsets;
+import org.sqlite.jni.capi.*;
+import org.sqlite.jni.annotation.*;
 
 /**
-   ALMOST COMPLETELY UNTESTED.
-
-   FAR FROM COMPLETE and the feasibility of binding this to Java
-   is still undetermined. This might be removed.
 */
 public final class Fts5ExtensionApi extends NativePointerHolder<Fts5ExtensionApi> {
   //! Only called from JNI
   private Fts5ExtensionApi(){}
-  private int iVersion = 2;
+  private final int iVersion = 2;
 
   /* Callback type for used by xQueryPhrase(). */
-  public static interface xQueryPhraseCallback {
+  public static interface XQueryPhraseCallback {
-    int xCallback(Fts5ExtensionApi fapi, Fts5Context cx);
+    int call(Fts5ExtensionApi fapi, Fts5Context cx);
   }
 
   /**
      Returns the singleton instance of this class.
   */
-  public static synchronized native Fts5ExtensionApi getInstance();
+  public static native Fts5ExtensionApi getInstance();
 
-  public synchronized native int xColumnCount(@NotNull Fts5Context fcx);
+  public native int xColumnCount(@NotNull Fts5Context fcx);
-  public synchronized native int xColumnSize(@NotNull Fts5Context cx, int iCol,
+
+  public native int xColumnSize(@NotNull Fts5Context cx, int iCol,
                                 @NotNull OutputPointer.Int32 pnToken);
-  public synchronized native int xColumnText(@NotNull Fts5Context cx, int iCol,
+
+  public native int xColumnText(@NotNull Fts5Context cx, int iCol,
                                 @NotNull OutputPointer.String txt);
-  public synchronized native int xColumnTotalSize(@NotNull Fts5Context fcx, int iCol,
+
+  public native int xColumnTotalSize(@NotNull Fts5Context fcx, int iCol,
                                      @NotNull OutputPointer.Int64 pnToken);
-  public synchronized native Object xGetAuxdata(@NotNull Fts5Context cx, boolean clearIt);
+
-  public synchronized native int xInst(@NotNull Fts5Context cx, int iIdx,
+  public native Object xGetAuxdata(@NotNull Fts5Context cx, boolean clearIt);
+
+  public native int xInst(@NotNull Fts5Context cx, int iIdx,
                           @NotNull OutputPointer.Int32 piPhrase,
                           @NotNull OutputPointer.Int32 piCol,
                           @NotNull OutputPointer.Int32 piOff);
-  public synchronized native int xInstCount(@NotNull Fts5Context fcx,
+
+  public native int xInstCount(@NotNull Fts5Context fcx,
                                @NotNull OutputPointer.Int32 pnInst);
-  public synchronized native int xPhraseCount(@NotNull Fts5Context fcx);
+
-  public synchronized native int xPhraseFirst(@NotNull Fts5Context cx, int iPhrase,
+  public native int xPhraseCount(@NotNull Fts5Context fcx);
+
+  public native int xPhraseFirst(@NotNull Fts5Context cx, int iPhrase,
                                  @NotNull Fts5PhraseIter iter,
                                  @NotNull OutputPointer.Int32 iCol,
                                  @NotNull OutputPointer.Int32 iOff);
-  public synchronized native int xPhraseFirstColumn(@NotNull Fts5Context cx, int iPhrase,
+
+  public native int xPhraseFirstColumn(@NotNull Fts5Context cx, int iPhrase,
                                        @NotNull Fts5PhraseIter iter,
                                        @NotNull OutputPointer.Int32 iCol);
-  public synchronized native void xPhraseNext(@NotNull Fts5Context cx,
+  public native void xPhraseNext(@NotNull Fts5Context cx,
                                  @NotNull Fts5PhraseIter iter,
                                  @NotNull OutputPointer.Int32 iCol,
                                  @NotNull OutputPointer.Int32 iOff);
-  public synchronized native void xPhraseNextColumn(@NotNull Fts5Context cx,
+  public native void xPhraseNextColumn(@NotNull Fts5Context cx,
                                        @NotNull Fts5PhraseIter iter,
                                        @NotNull OutputPointer.Int32 iCol);
-  public synchronized native int xPhraseSize(@NotNull Fts5Context fcx, int iPhrase);
+  public native int xPhraseSize(@NotNull Fts5Context fcx, int iPhrase);
-  public synchronized native int xQueryPhrase(@NotNull Fts5Context cx, int iPhrase,
+
-                                 @NotNull xQueryPhraseCallback callback);
+  public native int xQueryPhrase(@NotNull Fts5Context cx, int iPhrase,
-  public synchronized native int xRowCount(@NotNull Fts5Context fcx,
+                                 @NotNull XQueryPhraseCallback callback);
+  public native int xRowCount(@NotNull Fts5Context fcx,
                               @NotNull OutputPointer.Int64 nRow);
-  public synchronized native long xRowid(@NotNull Fts5Context cx);
+
+  public native long xRowid(@NotNull Fts5Context cx);
   /* Note that the JNI binding lacks the C version's xDelete()
      callback argument. Instead, if pAux has an xDestroy() method, it
      is called if the FTS5 API finalizes the aux state (including if
      allocation of storage for the auxdata fails). Any reference to
      pAux held by the JNI layer will be relinquished regardless of
      whether pAux has an xDestroy() method. */
-  public synchronized native int xSetAuxdata(@NotNull Fts5Context cx, @Nullable Object pAux);
-  public synchronized native int xTokenize(@NotNull Fts5Context cx, @NotNull byte pText[],
-                              @NotNull Fts5.xTokenizeCallback callback);
 
-  public synchronized native Object xUserData(Fts5Context cx);
+  public native int xSetAuxdata(@NotNull Fts5Context cx, @Nullable Object pAux);
+
+  public native int xTokenize(@NotNull Fts5Context cx, @NotNull byte[] pText,
+                              @NotNull XTokenizeCallback callback);
+
+  public native Object xUserData(Fts5Context cx);
   //^^^ returns the pointer passed as the 3rd arg to the C-level
-  // fts5_api::xCreateFunction.
+  // fts5_api::xCreateFunction().
 }