database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2025-07-30 19:03:16 +03:00
Code Activity

Bring the extra-src branch up to date with the trunk.

Browse Source 
FossilOrigin-Name: 12ff5c5c4162951a29b638a5bc6cffa50e057c5a5e8f5e9c627af5f4ab1e4cdb
This commit is contained in:
stephan
2024-02-27 00:58:51 +00:00
parent 45cf8ad02a f79b0bdcbf
commit 32888c2b30
416 changed files with 35602 additions and 13757 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
Makefile.in
View File

@ -57,6 +57,7 @@ LIBTCL = @TCL_LIB_SPEC@
#
READLINE_FLAGS = -DHAVE_READLINE=@TARGET_HAVE_READLINE@ @TARGET_READLINE_INC@
READLINE_FLAGS += -DHAVE_EDITLINE=@TARGET_HAVE_EDITLINE@
READLINE_FLAGS += -DHAVE_LINENOISE=@TARGET_HAVE_LINENOISE@
# The library that programs using readline() must link against.
#
@ -417,6 +418,8 @@ TESTSRC = \
$(TOP)/ext/recover/sqlite3recover.c \
$(TOP)/ext/recover/dbdata.c \
$(TOP)/ext/recover/test_recover.c \
$(TOP)/ext/intck/test_intck.c \
$(TOP)/ext/intck/sqlite3intck.c \
$(TOP)/ext/rbu/test_rbu.c
# Statically linked extensions
@ -446,6 +449,7 @@ TESTSRC += \
$(TOP)/ext/misc/percentile.c \
$(TOP)/ext/misc/prefixes.c \
$(TOP)/ext/misc/qpvtab.c \
$(TOP)/ext/misc/randomjson.c \
$(TOP)/ext/misc/regexp.c \
$(TOP)/ext/misc/remember.c \
$(TOP)/ext/misc/series.c \
@ -599,6 +603,7 @@ SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB
SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
SHELL_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB
SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC
SHELL_OPT += -DSQLITE_STRICT_SUBTYPE=1
FUZZERSHELL_OPT =
FUZZCHECK_OPT += -I$(TOP)/test
FUZZCHECK_OPT += -I$(TOP)/ext/recover
@ -629,7 +634,9 @@ FUZZCHECK_OPT += \
-DSQLITE_MAX_MMAP_SIZE=0 \
-DSQLITE_OMIT_LOAD_EXTENSION \
-DSQLITE_PRINTF_PRECISION_LIMIT=1000 \
-DSQLITE_PRIVATE=""
-DSQLITE_PRIVATE="" \
-DSQLITE_STRICT_SUBTYPE=1 \
-DSQLITE_STATIC_RANDOMJSON
FUZZCHECK_SRC += $(TOP)/test/fuzzcheck.c
FUZZCHECK_SRC += $(TOP)/test/ossfuzz.c
@ -637,6 +644,7 @@ FUZZCHECK_SRC += $(TOP)/test/fuzzinvariants.c
FUZZCHECK_SRC += $(TOP)/ext/recover/dbdata.c
FUZZCHECK_SRC += $(TOP)/ext/recover/sqlite3recover.c
FUZZCHECK_SRC += $(TOP)/test/vt02.c
FUZZCHECK_SRC += $(TOP)/ext/misc/randomjson.c
DBFUZZ_OPT =
ST_OPT = -DSQLITE_OS_KV_OPTIONAL
@ -706,6 +714,24 @@ fuzzcheck$(TEXE): $(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP)
fuzzcheck-asan$(TEXE): $(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP)
$(LTLINK) -o $@ -fsanitize=address $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) sqlite3.c $(TLIBS)
fuzzcheck-ubsan$(TEXE): $(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP)
$(LTLINK) -o $@ -fsanitize=undefined $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) sqlite3.c $(TLIBS)
# Usage: FUZZDB=filename make run-fuzzcheck
#
# Where filename is a fuzzcheck database, this target builds and runs
# fuzzcheck, fuzzcheck-asan, and fuzzcheck-ubsan on that database.
#
# FUZZDB can be a glob pattern of two or more databases. Example:
#
# FUZZDB=test/fuzzdata*.db make run-fuzzcheck
#
run-fuzzcheck: fuzzcheck$(TEXE) fuzzcheck-asan$(TEXE) fuzzcheck-ubsan$(TEXE)
@if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi
./fuzzcheck$(TEXE) --spinner $(FUZZDB)
./fuzzcheck-asan$(TEXE) --spinner $(FUZZDB)
./fuzzcheck-ubsan$(TEXE) --spinner $(FUZZDB)
ossshell$(TEXE): $(TOP)/test/ossfuzz.c $(TOP)/test/ossshell.c sqlite3.c sqlite3.h
$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/ossshell.c \
$(TOP)/test/ossfuzz.c sqlite3.c $(TLIBS)
@ -765,13 +791,22 @@ mptest: mptester$(TEXE)
$(MPTEST2) --journalmode DELETE
has_tclsh84:
sh $(TOP)/tool/cktclsh.sh 8.4 $(TCLSH_CMD)
touch has_tclsh84
has_tclsh85:
sh $(TOP)/tool/cktclsh.sh 8.5 $(TCLSH_CMD)
touch has_tclsh85
# This target creates a directory named "tsrc" and fills it with
# copies of all of the C source code and header files needed to
# build on the target system. Some of the C source code and header
# files are automatically generated. This target takes care of
# all that automatic generation.
#
.target_source: $(SRC) $(TOP)/tool/vdbe-compress.tcl fts5.c
.target_source: $(SRC) $(TOP)/tool/vdbe-compress.tcl has_tclsh84 fts5.c
rm -rf tsrc
mkdir tsrc
cp -f $(SRC) tsrc
@ -781,16 +816,16 @@ mptest: mptester$(TEXE)
cp fts5.c fts5.h tsrc
touch .target_source
sqlite3.c: .target_source $(TOP)/tool/mksqlite3c.tcl src-verify
sqlite3.c: .target_source $(TOP)/tool/mksqlite3c.tcl src-verify has_tclsh84
$(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl $(AMALGAMATION_LINE_MACROS) \
$(EXTRA_SRC)
cp tsrc/sqlite3ext.h .
cp $(TOP)/ext/session/sqlite3session.h .
sqlite3r.h: sqlite3.h
sqlite3r.h: sqlite3.h has_tclsh84
$(TCLSH_CMD) $(TOP)/tool/mksqlite3h.tcl $(TOP) --enable-recover >sqlite3r.h
sqlite3r.c: sqlite3.c sqlite3r.h
sqlite3r.c: sqlite3.c sqlite3r.h has_tclsh84
cp $(TOP)/ext/recover/sqlite3recover.c tsrc/
cp $(TOP)/ext/recover/sqlite3recover.h tsrc/
cp $(TOP)/ext/recover/dbdata.c tsrc/
@ -806,7 +841,7 @@ tclsqlite3.c: sqlite3.c
echo '#endif /* USE_SYSTEM_SQLITE */' >>tclsqlite3.c
cat $(TOP)/src/tclsqlite.c >>tclsqlite3.c
sqlite3-all.c: sqlite3.c $(TOP)/tool/split-sqlite3c.tcl
sqlite3-all.c: sqlite3.c $(TOP)/tool/split-sqlite3c.tcl has_tclsh84
$(TCLSH_CMD) $(TOP)/tool/split-sqlite3c.tcl
# Rule to build the amalgamation
@ -1094,10 +1129,10 @@ tclsqlite3$(TEXE): tclsqlite-shell.lo libsqlite3.la
# Rules to build opcodes.c and opcodes.h
#
opcodes.c: opcodes.h $(TOP)/tool/mkopcodec.tcl
opcodes.c: opcodes.h $(TOP)/tool/mkopcodec.tcl has_tclsh84
$(TCLSH_CMD) $(TOP)/tool/mkopcodec.tcl opcodes.h >opcodes.c
opcodes.h: parse.h $(TOP)/src/vdbe.c $(TOP)/tool/mkopcodeh.tcl
opcodes.h: parse.h $(TOP)/src/vdbe.c $(TOP)/tool/mkopcodeh.tcl has_tclsh84
cat parse.h $(TOP)/src/vdbe.c | $(TCLSH_CMD) $(TOP)/tool/mkopcodeh.tcl >opcodes.h
# Rules to build parse.c and parse.h - the outputs of lemon.
@ -1108,10 +1143,10 @@ parse.c: $(TOP)/src/parse.y lemon$(BEXE)
cp $(TOP)/src/parse.y .
./lemon$(BEXE) $(OPT_FEATURE_FLAGS) $(OPTS) -S parse.y
sqlite3.h: $(TOP)/src/sqlite.h.in $(TOP)/manifest mksourceid$(BEXE) $(TOP)/VERSION
sqlite3.h: $(TOP)/src/sqlite.h.in $(TOP)/manifest mksourceid$(BEXE) $(TOP)/VERSION has_tclsh84
$(TCLSH_CMD) $(TOP)/tool/mksqlite3h.tcl $(TOP) >sqlite3.h
sqlite3rc.h: $(TOP)/src/sqlite3.rc $(TOP)/VERSION
sqlite3rc.h: $(TOP)/src/sqlite3.rc $(TOP)/VERSION has_tclsh84
echo '#ifndef SQLITE_RESOURCE_VERSION' >$@
echo -n '#define SQLITE_RESOURCE_VERSION ' >>$@
cat $(TOP)/VERSION | $(TCLSH_CMD) $(TOP)/tool/replace.tcl exact . , >>$@
@ -1124,6 +1159,8 @@ keywordhash.h: $(TOP)/tool/mkkeywordhash.c
# Source files that go into making shell.c
SHELL_SRC = \
$(TOP)/src/shell.c.in \
$(TOP)/ext/consio/console_io.c \
$(TOP)/ext/consio/console_io.h \
$(TOP)/ext/misc/appendvfs.c \
$(TOP)/ext/misc/completion.c \
$(TOP)/ext/misc/decimal.c \
@ -1147,7 +1184,7 @@ SHELL_SRC = \
$(TOP)/ext/recover/sqlite3recover.h \
$(TOP)/src/test_windirent.c
shell.c: $(SHELL_SRC) $(TOP)/tool/mkshellc.tcl
shell.c: $(SHELL_SRC) $(TOP)/tool/mkshellc.tcl has_tclsh84
$(TCLSH_CMD) $(TOP)/tool/mkshellc.tcl >shell.c
@ -1235,7 +1272,7 @@ fts5parse.c: $(TOP)/ext/fts5/fts5parse.y lemon$(BEXE)
fts5parse.h: fts5parse.c
fts5.c: $(FTS5_SRC)
fts5.c: $(FTS5_SRC) has_tclsh84
$(TCLSH_CMD) $(TOP)/ext/fts5/tool/mkfts5c.tcl
cp $(TOP)/ext/fts5/fts5.h .
@ -1263,13 +1300,15 @@ TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_STMTVTAB
TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_DBPAGE_VTAB
TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_BYTECODE_VTAB
TESTFIXTURE_FLAGS += -DSQLITE_CKSUMVFS_STATIC
TESTFIXTURE_FLAGS += -DSQLITE_STATIC_RANDOMJSON
TESTFIXTURE_FLAGS += -DSQLITE_STRICT_SUBTYPE=1
TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.la
TESTFIXTURE_SRC1 = sqlite3.c
TESTFIXTURE_SRC = $(TESTSRC) $(TOP)/src/tclsqlite.c
TESTFIXTURE_SRC += $(TESTFIXTURE_SRC$(USE_AMALGAMATION))
testfixture$(TEXE): $(TESTFIXTURE_SRC)
testfixture$(TEXE): has_tclsh85 $(TESTFIXTURE_SRC)
$(LTLINK) -DSQLITE_NO_SYNC=1 $(TEMP_STORE) $(TESTFIXTURE_FLAGS) \
-o $@ $(TESTFIXTURE_SRC) $(LIBTCL) $(TLIBS)
@ -1320,17 +1359,23 @@ testrunner: testfixture$(TEXE)
# Runs both fuzztest and testrunner, consecutively.
#
devtest: testfixture$(TEXE) fuzztest testrunner
devtest: srctree-check testfixture$(TEXE) fuzztest testrunner
mdevtest:
mdevtest: srctree-check has_tclsh85
$(TCLSH_CMD) $(TOP)/test/testrunner.tcl mdevtest
sdevtest:
sdevtest: has_tclsh85
$(TCLSH_CMD) $(TOP)/test/testrunner.tcl sdevtest
# Validate that various generated files in the source tree
# are up-to-date.
#
srctree-check: $(TOP)/tool/srctree-check.tcl
$(TCLSH_CMD) $(TOP)/tool/srctree-check.tcl
# Testing for a release
#
releasetest: testfixture$(TEXE)
releasetest: srctree-check testfixture$(TEXE)
./testfixture$(TEXE) $(TOP)/test/testrunner.tcl release
# Minimal testing that runs in less than 3 minutes
@ -1341,7 +1386,7 @@ quicktest: ./testfixture$(TEXE)
# This is the common case. Run many tests that do not take too long,
# including fuzzcheck, sqlite3_analyzer, and sqldiff tests.
#
test: fuzztest sourcetest $(TESTPROGS) tcltest
test: srctree-check fuzztest sourcetest $(TESTPROGS) tcltest
# Run a test using valgrind. This can take a really long time
# because valgrind is so much slower than a native machine.
@ -1359,13 +1404,13 @@ smoketest: $(TESTPROGS) fuzzcheck$(TEXE)
shelltest: $(TESTPROGS)
./testfixture$(TEXT) $(TOP)/test/permutations.test shell
sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in
sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in has_tclsh85
$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in >sqlite3_analyzer.c
sqlite3_analyzer$(TEXE): sqlite3_analyzer.c
$(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS)
sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in
sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in has_tclsh85
$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in >sqltclsh.c
sqltclsh$(TEXE): sqltclsh.c
@ -1384,7 +1429,7 @@ CHECKER_DEPS =\
$(TOP)/ext/misc/btreeinfo.c \
$(TOP)/ext/repair/sqlite3_checker.c.in
sqlite3_checker.c: $(CHECKER_DEPS)
sqlite3_checker.c: $(CHECKER_DEPS) has_tclsh85
$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/ext/repair/sqlite3_checker.c.in >$@
sqlite3_checker$(TEXE): sqlite3_checker.c
@ -1470,6 +1515,11 @@ amalgamation-tarball: sqlite3.c sqlite3rc.h
snapshot-tarball: sqlite3.c sqlite3rc.h
TOP=$(TOP) sh $(TOP)/tool/mkautoconfamal.sh --snapshot
# Build a ZIP archive containing various command-line tools.
#
tool-zip: testfixture sqlite3 sqldiff sqlite3_analyzer $(TOP)/tool/mktoolzip.tcl
./testfixture $(TOP)/tool/mktoolzip.tcl
# The next two rules are used to support the "threadtest" target. Building
# threadtest runs a few thread-safety tests that are implemented in C. This
# target is invoked by the releasetest.tcl script.
@ -1545,6 +1595,7 @@ clean:
rm -f threadtest5
rm -f src-verify
rm -f custom.rws
rm -f has_tclsh84 has_tclsh85
distclean: clean
rm -f sqlite_cfg.h config.log config.status libtool Makefile sqlite3.pc \
@ -1589,7 +1640,7 @@ fiddle: sqlite3.c shell.c
@echo 'Updating custom dictionary from tool/custom.txt'
aspell --lang=en create master ./custom.rws < $<
misspell: ./custom.rws
misspell: ./custom.rws has_tclsh84
$(TCLSH_CMD) ./tool/spellsift.tcl ./src/*.c ./src/*.h ./src/*.in
#

40
Makefile.msc
View File

@ -56,8 +56,8 @@ MINIMAL_AMALGAMATION = 0
USE_STDCALL = 0
!ENDIF
# Set this non-0 to use structured exception handling (SEH) for WAL mode
# in the core library.
# Use the USE_SEH=0 option on the nmake command line to omit structured
# exception handling (SEH) support. SEH is on by default.
#
!IFNDEF USE_SEH
USE_SEH = 1
@ -378,6 +378,7 @@ SQLITE_TCL_DEP =
!IFNDEF OPT_FEATURE_FLAGS
!IF $(MINIMAL_AMALGAMATION)==0
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS5=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_GEOPOLY=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1
@ -407,10 +408,11 @@ OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RBU=1
!ENDIF
# Should structured exception handling (SEH) be enabled for WAL mode in
# the core library?
# the core library? It is on by default. Only omit it if the
# USE_SEH=0 option is provided on the nmake command-line.
#
!IF $(USE_SEH)!=0
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_USE_SEH=1
!IF $(USE_SEH)==0
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_OMIT_SEH=1
!ENDIF
# These are the "extended" SQLite compilation options used when compiling for
@ -1586,6 +1588,7 @@ TESTEXT = \
$(TOP)\ext\misc\percentile.c \
$(TOP)\ext\misc\prefixes.c \
$(TOP)\ext\misc\qpvtab.c \
$(TOP)\ext\misc\randomjson.c \
$(TOP)\ext\misc\regexp.c \
$(TOP)\ext\misc\remember.c \
$(TOP)\ext\misc\series.c \
@ -1596,8 +1599,9 @@ TESTEXT = \
$(TOP)\ext\rtree\test_rtreedoc.c \
$(TOP)\ext\recover\sqlite3recover.c \
$(TOP)\ext\recover\test_recover.c \
$(TOP)\ext\recover\dbdata.c \
fts5.c
$(TOP)\ext\intck\test_intck.c \
$(TOP)\ext\intck\sqlite3intck.c \
$(TOP)\ext\recover\dbdata.c
# If use of zlib is enabled, add the "zipfile.c" source file.
#
@ -1613,7 +1617,8 @@ TESTSRC2 = \
$(SRC01) \
$(SRC07) \
$(SRC10) \
$(TOP)\ext\async\sqlite3async.c
$(TOP)\ext\async\sqlite3async.c \
fts5.c
# Header files used by all library source files.
#
@ -1692,6 +1697,9 @@ SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_DQS=0
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION=1
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_STMT_SCANSTATUS=1
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_STRICT_SUBTYPE=1
!ENDIF
# <<mark>>
@ -1728,6 +1736,8 @@ FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_MAX_MMAP_SIZE=0
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_OMIT_LOAD_EXTENSION
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_PRINTF_PRECISION_LIMIT=1000
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_PRIVATE=""
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_STRICT_SUBTYPE=1
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_STATIC_RANDOMJSON
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_MAX_MEMORY=50000000
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_PRINTF_PRECISION_LIMIT=1000
@ -1744,6 +1754,7 @@ FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\test\fuzzinvariants.c
FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\test\vt02.c
FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\recover\dbdata.c
FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\recover\sqlite3recover.c
FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\misc\randomjson.c
OSSSHELL_SRC = $(TOP)\test\ossshell.c $(TOP)\test\ossfuzz.c
DBFUZZ_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION
@ -1823,8 +1834,8 @@ $(SQLITE3EXE): shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLIT
/link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS)
# <<mark>>
sqldiff.exe: $(TOP)\tool\sqldiff.c $(SQLITE3C) $(SQLITE3H)
$(LTLINK) $(NO_WARN) $(TOP)\tool\sqldiff.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
sqldiff.exe: $(TOP)\tool\sqldiff.c $(TOP)\ext\consio\console_io.h $(TOP)\ext\consio\console_io.c $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS)
$(LTLINK) $(NO_WARN) -I$(TOP)\ext\consio $(TOP)\tool\sqldiff.c $(TOP)\ext\consio\console_io.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(LIBRESOBJS)
dbhash.exe: $(TOP)\tool\dbhash.c $(SQLITE3C) $(SQLITE3H)
$(LTLINK) $(NO_WARN) $(TOP)\tool\dbhash.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
@ -2261,6 +2272,8 @@ keywordhash.h: $(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe
# Source files that go into making shell.c
SHELL_SRC = \
$(TOP)\src\shell.c.in \
$(TOP)\ext\consio\console_io.c \
$(TOP)\ext\consio\console_io.h \
$(TOP)\ext\misc\appendvfs.c \
$(TOP)\ext\misc\completion.c \
$(TOP)\ext\misc\base64.c \
@ -2431,6 +2444,8 @@ TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1
TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_BYTECODE_VTAB=1
TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_CKSUMVFS_STATIC=1
TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) $(TEST_CCONV_OPTS)
TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_STATIC_RANDOMJSON
TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_STRICT_SUBTYPE=1
TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2)
TESTFIXTURE_SRC1 = $(TESTEXT) $(SQLITE3C)
@ -2470,6 +2485,9 @@ extensiontest: testfixture.exe testloadext.dll
@set PATH=$(LIBTCLPATH);$(PATH)
.\testfixture.exe $(TOP)\test\loadext.test $(TESTOPTS)
tool-zip: testfixture.exe sqlite3.exe sqldiff.exe sqlite3_analyzer.exe $(TOP)\tool\mktoolzip.tcl
.\testfixture.exe $(TOP)\tool\mktoolzip.tcl
coretestprogs: $(TESTPROGS)
testprogs: coretestprogs srcck1.exe fuzzcheck.exe sessionfuzz.exe
@ -2538,7 +2556,7 @@ smoketest: $(TESTPROGS)
shelltest: $(TESTPROGS)
.\testfixture.exe $(TOP)\test\permutations.test shell
sqlite3_analyzer.c: $(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in $(SQLITE_TCL_DEP)
sqlite3_analyzer.c: $(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in $(TOP)\ext\consio\console_io.h $(TOP)\ext\consio\console_io.c $(SQLITE_TCL_DEP)
$(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in > $@
sqlite3_analyzer.exe: sqlite3_analyzer.c $(LIBRESOBJS)

20
README.md
View File

@ -159,7 +159,7 @@ extension and only later escaped to the wild as an independent library.)
Test scripts and programs are found in the **test/** subdirectory.
Additional test code is found in other source repositories.
See [How SQLite Is Tested](http://www.sqlite.org/testing.html) for
See [How SQLite Is Tested](https://www.sqlite.org/testing.html) for
additional information.
The **ext/** subdirectory contains code for extensions. The
@ -183,7 +183,7 @@ manually-edited files and automatically-generated files.
The SQLite interface is defined by the **sqlite3.h** header file, which is
generated from src/sqlite.h.in, ./manifest.uuid, and ./VERSION. The
[Tcl script](http://www.tcl.tk) at tool/mksqlite3h.tcl does the conversion.
[Tcl script](https://www.tcl.tk) at tool/mksqlite3h.tcl does the conversion.
The manifest.uuid file contains the SHA3 hash of the particular check-in
and is used to generate the SQLITE\_SOURCE\_ID macro. The VERSION file
contains the current SQLite version number. The sqlite3.h header is really
@ -250,14 +250,14 @@ individual source file exceeds 32K lines in length.
## How It All Fits Together
SQLite is modular in design.
See the [architectural description](http://www.sqlite.org/arch.html)
See the [architectural description](https://www.sqlite.org/arch.html)
for details. Other documents that are useful in
(helping to understand how SQLite works include the
[file format](http://www.sqlite.org/fileformat2.html) description,
the [virtual machine](http://www.sqlite.org/opcode.html) that runs
[file format](https://www.sqlite.org/fileformat2.html) description,
the [virtual machine](https://www.sqlite.org/opcode.html) that runs
prepared statements, the description of
[how transactions work](http://www.sqlite.org/atomiccommit.html), and
the [overview of the query planner](http://www.sqlite.org/optoverview.html).
[how transactions work](https://www.sqlite.org/atomiccommit.html), and
the [overview of the query planner](https://www.sqlite.org/optoverview.html).
Years of effort have gone into optimizing SQLite, both
for small size and high performance. And optimizations tend to result in
@ -353,7 +353,7 @@ hidden by also modifying the makefiles.
## Contacts
The main SQLite website is [http:/sqlite.org/](http://sqlite.org/)
The main SQLite website is [https://sqlite.org/](https://sqlite.org/)
with geographically distributed backups at
[http://www2.sqlite.org/](http://www2.sqlite.org) and
[http://www3.sqlite.org/](http://www3.sqlite.org).
[https://www2.sqlite.org/](https://www2.sqlite.org) and
[https://www3.sqlite.org/](https://www3.sqlite.org).

2
VERSION
View File

@ -1 +1 @@
3.44.0
3.46.0

BIN
art/icon-243x273.gif Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 24 KiB

BIN
art/icon-80x90.gif Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 3.3 KiB

15
autoconf/Makefile.msc
View File

@ -52,8 +52,8 @@ MINIMAL_AMALGAMATION = 0
USE_STDCALL = 0
!ENDIF
# Set this non-0 to use structured exception handling (SEH) for WAL mode
# in the core library.
# Use the USE_SEH=0 option on the nmake command line to omit structured
# exception handling (SEH) support. SEH is on by default.
#
!IFNDEF USE_SEH
USE_SEH = 1
@ -296,6 +296,7 @@ SQLITE3EXEPDB = /pdb:sqlite3sh.pdb
!IFNDEF OPT_FEATURE_FLAGS
!IF $(MINIMAL_AMALGAMATION)==0
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS5=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_GEOPOLY=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1
@ -325,10 +326,11 @@ OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RBU=1
!ENDIF
# Should structured exception handling (SEH) be enabled for WAL mode in
# the core library?
# the core library? It is on by default. Only omit it if the
# USE_SEH=0 option is provided on the nmake command-line.
#
!IF $(USE_SEH)!=0
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_USE_SEH=1
!IF $(USE_SEH)==0
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_OMIT_SEH=1
!ENDIF
# These are the "extended" SQLite compilation options used when compiling for
@ -986,6 +988,9 @@ SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_DQS=0
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION=1
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_STMT_SCANSTATUS=1
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_STRICT_SUBTYPE=1
!ENDIF

2
autoconf/tea/configure.ac
View File

@ -19,7 +19,7 @@ dnl to configure the system for the local environment.
# so that we create the export library with the dll.
#-----------------------------------------------------------------------
AC_INIT([sqlite],[3.43.0])
AC_INIT([sqlite],[3.46.0])
#--------------------------------------------------------------------
# Call TEA_INIT as the first TEA_ macro to set up initial vars.

72
configure vendored
View File

@ -1,6 +1,6 @@
#! /bin/sh
# Guess values for system-dependent variables and create Makefiles.
# Generated by GNU Autoconf 2.69 for sqlite 3.44.0.
# Generated by GNU Autoconf 2.69 for sqlite 3.46.0.
#
#
# Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@ -726,8 +726,8 @@ MAKEFLAGS=
# Identity of this package.
PACKAGE_NAME='sqlite'
PACKAGE_TARNAME='sqlite'
PACKAGE_VERSION='3.44.0'
PACKAGE_STRING='sqlite 3.44.0'
PACKAGE_VERSION='3.46.0'
PACKAGE_STRING='sqlite 3.46.0'
PACKAGE_BUGREPORT=''
PACKAGE_URL=''
@ -776,6 +776,7 @@ OPT_FEATURE_FLAGS
HAVE_ZLIB
USE_AMALGAMATION
TARGET_DEBUG
TARGET_HAVE_LINENOISE
TARGET_HAVE_EDITLINE
TARGET_HAVE_READLINE
TARGET_READLINE_INC
@ -903,6 +904,7 @@ enable_editline
enable_readline
with_readline_lib
with_readline_inc
with_linenoise
enable_debug
enable_amalgamation
enable_load_extension
@ -1470,7 +1472,7 @@ if test "$ac_init_help" = "long"; then
# Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF
\`configure' configures sqlite 3.44.0 to adapt to many kinds of systems.
\`configure' configures sqlite 3.46.0 to adapt to many kinds of systems.
Usage: $0 [OPTION]... [VAR=VALUE]...
@ -1535,7 +1537,7 @@ fi
if test -n "$ac_init_help"; then
case $ac_init_help in
short | recursive ) echo "Configuration of sqlite 3.44.0:";;
short | recursive ) echo "Configuration of sqlite 3.46.0:";;
esac
cat <<\_ACEOF
@ -1587,6 +1589,7 @@ Optional Packages:
(tclConfig.sh)
--with-readline-lib specify readline library
--with-readline-inc specify readline include paths
--with-linenoise=DIR source directory for linenoise library
Some influential environment variables:
CC C compiler command
@ -1665,7 +1668,7 @@ fi
test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then
cat <<\_ACEOF
sqlite configure 3.44.0
sqlite configure 3.46.0
generated by GNU Autoconf 2.69
Copyright (C) 2012 Free Software Foundation, Inc.
@ -2084,7 +2087,7 @@ cat >config.log <<_ACEOF
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.
It was created by sqlite $as_me 3.44.0, which was
It was created by sqlite $as_me 3.46.0, which was
generated by GNU Autoconf 2.69. Invocation command line was
$ $0 $@
@ -3942,13 +3945,13 @@ if ${lt_cv_nm_interface+:} false; then :
else
lt_cv_nm_interface="BSD nm"
echo "int some_variable = 0;" > conftest.$ac_ext
(eval echo "\"\$as_me:3945: $ac_compile\"" >&5)
(eval echo "\"\$as_me:3948: $ac_compile\"" >&5)
(eval "$ac_compile" 2>conftest.err)
cat conftest.err >&5
(eval echo "\"\$as_me:3948: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
(eval echo "\"\$as_me:3951: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
(eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out)
cat conftest.err >&5
(eval echo "\"\$as_me:3951: output\"" >&5)
(eval echo "\"\$as_me:3954: output\"" >&5)
cat conftest.out >&5
if $GREP 'External.*some_variable' conftest.out > /dev/null; then
lt_cv_nm_interface="MS dumpbin"
@ -5154,7 +5157,7 @@ ia64-*-hpux*)
;;
*-*-irix6*)
# Find out which ABI we are using.
echo '#line 5157 "configure"' > conftest.$ac_ext
echo '#line 5160 "configure"' > conftest.$ac_ext
if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5
(eval $ac_compile) 2>&5
ac_status=$?
@ -6679,11 +6682,11 @@ else
-e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
-e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
-e 's:$: $lt_compiler_flag:'`
(eval echo "\"\$as_me:6682: $lt_compile\"" >&5)
(eval echo "\"\$as_me:6685: $lt_compile\"" >&5)
(eval "$lt_compile" 2>conftest.err)
ac_status=$?
cat conftest.err >&5
echo "$as_me:6686: \$? = $ac_status" >&5
echo "$as_me:6689: \$? = $ac_status" >&5
if (exit $ac_status) && test -s "$ac_outfile"; then
# The compiler can only warn and ignore the option if not recognized
# So say no if there are warnings other than the usual output.
@ -7018,11 +7021,11 @@ else
-e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
-e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
-e 's:$: $lt_compiler_flag:'`
(eval echo "\"\$as_me:7021: $lt_compile\"" >&5)
(eval echo "\"\$as_me:7024: $lt_compile\"" >&5)
(eval "$lt_compile" 2>conftest.err)
ac_status=$?
cat conftest.err >&5
echo "$as_me:7025: \$? = $ac_status" >&5
echo "$as_me:7028: \$? = $ac_status" >&5
if (exit $ac_status) && test -s "$ac_outfile"; then
# The compiler can only warn and ignore the option if not recognized
# So say no if there are warnings other than the usual output.
@ -7123,11 +7126,11 @@ else
-e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
-e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
-e 's:$: $lt_compiler_flag:'`
(eval echo "\"\$as_me:7126: $lt_compile\"" >&5)
(eval echo "\"\$as_me:7129: $lt_compile\"" >&5)
(eval "$lt_compile" 2>out/conftest.err)
ac_status=$?
cat out/conftest.err >&5
echo "$as_me:7130: \$? = $ac_status" >&5
echo "$as_me:7133: \$? = $ac_status" >&5
if (exit $ac_status) && test -s out/conftest2.$ac_objext
then
# The compiler can only warn and ignore the option if not recognized
@ -7178,11 +7181,11 @@ else
-e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
-e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
-e 's:$: $lt_compiler_flag:'`
(eval echo "\"\$as_me:7181: $lt_compile\"" >&5)
(eval echo "\"\$as_me:7184: $lt_compile\"" >&5)
(eval "$lt_compile" 2>out/conftest.err)
ac_status=$?
cat out/conftest.err >&5
echo "$as_me:7185: \$? = $ac_status" >&5
echo "$as_me:7188: \$? = $ac_status" >&5
if (exit $ac_status) && test -s out/conftest2.$ac_objext
then
# The compiler can only warn and ignore the option if not recognized
@ -9558,7 +9561,7 @@ else
lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
lt_status=$lt_dlunknown
cat > conftest.$ac_ext <<_LT_EOF
#line 9561 "configure"
#line 9564 "configure"
#include "confdefs.h"
#if HAVE_DLFCN_H
@ -9654,7 +9657,7 @@ else
lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
lt_status=$lt_dlunknown
cat > conftest.$ac_ext <<_LT_EOF
#line 9657 "configure"
#line 9660 "configure"
#include "confdefs.h"
#if HAVE_DLFCN_H
@ -11245,6 +11248,27 @@ fi
fi
fi
# Check whether --with-linenoise was given.
if test "${with_linenoise+set}" = set; then :
withval=$with_linenoise; with_linenoise=$withval
else
with_linenoise="no"
fi
if test "x$with_linenoise" != "xno"; then
TARGET_HAVE_READLINE=0
TARGET_HAVE_EDITLINE=0
TARGET_HAVE_LINENOISE=1
TARGET_READLINE_INC="-I${with_linenoise}"
TARGET_READLINE_LIBS="${with_linenoise}/linenoise.c"
echo "using linenoise source code at ${with_linenoise}"
else
TARGET_HAVE_LINENOISE=0
echo "not using linenoise"
fi
@ -11321,7 +11345,7 @@ fi
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking build type" >&5
$as_echo_n "checking build type... " >&6; }
if test "${enable_debug}" = "yes" ; then
TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0"
TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0 -Wall"
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: debug" >&5
$as_echo "debug" >&6; }
else
@ -12457,7 +12481,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# report actual input values of CONFIG_FILES etc. instead of their
# values after options handling.
ac_log="
This file was extended by sqlite $as_me 3.44.0, which was
This file was extended by sqlite $as_me 3.46.0, which was
generated by GNU Autoconf 2.69. Invocation command line was
CONFIG_FILES = $CONFIG_FILES
@ -12523,7 +12547,7 @@ _ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
ac_cs_version="\\
sqlite config.status 3.44.0
sqlite config.status 3.46.0
configured by $0, generated by GNU Autoconf 2.69,
with options \\"\$ac_cs_config\\"

21
configure.ac
View File

@ -74,7 +74,7 @@
# you don't need (for example BLT) by erasing or commenting out
# the corresponding code.
#
AC_INIT([sqlite],[m4_esyscmd(cat VERSION | tr -d '\n')])
AC_INIT([sqlite],m4_esyscmd(cat VERSION | tr -d '\n'))
dnl Make sure the local VERSION file matches this configure script
sqlite_version_sanity_check=`cat $srcdir/VERSION | tr -d '\n'`
@ -598,11 +598,28 @@ if test x"$with_readline" != xno; then
TARGET_HAVE_READLINE=1
fi
fi
AC_ARG_WITH([linenoise],
[AS_HELP_STRING([--with-linenoise=DIR],[source directory for linenoise library])],
[with_linenoise=$withval],
[with_linenoise="no"])
if test "x$with_linenoise" != "xno"; then
TARGET_HAVE_READLINE=0
TARGET_HAVE_EDITLINE=0
TARGET_HAVE_LINENOISE=1
TARGET_READLINE_INC="-I${with_linenoise}"
TARGET_READLINE_LIBS="${with_linenoise}/linenoise.c"
echo "using linenoise source code at ${with_linenoise}"
else
TARGET_HAVE_LINENOISE=0
echo "not using linenoise"
fi
AC_SUBST(TARGET_READLINE_LIBS)
AC_SUBST(TARGET_READLINE_INC)
AC_SUBST(TARGET_HAVE_READLINE)
AC_SUBST(TARGET_HAVE_EDITLINE)
AC_SUBST(TARGET_HAVE_LINENOISE)
##########
# Figure out what C libraries are required to compile programs
@ -615,7 +632,7 @@ AC_SEARCH_LIBS(fdatasync, [rt])
AC_ARG_ENABLE(debug, AS_HELP_STRING([--enable-debug],[enable debugging & verbose explain]))
AC_MSG_CHECKING([build type])
if test "${enable_debug}" = "yes" ; then
TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0"
TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0 -Wall"
AC_MSG_RESULT([debug])
else
TARGET_DEBUG="-DNDEBUG"

67
doc/compile-for-windows.md
View File

@ -1,7 +1,7 @@
# Notes On Compiling SQLite On Windows 11
Here are step-by-step instructions on how to build SQLite from
canonical source on a new Windows 11 PC, as of 2023-08-16:
canonical source on a new Windows 11 PC, as of 2023-11-01:
1. Install Microsoft Visual Studio. The free "community edition"
will work fine. Do a standard install for C++ development.
@ -16,7 +16,7 @@ canonical source on a new Windows 11 PC, as of 2023-08-16:
a 32-bit build.) The subsequent steps will not work in a vanilla
DOS prompt. Nor will they work in PowerShell.
3. Install TCL development libraries. This note assumes that you wil
3. Install TCL development libraries. This note assumes that you will
install the TCL development libraries in the "`c:\Tcl`" directory.
Make adjustments
if you want TCL installed somewhere else. SQLite needs both the
@ -83,3 +83,66 @@ following minor changes:
<ul>
<li> `set PATH=c:\tcl32\bin;%PATH%`
</ul>
## Building a DLL
The command the developers use for building the deliverable DLL on the
[download page](https://sqlite.org/download.html) is as follows:
> ~~~~
nmake /f Makefile.msc sqlite3.dll USE_NATIVE_LIBPATHS=1 "OPTS=-DSQLITE_ENABLE_FTS3=1 -DSQLITE_ENABLE_FTS4=1 -DSQLITE_ENABLE_FTS5=1 -DSQLITE_ENABLE_RTREE=1 -DSQLITE_ENABLE_JSON1=1 -DSQLITE_ENABLE_GEOPOLY=1 -DSQLITE_ENABLE_SESSION=1 -DSQLITE_ENABLE_PREUPDATE_HOOK=1 -DSQLITE_ENABLE_SERIALIZE=1 -DSQLITE_ENABLE_MATH_FUNCTIONS=1"
~~~~
That command generates both the sqlite3.dll and sqlite3.def files. The same
command works for both 32-bit and 64-bit builds.
## Statically Linking The TCL Library
Some utility programs associated with SQLite need to be linked
with TCL in order to function. The [sqlite3_analyzer.exe program](https://sqlite.org/sqlanalyze.html)
is an example. You can build as described above, and then
enter:
> ~~~~
nmake /f Makefile.msc sqlite3_analyzer.exe
~~~~
And you will end up with a working executable. However, that executable
will depend on having the "tcl86.dll" library somewhere on your %PATH%.
Use the following steps to build an executable that has the TCL library
statically linked so that it does not depend on separate DLL:
1. Use the appropriate "Command Prompt" window - either x86 or
x64, depending on whether you want a 32-bit or 64-bit executable.
2. Untar the TCL source tarball into a fresh directory. CD into
the "win/" subfolder.
3. Run: `nmake /f makefile.vc OPTS=nothreads,static shell`
4. CD into the "Release*" subfolder that is created (note the
wildcard - the full name of the directory might vary). There
you will find the "tcl86s.lib" file. Copy this file into the
same directory that you put the "tcl86.lib" on your initial
installation. (In this document, that directory is
"`C:\Tcl32\lib`" for 32-bit builds and
"`C:\Tcl\lib`" for 64-bit builds.)
5. CD into your SQLite source code directory and build the desired
utility program, but add the following extra arguments to the
nmake command line:
<blockquote><pre>
CCOPTS="-DSTATIC_BUILD" LIBTCL="tcl86s.lib netapi32.lib user32.lib"
</pre></blockquote>
<p>So, for example, to build a statically linked version of
sqlite3_analyzer.exe, you might type:
<blockquote><pre>
nmake /f Makefile.msc CCOPTS="-DSTATIC_BUILD" LIBTCL="tcl86s.lib netapi32.lib user32.lib" sqlite3_analyzer.exe
</pre></blockquote>
6. After your executable is built, you can verify that it does not
depend on the TCL DLL by running:
<blockquote><pre>
dumpbin /dependents sqlite3_analyzer.exe
</pre></blockquote>

290
doc/jsonb.md Normal file
View File

@ -0,0 +1,290 @@
# The JSONB Format
This document describes SQLite's JSONB binary encoding of
JSON.
## 1.0 What Is JSONB?
Beginning with version 3.45.0 (circa 2024-01-01), SQLite supports an
alternative binary encoding of JSON which we call "JSONB". JSONB is
a binary format that stored as a BLOB.
The advantage of JSONB over ordinary text RFC 8259 JSON is that JSONB
is both slightly smaller (by between 5% and 10% in most cases) and
can be processed in less than half the number of CPU cycles. The built-in
[JSON SQL functions] of SQLite can accept either ordinary text JSON
or the binary JSONB encoding for any of their JSON inputs.
The "JSONB" name is inspired by [PostgreSQL](https://postgresql.org), but the
on-disk format for SQLite's JSONB is not the same as PostgreSQL's.
The two formats have the same name, but they have wildly different internal
representations and are not in any way binary compatible.
The central idea behind this JSONB specification is that each element
begins with a header that includes the size and type of that element.
The header takes the place of punctuation such as double-quotes,
curly-brackes, square-brackets, commas, and colons. Since the size
and type of each element is contained in its header, the element can
be read faster since it is no longer necessary to carefully scan forward
looking for the closing delimiter. The payload of JSONB is the same
as for corresponding text JSON. The same payload bytes occur in the
same order. The only real difference between JSONB and ordinary text
JSON is that JSONB includes a binary header on
each element and omits delimiter and separator punctuation.
### 1.1 Internal Use Only
The details of the JSONB are not intended to be visible to application
developers. Application developers should look at JSONB as an opaque BLOB
used internally by SQLite. Nevertheless, we want the format to be backwards
compatible across all future versions of SQLite. To that end, the format
is documented by this file in the source tree. But this file should be
used only by SQLite core developers, not by developers of applications
that only use SQLite.
## 2.0 The Purpose Of This Document
JSONB is not intended as an external format to be used by
applications. JSONB is designed for internal use by SQLite only.
Programmers do not need to understand the JSONB format in order to
use it effectively.
Applications should access JSONB only through the [JSON SQL functions],
not by looking at individual bytes of the BLOB.
However, JSONB is intended to be portable and backwards compatible
for all future versions of SQLite. In other words, you should not have
to export and reimport your SQLite database files when you upgrade to
a newer SQLite version. For that reason, the JSONB format needs to
be well-defined.
This document is therefore similar in purpose to the
[SQLite database file format] document that describes the on-disk
format of an SQLite database file. Applications are not expected
to directly read and write the bits and bytes of SQLite database files.
The SQLite database file format is carefully documented so that it
can be stable and enduring. In the same way, the JSONB representation
of JSON is documented here so that it too can be stable and enduring,
not so that applications can read or writes individual bytes.
## 3.0 Encoding
JSONB is a direct translation of the underlying text JSON. The difference
is that JSONB uses a binary encoding that is faster to parse compared to
the detailed syntax of text JSON.
Each JSON element is encoded as a header and a payload. The header
determines type of element (string, numeric, boolean, null, object, or
array) and the size of the payload. The header can be between 1 and
9 bytes in size. The payload can be any size from zero bytes up to the
maximum allowed BLOB size.
### 3.1 Payload Size
The upper four bits of the first byte of the header determine size of the
header and possibly also the size of the payload.
If the upper four bits have a value between 0 and 11, then the header is
exactly one byte in size and the payload size is determined by those
upper four bits. If the upper four bits have a value between 12 and 15,
that means that the total header size is 2, 3, 5, or 9 bytes and the
payload size is unsigned big-endian integer that is contained in the
subsequent bytes. The size integer is the one byte that following the
initial header byte if the upper four bits
are 12, two bytes if the upper bits are 13, four bytes if the upper bits
are 14, and eight bytes if the upper bits are 15. The current design
of SQLite does not support BLOB values larger than 2GiB, so the eight-byte
variant of the payload size integer will never be used by the current code.
The eight-byte payload size integer is included in the specification
to allow for future expansion.
The header for an element does *not* need to be in its simplest
form. For example, consider the JSON numeric value "`1`".
That element can be encode in five different ways:
* `0x13 0x31`
* `0xc3 0x01 0x31`
* `0xd3 0x00 0x01 0x31`
* `0xe3 0x00 0x00 0x00 0x01 0x31`
* `0xf3 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x01 0x31`
The shortest encoding is preferred, of course, and usually happens with
primitive elements such as numbers. However the total size of an array
or object might not be known exactly when the header of the element is
first generated. It is convenient to reserve space for the largest
possible header and then go back and fill in the correct payload size
at the end. This technique can result in array or object headers that
are larger than absolutely necessary.
### 3.2 Element Type
The least-significant four bits of the first byte of the header (the first
byte masked against 0x0f) determine element type. The following codes are
used:
<ol>
<li type="0"><p><b>NULL</b> &rarr;
The element is a JSON "null". The payload size for a true JSON NULL must
must be zero. Future versions of SQLite might extend the JSONB format
with elements that have a zero element type but a non-zero size. In that
way, legacy versions of SQLite will interpret the element as a NULL
for backwards compatibility while newer versions will interpret the
element in some other way.
<li value="1"><p><b>TRUE</b> &rarr;
The element is a JSON "true". The payload size must be zero for a actual
"true" value. Elements with type 1 and a non-zero payload size are
reserved for future expansion. Legacy implementations that see an element
type of 1 with a non-zero payload size should continue to interpret that
element as "true" for compatibility.
<li value="2"><p><b>FALSE</b> &rarr;
The element is a JSON "false". The payload size must be zero for a actual
"false" value. Elements with type 2 and a non-zero payload size are
reserved for future expansion. Legacy implementations that see an element
type of 2 with a non-zero payload size should continue to interpret that
element as "false" for compatibility.
<li value="3"><p><b>INT</b> &rarr;
The element is a JSON integer value in the canonical
RFC 8259 format, without extensions. The payload is the ASCII
text representation of that numeric value.
<li value="4"><p><b>INT5</b> &rarr;
The element is a JSON integer value that is not in the
canonical format. The payload is the ASCII
text representation of that numeric value. Because the payload is in a
non-standard format, it will need to be translated when the JSONB is
converted into RFC 8259 text JSON.
<li value="5"><p><b>FLOAT</b> &rarr;
The element is a JSON floating-point value in the canonical
RFC 8259 format, without extensions. The payload is the ASCII
text representation of that numeric value.
<li value="6"><p><b>FLOAT5</b> &rarr;
The element is a JSON floating-point value that is not in the
canonical format. The payload is the ASCII
text representation of that numeric value. Because the payload is in a
non-standard format, it will need to be translated when the JSONB is
converted into RFC 8259 text JSON.
<li value="7"><p><b>TEXT</b> &rarr;
The element is a JSON string value that does not contain
any escapes nor any characters that need to be escaped for either SQL or
JSON. The payload is the UTF8 text representation of the string value.
The payload does <i>not</i> include string delimiters.
<li value="8"><p><b>TEXTJ</b> &rarr;
The element is a JSON string value that contains
RFC 8259 character escapes (such as "<tt>\n</tt>" or "<tt>\u0020</tt>").
Those escapes will need to be translated into actual UTF8 if this element
is [json_extract|extracted] into SQL.
The payload is the UTF8 text representation of the escaped string value.
The payload does <i>not</i> include string delimiters.
<li value="9"><p><b>TEXT5</b> &rarr;
The element is a JSON string value that contains
character escapes, including some character escapes that part of JSON5
and which are not found in the canonical RFC 8259 spec.
Those escapes will need to be translated into standard JSON prior to
rendering the JSON as text, or into their actual UTF8 characters if this
element is [json_extract|extracted] into SQL.
The payload is the UTF8 text representation of the escaped string value.
The payload does <i>not</i> include string delimiters.
<li value="10"><p><b>TEXTRAW</b> &rarr;
The element is a JSON string value that contains
UTF8 characters that need to be escaped if this string is rendered into
standard JSON text.
The payload does <i>not</i> include string delimiters.
<li value="11"><p><b>ARRAY</b> &rarr;
The element is a JSON array. The payload contains
JSONB elements that comprise values contained within the array.
<li value="12"><p><b>OBJECT</b> &rarr;
The element is a JSON object. The payload contains
pairs of JSONB elements that comprise entries for the JSON object.
The first element in each pair must be a string (types 7 through 10).
The second element of each pair may be any types, including nested
arrays or objects.
<li value="13"><p><b>RESERVED-13</b> &rarr;
Reserved for future expansion. Legacy implements that encounter this
element type should raise an error.
<li value="14"><p><b>RESERVED-14</b> &rarr;
Reserved for future expansion. Legacy implements that encounter this
element type should raise an error.
<li value="15"><p><b>RESERVED-15</b> &rarr;
Reserved for future expansion. Legacy implements that encounter this
element type should raise an error.
</ol>
Element types outside the range of 0 to 12 are reserved for future
expansion. The current implement raises an error if see an element type
other than those listed above. However, future versions of SQLite might
use of the three remaining element types to implement indexing or similar
optimizations, to speed up lookup against large JSON arrays and/or objects.
### 3.3 Design Rationale For Element Types
A key goal of JSONB is that it should be quick to translate
to and from text JSON and/or be constructed from SQL values.
When converting from text into JSONB, we do not want the
converter subroutine to burn CPU cycles converting elements
values into some standard format which might never be used.
Format conversion is "lazy" - it is deferred until actually
needed. This has implications for the JSONB format design:
1. Numeric values are stored as text, not a numbers. The values are
a direct copy of the text JSON values from which they are derived.
2. There are multiple element types depending on the details of value
formats. For example, INT is used for pure RFC-8259 integer
literals and INT5 exists for JSON5 extensions such as hexadecimal
notation. FLOAT is used for pure RFC-8259 floating point literals
and FLOAT5 is used for JSON5 extensions. There are four different
representations of strings, depending on where the string came from
and how special characters within the string are escaped.
A second goal of JSONB is that it should be capable of serving as the
"parse tree" for JSON when a JSON value is being processed by the
various [JSON SQL functions] built into SQLite. Before JSONB was
developed, operations such [json_replace()] and [json_patch()]
and similar worked in three stages:
1. Translate the text JSON into a internal format that is
easier to scan and edit.
2. Perform the requested operation on the JSON.
3. Translate the internal format back into text.
JSONB seeks to serve as the internal format directly - bypassing
the first and third stages of that process. Since most of the CPU
cycles are spent on the first and third stages, that suggests that
JSONB processing will be much faster than text JSON processing.
So when processing JSONB, only the second stage of the three-stage
process is required. But when processing text JSON, it is still necessary
to do stages one and three. If JSONB is to be used as the internal
binary representation, this is yet another reason to store numeric
values as text. Storing numbers as text minimizes the amount of
conversion work needed for stages one and three. This is also why
there are four different representations of text in JSONB. Different
text representations are used for text coming from different sources
(RFC-8259 JSON, JSON5, or SQL string values) and conversions only
happen if and when they are actually needed.
### 3.4 Valid JSONB BLOBs
A valid JSONB BLOB consists of a single JSON element. The element must
exactly fill the BLOB. This one element is often a JSON object or array
and those usually contain additional elements as its payload, but the
element can be a primite value such a string, number, boolean, or null.
When the built-in JSON functions are attempting to determine if a BLOB
argument is a JSONB or just a random BLOB, they look at the header of
the outer element to see that it is well-formed and that the element
completely fills the BLOB. If these conditions are met, then the BLOB
is accepted as a JSONB value.

18
doc/lemon.html
View File

@ -683,6 +683,7 @@ other than that, the order of directives in Lemon is arbitrary.</p>
<li><tt><a href='#pifdef'>%endif</a></tt>
<li><tt><a href='#extraarg'>%extra_argument</a></tt>
<li><tt><a href='#pfallback'>%fallback</a></tt>
<li><tt><a href='#reallc'>%free</a></tt>
<li><tt><a href='#pifdef'>%if</a></tt>
<li><tt><a href='#pifdef'>%ifdef</a></tt>
<li><tt><a href='#pifdef'>%ifndef</a></tt>
@ -693,6 +694,7 @@ other than that, the order of directives in Lemon is arbitrary.</p>
<li><tt><a href='#parse_accept'>%parse_accept</a></tt>
<li><tt><a href='#parse_failure'>%parse_failure</a></tt>
<li><tt><a href='#pright'>%right</a></tt>
<li><tt><a href='#reallc'>%realloc</a></tt>
<li><tt><a href='#stack_overflow'>%stack_overflow</a></tt>
<li><tt><a href='#stack_size'>%stack_size</a></tt>
<li><tt><a href='#start_symbol'>%start_symbol</a></tt>
@ -1200,6 +1202,21 @@ match any input token.</p>
the wildcard token and some other token, the other token is always used.
The wildcard token is only matched if there are no alternatives.</p>
<a id='reallc'></a>
<h4>4.4.26 The <tt>%realloc</tt> and <tt>%free</tt> directives</h4>
<p>The <tt>%realloc</tt> and <tt>%free</tt> directives defines function
that allocate and free heap memory. The signatures of these functions
should be the same as the realloc() and free() functions from the standard
C library.
<p>If both of these functions are defined
then these functions are used to allocate and free
memory for supplemental parser stack space, if the initial
parse stack space is exceeded. The initial parser stack size
is specified by either <tt>%stack_size</tt> or the
-DYYSTACKDEPTH compile-time flag.
<a id='errors'></a>
<h2>5.0 Error Processing</h2>
@ -1224,6 +1241,7 @@ to begin parsing a new file. This is what will happen at the very
first syntax error, of course, if there are no instances of the
"error" non-terminal in your grammar.</p>
<a id='history'></a>
<h2>6.0 History of Lemon</h2>

78
doc/testrunner.md
View File

@ -2,6 +2,26 @@
# The testrunner.tcl Script
<ul type=none>
<li> 1. <a href=#overview>Overview</a>
<li> 2. <a href=#binary_tests>Binary Tests</a>
<ul type=none>
<li> 2.1. <a href=#organization_tests>Organization of Tcl Tests</a>
<li> 2.2. <a href=#run_tests>Commands to Run Tests</a>
<li> 2.3. <a href=#binary_test_failures>Investigating Binary Test Failures</a>
</ul>
<li> 3. <a href=#source_code_tests>Source Tests</a>
<ul type=none>
<li> 3.1. <a href=#commands_to_run_tests>Commands to Run SQLite Tests</a>
<li> 3.2. <a href=#zipvfs_tests>Running ZipVFS Tests</a>
<li> 3.3. <a href=#source_code_test_failures>Investigating Source Code Test Failures</a>
</ul>
<li> 4. <a href=#testrunner_options>Extra testrunner.tcl Options</a>
# 4. Extra testrunner.tcl Options
<li> 5. <a href=#cpu_cores>Controlling CPU Core Utilization</a>
</ul>
<a name=overview></a>
# 1. Overview
testrunner.tcl is a Tcl script used to run multiple SQLite tests using
@ -44,6 +64,7 @@ Sometimes testrunner.tcl uses the [testfixture] binary that it is run with
to run tests (see "Binary Tests" below). Sometimes it builds testfixture and
other binaries in specific configurations to test (see "Source Tests").
<a name=binary_tests></a>
# 2. Binary Tests
The commands described in this section all run various combinations of the Tcl
@ -61,6 +82,7 @@ these tests is therefore:
The following sub-sections describe the various options that can be
passed to testrunner.tcl to test binary testfixture builds.
<a name=organization_tests></a>
## 2.1. Organization of Tcl Tests
Tcl tests are stored in files that match the pattern *\*.test*. They are
@ -91,6 +113,7 @@ Running **all** tests is to run all tests in the full test set, plus a dozen
or so permutations. The specific permutations that are run as part of "all"
are defined in file *testrunner_data.tcl*.
<a name=run_tests></a>
## 2.2. Commands to Run Tests
To run the "veryquick" test set, use either of the following:
@ -114,6 +137,12 @@ a specified pattern (e.g. all tests that start with "fts5"), either of:
./testfixture $TESTDIR/testrunner.tcl 'fts5*'
```
Strictly speaking, for a test to be run the pattern must match the script
filename, not including the directory, using the rules of Tcl's
\[string match\] command. Except that before the matching is done, any "%"
characters specified as part of the pattern are transformed to "\*".
To run "all" tests (full + permutations):
```
@ -141,6 +170,7 @@ Or, if the failure occured as part of a permutation:
TODO: An example instead of "$PERMUTATION" and $PATH\_TO\_SCRIPT?
<a name=source_code_tests></a>
# 3. Source Code Tests
The commands described in this section invoke the C compiler to build
@ -159,7 +189,8 @@ shell that supports SQLite 3.31.1 or newer via "package require sqlite3".
TODO: ./configure + Makefile.msc build systems.
## Commands to Run SQLite Tests
<a name=commands_to_run_tests></a>
## 3.1. Commands to Run SQLite Tests
The **mdevtest** command is equivalent to running the veryquick tests and
the [make fuzztest] target once for each of two --enable-all builds - one
@ -201,7 +232,18 @@ of the specific tests run.
tclsh $TESTDIR/testrunner.tcl release
```
## Running ZipVFS Tests
As with <a href=#source code tests>source code tests</a>, one or more patterns
may be appended to any of the above commands (mdevtest, sdevtest or release).
In that case only Tcl tests (no fuzz or other tests) that match the specified
pattern are run. For example, to run the just the Tcl rtree tests in all
builds and configurations supported by "release":
```
tclsh $TESTDIR/testrunner.tcl release rtree%
```
<a name=zipvfs_tests></a>
## 3.2. Running ZipVFS Tests
testrunner.tcl can build a zipvfs-enabled testfixture and use it to run
tests from the Zipvfs project with the following command:
@ -217,7 +259,8 @@ test both SQLite and Zipvfs with a single command:
tclsh $TESTDIR/testrunner.tcl --zipvfs $PATH_TO_ZIPVFS mdevtest
```
## Investigating Source Code Test Failures
<a name=source_code_test_failures></a>
## 3.3. Investigating Source Code Test Failures
Investigating a test failure that occurs during source code testing is a
two step process:
@ -244,9 +287,31 @@ target to build. This may be used either to run a [make] command test directly,
or else to build a testfixture (or testfixture.exe) binary with which to
run a Tcl test script, as <a href=#binary_test_failures>described above</a>.
<a name=testrunner_options></a>
# 4. Extra testrunner.tcl Options
The testrunner.tcl script options in this section may be used with both source
code and binary tests.
# 4. Controlling CPU Core Utilization
The **--buildonly** option instructs testrunner.tcl just to build the binaries
required by a test, not to run any actual tests. For example:
```
# Build binaries required by release test.
tclsh $TESTDIR/testrunner.tcl --buildonly release"
```
The **--dryrun** option prevents testrunner.tcl from building any binaries
or running any tests. Instead, it just writes the shell commands that it
would normally execute into the testrunner.log file. Example:
```
# Log the shell commmands that make up the mdevtest test.
tclsh $TESTDIR/testrunner.tcl --dryrun mdevtest"
```
<a name=cpu_cores></a>
# 5. Controlling CPU Core Utilization
When running either binary or source code tests, testrunner.tcl reports the
number of jobs it intends to use to stdout. e.g.
@ -277,8 +342,3 @@ testrunner.log and testrunner.db files:

691
ext/consio/console_io.c Executable file
View File

@ -0,0 +1,691 @@
/*
** 2023 November 4
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
********************************************************************************
** This file implements various interfaces used for console and stream I/O
** by the SQLite project command-line tools, as explained in console_io.h .
** Functions prefixed by "SQLITE_INTERNAL_LINKAGE" behave as described there.
*/
#ifndef SQLITE_CDECL
# define SQLITE_CDECL
#endif
#ifndef SHELL_NO_SYSINC
# include <stdarg.h>
# include <string.h>
# include <stdlib.h>
# include <limits.h>
# include <assert.h>
# include "sqlite3.h"
#endif
#ifndef HAVE_CONSOLE_IO_H
# include "console_io.h"
#endif
#if defined(_MSC_VER)
# pragma warning(disable : 4204)
#endif
#ifndef SQLITE_CIO_NO_TRANSLATE
# if (defined(_WIN32) || defined(WIN32)) && !SQLITE_OS_WINRT
# ifndef SHELL_NO_SYSINC
# include <io.h>
# include <fcntl.h>
# undef WIN32_LEAN_AND_MEAN
# define WIN32_LEAN_AND_MEAN
# include <windows.h>
# endif
# define CIO_WIN_WC_XLATE 1 /* Use WCHAR Windows APIs for console I/O */
# else
# ifndef SHELL_NO_SYSINC
# include <unistd.h>
# endif
# define CIO_WIN_WC_XLATE 0 /* Use plain C library stream I/O at console */
# endif
#else
# define CIO_WIN_WC_XLATE 0 /* Not exposing translation routines at all */
#endif
#if CIO_WIN_WC_XLATE
/* Character used to represent a known-incomplete UTF-8 char group (<28>) */
static WCHAR cBadGroup = 0xfffd;
#endif
#if CIO_WIN_WC_XLATE
static HANDLE handleOfFile(FILE *pf){
int fileDesc = _fileno(pf);
union { intptr_t osfh; HANDLE fh; } fid = {
(fileDesc>=0)? _get_osfhandle(fileDesc) : (intptr_t)INVALID_HANDLE_VALUE
};
return fid.fh;
}
#endif
#ifndef SQLITE_CIO_NO_TRANSLATE
typedef struct PerStreamTags {
# if CIO_WIN_WC_XLATE
HANDLE hx;
DWORD consMode;
char acIncomplete[4];
# else
short reachesConsole;
# endif
FILE *pf;
} PerStreamTags;
/* Define NULL-like value for things which can validly be 0. */
# define SHELL_INVALID_FILE_PTR ((FILE *)~0)
# if CIO_WIN_WC_XLATE
# define SHELL_INVALID_CONS_MODE 0xFFFF0000
# endif
# if CIO_WIN_WC_XLATE
# define PST_INITIALIZER { INVALID_HANDLE_VALUE, SHELL_INVALID_CONS_MODE, \
{0,0,0,0}, SHELL_INVALID_FILE_PTR }
# else
# define PST_INITIALIZER { 0, SHELL_INVALID_FILE_PTR }
# endif
/* Quickly say whether a known output is going to the console. */
# if CIO_WIN_WC_XLATE
static short pstReachesConsole(PerStreamTags *ppst){
return (ppst->hx != INVALID_HANDLE_VALUE);
}
# else
# define pstReachesConsole(ppst) 0
# endif
# if CIO_WIN_WC_XLATE
static void restoreConsoleArb(PerStreamTags *ppst){
if( pstReachesConsole(ppst) ) SetConsoleMode(ppst->hx, ppst->consMode);
}
# else
# define restoreConsoleArb(ppst)
# endif
/* Say whether FILE* appears to be a console, collect associated info. */
static short streamOfConsole(FILE *pf, /* out */ PerStreamTags *ppst){
# if CIO_WIN_WC_XLATE
short rv = 0;
DWORD dwCM = SHELL_INVALID_CONS_MODE;
HANDLE fh = handleOfFile(pf);
ppst->pf = pf;
if( INVALID_HANDLE_VALUE != fh ){
rv = (GetFileType(fh) == FILE_TYPE_CHAR && GetConsoleMode(fh,&dwCM));
}
ppst->hx = (rv)? fh : INVALID_HANDLE_VALUE;
ppst->consMode = dwCM;
return rv;
# else
ppst->pf = pf;
ppst->reachesConsole = ( (short)isatty(fileno(pf)) );
return ppst->reachesConsole;
# endif
}
# ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING
# define ENABLE_VIRTUAL_TERMINAL_PROCESSING (0x4)
# endif
# if CIO_WIN_WC_XLATE
/* Define console modes for use with the Windows Console API. */
# define SHELL_CONI_MODE \
(ENABLE_ECHO_INPUT | ENABLE_INSERT_MODE | ENABLE_LINE_INPUT | 0x80 \
| ENABLE_QUICK_EDIT_MODE | ENABLE_EXTENDED_FLAGS | ENABLE_PROCESSED_INPUT)
# define SHELL_CONO_MODE (ENABLE_PROCESSED_OUTPUT | ENABLE_WRAP_AT_EOL_OUTPUT \
| ENABLE_VIRTUAL_TERMINAL_PROCESSING)
# endif
typedef struct ConsoleInfo {
PerStreamTags pstSetup[3];
PerStreamTags pstDesignated[3];
StreamsAreConsole sacSetup;
} ConsoleInfo;
static short isValidStreamInfo(PerStreamTags *ppst){
return (ppst->pf != SHELL_INVALID_FILE_PTR);
}
static ConsoleInfo consoleInfo = {
{ /* pstSetup */ PST_INITIALIZER, PST_INITIALIZER, PST_INITIALIZER },
{ /* pstDesignated[] */ PST_INITIALIZER, PST_INITIALIZER, PST_INITIALIZER },
SAC_NoConsole /* sacSetup */
};
SQLITE_INTERNAL_LINKAGE FILE* invalidFileStream = (FILE *)~0;
# if CIO_WIN_WC_XLATE
static void maybeSetupAsConsole(PerStreamTags *ppst, short odir){
if( pstReachesConsole(ppst) ){
DWORD cm = odir? SHELL_CONO_MODE : SHELL_CONI_MODE;
SetConsoleMode(ppst->hx, cm);
}
}
# else
# define maybeSetupAsConsole(ppst,odir)
# endif
SQLITE_INTERNAL_LINKAGE void consoleRenewSetup(void){
# if CIO_WIN_WC_XLATE
int ix = 0;
while( ix < 6 ){
PerStreamTags *ppst = (ix<3)?
&consoleInfo.pstSetup[ix] : &consoleInfo.pstDesignated[ix-3];
maybeSetupAsConsole(ppst, (ix % 3)>0);
++ix;
}
# endif
}
SQLITE_INTERNAL_LINKAGE StreamsAreConsole
consoleClassifySetup( FILE *pfIn, FILE *pfOut, FILE *pfErr ){
StreamsAreConsole rv = SAC_NoConsole;
FILE* apf[3] = { pfIn, pfOut, pfErr };
int ix;
for( ix = 2; ix >= 0; --ix ){
PerStreamTags *ppst = &consoleInfo.pstSetup[ix];
if( streamOfConsole(apf[ix], ppst) ){
rv |= (SAC_InConsole<<ix);
}
consoleInfo.pstDesignated[ix] = *ppst;
if( ix > 0 ) fflush(apf[ix]);
}
consoleInfo.sacSetup = rv;
consoleRenewSetup();
return rv;
}
SQLITE_INTERNAL_LINKAGE void SQLITE_CDECL consoleRestore( void ){
# if CIO_WIN_WC_XLATE
static ConsoleInfo *pci = &consoleInfo;
if( pci->sacSetup ){
int ix;
for( ix=0; ix<3; ++ix ){
if( pci->sacSetup & (SAC_InConsole<<ix) ){
PerStreamTags *ppst = &pci->pstSetup[ix];
SetConsoleMode(ppst->hx, ppst->consMode);
}
}
}
# endif
}
#endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */
#ifdef SQLITE_CIO_INPUT_REDIR
/* Say whether given FILE* is among those known, via either
** consoleClassifySetup() or set{Output,Error}Stream, as
** readable, and return an associated PerStreamTags pointer
** if so. Otherwise, return 0.
*/
static PerStreamTags * isKnownReadable(FILE *pf){
static PerStreamTags *apst[] = {
&consoleInfo.pstDesignated[0], &consoleInfo.pstSetup[0], 0
};
int ix = 0;
do {
if( apst[ix]->pf == pf ) break;
} while( apst[++ix] != 0 );
return apst[ix];
}
#endif
#ifndef SQLITE_CIO_NO_TRANSLATE
/* Say whether given FILE* is among those known, via either
** consoleClassifySetup() or set{Output,Error}Stream, as
** writable, and return an associated PerStreamTags pointer
** if so. Otherwise, return 0.
*/
static PerStreamTags * isKnownWritable(FILE *pf){
static PerStreamTags *apst[] = {
&consoleInfo.pstDesignated[1], &consoleInfo.pstDesignated[2],
&consoleInfo.pstSetup[1], &consoleInfo.pstSetup[2], 0
};
int ix = 0;
do {
if( apst[ix]->pf == pf ) break;
} while( apst[++ix] != 0 );
return apst[ix];
}
static FILE *designateEmitStream(FILE *pf, unsigned chix){
FILE *rv = consoleInfo.pstDesignated[chix].pf;
if( pf == invalidFileStream ) return rv;
else{
/* Setting a possibly new output stream. */
PerStreamTags *ppst = isKnownWritable(pf);
if( ppst != 0 ){
PerStreamTags pst = *ppst;
consoleInfo.pstDesignated[chix] = pst;
}else streamOfConsole(pf, &consoleInfo.pstDesignated[chix]);
}
return rv;
}
SQLITE_INTERNAL_LINKAGE FILE *setOutputStream(FILE *pf){
return designateEmitStream(pf, 1);
}
# ifdef CONSIO_SET_ERROR_STREAM
SQLITE_INTERNAL_LINKAGE FILE *setErrorStream(FILE *pf){
return designateEmitStream(pf, 2);
}
# endif
#endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */
#ifndef SQLITE_CIO_NO_SETMODE
# if CIO_WIN_WC_XLATE
static void setModeFlushQ(FILE *pf, short bFlush, int mode){
if( bFlush ) fflush(pf);
_setmode(_fileno(pf), mode);
}
# else
# define setModeFlushQ(f, b, m) if(b) fflush(f)
# endif
SQLITE_INTERNAL_LINKAGE void setBinaryMode(FILE *pf, short bFlush){
setModeFlushQ(pf, bFlush, _O_BINARY);
}
SQLITE_INTERNAL_LINKAGE void setTextMode(FILE *pf, short bFlush){
setModeFlushQ(pf, bFlush, _O_TEXT);
}
# undef setModeFlushQ
#else /* defined(SQLITE_CIO_NO_SETMODE) */
# define setBinaryMode(f, bFlush) do{ if((bFlush)) fflush(f); }while(0)
# define setTextMode(f, bFlush) do{ if((bFlush)) fflush(f); }while(0)
#endif /* defined(SQLITE_CIO_NO_SETMODE) */
#ifndef SQLITE_CIO_NO_TRANSLATE
# if CIO_WIN_WC_XLATE
/* Write buffer cBuf as output to stream known to reach console,
** limited to ncTake char's. Return ncTake on success, else 0. */
static int conZstrEmit(PerStreamTags *ppst, const char *z, int ncTake){
int rv = 0;
if( z!=NULL ){
int nwc = MultiByteToWideChar(CP_UTF8,0, z,ncTake, 0,0);
if( nwc > 0 ){
WCHAR *zw = sqlite3_malloc64(nwc*sizeof(WCHAR));
if( zw!=NULL ){
nwc = MultiByteToWideChar(CP_UTF8,0, z,ncTake, zw,nwc);
if( nwc > 0 ){
/* Translation from UTF-8 to UTF-16, then WCHARs out. */
if( WriteConsoleW(ppst->hx, zw,nwc, 0, NULL) ){
rv = ncTake;
}
}
sqlite3_free(zw);
}
}
}
return rv;
}
/* For {f,o,e}PrintfUtf8() when stream is known to reach console. */
static int conioVmPrintf(PerStreamTags *ppst, const char *zFormat, va_list ap){
char *z = sqlite3_vmprintf(zFormat, ap);
if( z ){
int rv = conZstrEmit(ppst, z, (int)strlen(z));
sqlite3_free(z);
return rv;
}else return 0;
}
# endif /* CIO_WIN_WC_XLATE */
# ifdef CONSIO_GET_EMIT_STREAM
static PerStreamTags * getDesignatedEmitStream(FILE *pf, unsigned chix,
PerStreamTags *ppst){
PerStreamTags *rv = isKnownWritable(pf);
short isValid = (rv!=0)? isValidStreamInfo(rv) : 0;
if( rv != 0 && isValid ) return rv;
streamOfConsole(pf, ppst);
return ppst;
}
# endif
/* Get stream info, either for designated output or error stream when
** chix equals 1 or 2, or for an arbitrary stream when chix == 0.
** In either case, ppst references a caller-owned PerStreamTags
** struct which may be filled in if none of the known writable
** streams is being held by consoleInfo. The ppf parameter is a
** byref output when chix!=0 and a byref input when chix==0.
*/
static PerStreamTags *
getEmitStreamInfo(unsigned chix, PerStreamTags *ppst,
/* in/out */ FILE **ppf){
PerStreamTags *ppstTry;
FILE *pfEmit;
if( chix > 0 ){
ppstTry = &consoleInfo.pstDesignated[chix];
if( !isValidStreamInfo(ppstTry) ){
ppstTry = &consoleInfo.pstSetup[chix];
pfEmit = ppst->pf;
}else pfEmit = ppstTry->pf;
if( !isValidStreamInfo(ppstTry) ){
pfEmit = (chix > 1)? stderr : stdout;
ppstTry = ppst;
streamOfConsole(pfEmit, ppstTry);
}
*ppf = pfEmit;
}else{
ppstTry = isKnownWritable(*ppf);
if( ppstTry != 0 ) return ppstTry;
streamOfConsole(*ppf, ppst);
return ppst;
}
return ppstTry;
}
SQLITE_INTERNAL_LINKAGE int oPrintfUtf8(const char *zFormat, ...){
va_list ap;
int rv;
FILE *pfOut;
PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
# if CIO_WIN_WC_XLATE
PerStreamTags *ppst = getEmitStreamInfo(1, &pst, &pfOut);
# else
getEmitStreamInfo(1, &pst, &pfOut);
# endif
assert(zFormat!=0);
va_start(ap, zFormat);
# if CIO_WIN_WC_XLATE
if( pstReachesConsole(ppst) ){
rv = conioVmPrintf(ppst, zFormat, ap);
}else{
# endif
rv = vfprintf(pfOut, zFormat, ap);
# if CIO_WIN_WC_XLATE
}
# endif
va_end(ap);
return rv;
}
SQLITE_INTERNAL_LINKAGE int ePrintfUtf8(const char *zFormat, ...){
va_list ap;
int rv;
FILE *pfErr;
PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
# if CIO_WIN_WC_XLATE
PerStreamTags *ppst = getEmitStreamInfo(2, &pst, &pfErr);
# else
getEmitStreamInfo(2, &pst, &pfErr);
# endif
assert(zFormat!=0);
va_start(ap, zFormat);
# if CIO_WIN_WC_XLATE
if( pstReachesConsole(ppst) ){
rv = conioVmPrintf(ppst, zFormat, ap);
}else{
# endif
rv = vfprintf(pfErr, zFormat, ap);
# if CIO_WIN_WC_XLATE
}
# endif
va_end(ap);
return rv;
}
SQLITE_INTERNAL_LINKAGE int fPrintfUtf8(FILE *pfO, const char *zFormat, ...){
va_list ap;
int rv;
PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
# if CIO_WIN_WC_XLATE
PerStreamTags *ppst = getEmitStreamInfo(0, &pst, &pfO);
# else
getEmitStreamInfo(0, &pst, &pfO);
# endif
assert(zFormat!=0);
va_start(ap, zFormat);
# if CIO_WIN_WC_XLATE
if( pstReachesConsole(ppst) ){
maybeSetupAsConsole(ppst, 1);
rv = conioVmPrintf(ppst, zFormat, ap);
if( 0 == isKnownWritable(ppst->pf) ) restoreConsoleArb(ppst);
}else{
# endif
rv = vfprintf(pfO, zFormat, ap);
# if CIO_WIN_WC_XLATE
}
# endif
va_end(ap);
return rv;
}
SQLITE_INTERNAL_LINKAGE int fPutsUtf8(const char *z, FILE *pfO){
PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
# if CIO_WIN_WC_XLATE
PerStreamTags *ppst = getEmitStreamInfo(0, &pst, &pfO);
# else
getEmitStreamInfo(0, &pst, &pfO);
# endif
assert(z!=0);
# if CIO_WIN_WC_XLATE
if( pstReachesConsole(ppst) ){
int rv;
maybeSetupAsConsole(ppst, 1);
rv = conZstrEmit(ppst, z, (int)strlen(z));
if( 0 == isKnownWritable(ppst->pf) ) restoreConsoleArb(ppst);
return rv;
}else {
# endif
return (fputs(z, pfO)<0)? 0 : (int)strlen(z);
# if CIO_WIN_WC_XLATE
}
# endif
}
SQLITE_INTERNAL_LINKAGE int ePutsUtf8(const char *z){
FILE *pfErr;
PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
# if CIO_WIN_WC_XLATE
PerStreamTags *ppst = getEmitStreamInfo(2, &pst, &pfErr);
# else
getEmitStreamInfo(2, &pst, &pfErr);
# endif
assert(z!=0);
# if CIO_WIN_WC_XLATE
if( pstReachesConsole(ppst) ) return conZstrEmit(ppst, z, (int)strlen(z));
else {
# endif
return (fputs(z, pfErr)<0)? 0 : (int)strlen(z);
# if CIO_WIN_WC_XLATE
}
# endif
}
SQLITE_INTERNAL_LINKAGE int oPutsUtf8(const char *z){
FILE *pfOut;
PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
# if CIO_WIN_WC_XLATE
PerStreamTags *ppst = getEmitStreamInfo(1, &pst, &pfOut);
# else
getEmitStreamInfo(1, &pst, &pfOut);
# endif
assert(z!=0);
# if CIO_WIN_WC_XLATE
if( pstReachesConsole(ppst) ) return conZstrEmit(ppst, z, (int)strlen(z));
else {
# endif
return (fputs(z, pfOut)<0)? 0 : (int)strlen(z);
# if CIO_WIN_WC_XLATE
}
# endif
}
#endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */
#if !(defined(SQLITE_CIO_NO_UTF8SCAN) && defined(SQLITE_CIO_NO_TRANSLATE))
/* Skip over as much z[] input char sequence as is valid UTF-8,
** limited per nAccept char's or whole characters and containing
** no char cn such that ((1<<cn) & ccm)!=0. On return, the
** sequence z:return (inclusive:exclusive) is validated UTF-8.
** Limit: nAccept>=0 => char count, nAccept<0 => character
*/
SQLITE_INTERNAL_LINKAGE const char*
zSkipValidUtf8(const char *z, int nAccept, long ccm){
int ng = (nAccept<0)? -nAccept : 0;
const char *pcLimit = (nAccept>=0)? z+nAccept : 0;
assert(z!=0);
while( (pcLimit)? (z<pcLimit) : (ng-- != 0) ){
char c = *z;
if( (c & 0x80) == 0 ){
if( ccm != 0L && c < 0x20 && ((1L<<c) & ccm) != 0 ) return z;
++z; /* ASCII */
}else if( (c & 0xC0) != 0xC0 ) return z; /* not a lead byte */
else{
const char *zt = z+1; /* Got lead byte, look at trail bytes.*/
do{
if( pcLimit && zt >= pcLimit ) return z;
else{
char ct = *zt++;
if( ct==0 || (zt-z)>4 || (ct & 0xC0)!=0x80 ){
/* Trailing bytes are too few, too many, or invalid. */
return z;
}
}
} while( ((c <<= 1) & 0x40) == 0x40 ); /* Eat lead byte's count. */
z = zt;
}
}
return z;
}
#endif /*!(defined(SQLITE_CIO_NO_UTF8SCAN)&&defined(SQLITE_CIO_NO_TRANSLATE))*/
#ifndef SQLITE_CIO_NO_TRANSLATE
# ifdef CONSIO_SPUTB
SQLITE_INTERNAL_LINKAGE int
fPutbUtf8(FILE *pfO, const char *cBuf, int nAccept){
assert(pfO!=0);
# if CIO_WIN_WC_XLATE
PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
PerStreamTags *ppst = getEmitStreamInfo(0, &pst, &pfO);
if( pstReachesConsole(ppst) ){
int rv;
maybeSetupAsConsole(ppst, 1);
rv = conZstrEmit(ppst, cBuf, nAccept);
if( 0 == isKnownWritable(ppst->pf) ) restoreConsoleArb(ppst);
return rv;
}else {
# endif
return (int)fwrite(cBuf, 1, nAccept, pfO);
# if CIO_WIN_WC_XLATE
}
# endif
}
# endif
SQLITE_INTERNAL_LINKAGE int
oPutbUtf8(const char *cBuf, int nAccept){
FILE *pfOut;
PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
# if CIO_WIN_WC_XLATE
PerStreamTags *ppst = getEmitStreamInfo(1, &pst, &pfOut);
# else
getEmitStreamInfo(1, &pst, &pfOut);
# endif
# if CIO_WIN_WC_XLATE
if( pstReachesConsole(ppst) ){
return conZstrEmit(ppst, cBuf, nAccept);
}else {
# endif
return (int)fwrite(cBuf, 1, nAccept, pfOut);
# if CIO_WIN_WC_XLATE
}
# endif
}
# ifdef CONSIO_EPUTB
SQLITE_INTERNAL_LINKAGE int
ePutbUtf8(const char *cBuf, int nAccept){
FILE *pfErr;
PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
PerStreamTags *ppst = getEmitStreamInfo(2, &pst, &pfErr);
# if CIO_WIN_WC_XLATE
if( pstReachesConsole(ppst) ){
return conZstrEmit(ppst, cBuf, nAccept);
}else {
# endif
return (int)fwrite(cBuf, 1, nAccept, pfErr);
# if CIO_WIN_WC_XLATE
}
# endif
}
# endif /* defined(CONSIO_EPUTB) */
SQLITE_INTERNAL_LINKAGE char* fGetsUtf8(char *cBuf, int ncMax, FILE *pfIn){
if( pfIn==0 ) pfIn = stdin;
# if CIO_WIN_WC_XLATE
if( pfIn == consoleInfo.pstSetup[0].pf
&& (consoleInfo.sacSetup & SAC_InConsole)!=0 ){
# if CIO_WIN_WC_XLATE==1
# define SHELL_GULP 150 /* Count of WCHARS to be gulped at a time */
WCHAR wcBuf[SHELL_GULP+1];
int lend = 0, noc = 0;
if( ncMax > 0 ) cBuf[0] = 0;
while( noc < ncMax-8-1 && !lend ){
/* There is room for at least 2 more characters and a 0-terminator. */
int na = (ncMax > SHELL_GULP*4+1 + noc)? SHELL_GULP : (ncMax-1 - noc)/4;
# undef SHELL_GULP
DWORD nbr = 0;
BOOL bRC = ReadConsoleW(consoleInfo.pstSetup[0].hx, wcBuf, na, &nbr, 0);
if( bRC && nbr>0 && (wcBuf[nbr-1]&0xF800)==0xD800 ){
/* Last WHAR read is first of a UTF-16 surrogate pair. Grab its mate. */
DWORD nbrx;
bRC &= ReadConsoleW(consoleInfo.pstSetup[0].hx, wcBuf+nbr, 1, &nbrx, 0);
if( bRC ) nbr += nbrx;
}
if( !bRC || (noc==0 && nbr==0) ) return 0;
if( nbr > 0 ){
int nmb = WideCharToMultiByte(CP_UTF8, 0, wcBuf,nbr,0,0,0,0);
if( nmb != 0 && noc+nmb <= ncMax ){
int iseg = noc;
nmb = WideCharToMultiByte(CP_UTF8, 0, wcBuf,nbr,cBuf+noc,nmb,0,0);
noc += nmb;
/* Fixup line-ends as coded by Windows for CR (or "Enter".)
** This is done without regard for any setMode{Text,Binary}()
** call that might have been done on the interactive input.
*/
if( noc > 0 ){
if( cBuf[noc-1]=='\n' ){
lend = 1;
if( noc > 1 && cBuf[noc-2]=='\r' ) cBuf[--noc-1] = '\n';
}
}
/* Check for ^Z (anywhere in line) too, to act as EOF. */
while( iseg < noc ){
if( cBuf[iseg]=='\x1a' ){
noc = iseg; /* Chop ^Z and anything following. */
lend = 1; /* Counts as end of line too. */
break;
}
++iseg;
}
}else break; /* Drop apparent garbage in. (Could assert.) */
}else break;
}
/* If got nothing, (after ^Z chop), must be at end-of-file. */
if( noc > 0 ){
cBuf[noc] = 0;
return cBuf;
}else return 0;
# endif
}else{
# endif
return fgets(cBuf, ncMax, pfIn);
# if CIO_WIN_WC_XLATE
}
# endif
}
#endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */
#if defined(_MSC_VER)
# pragma warning(default : 4204)
#endif
#undef SHELL_INVALID_FILE_PTR

280
ext/consio/console_io.h Normal file
View File

@ -0,0 +1,280 @@
/*
** 2023 November 1
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
********************************************************************************
** This file exposes various interfaces used for console and other I/O
** by the SQLite project command-line tools. These interfaces are used
** at either source conglomeration time, compilation time, or run time.
** This source provides for either inclusion into conglomerated,
** "single-source" forms or separate compilation then linking.
**
** Platform dependencies are "hidden" here by various stratagems so
** that, provided certain conditions are met, the programs using this
** source or object code compiled from it need no explicit conditional
** compilation in their source for their console and stream I/O.
**
** The symbols and functionality exposed here are not a public API.
** This code may change in tandem with other project code as needed.
**
** When this .h file and its companion .c are directly incorporated into
** a source conglomeration (such as shell.c), the preprocessor symbol
** CIO_WIN_WC_XLATE is defined as 0 or 1, reflecting whether console I/O
** translation for Windows is effected for the build.
*/
#define HAVE_CONSOLE_IO_H 1
#ifndef SQLITE_INTERNAL_LINKAGE
# define SQLITE_INTERNAL_LINKAGE extern /* external to translation unit */
# include <stdio.h>
#else
# define SHELL_NO_SYSINC /* Better yet, modify mkshellc.tcl for this. */
#endif
#ifndef SQLITE3_H
# include "sqlite3.h"
#endif
#ifndef SQLITE_CIO_NO_CLASSIFY
/* Define enum for use with following function. */
typedef enum StreamsAreConsole {
SAC_NoConsole = 0,
SAC_InConsole = 1, SAC_OutConsole = 2, SAC_ErrConsole = 4,
SAC_AnyConsole = 0x7
} StreamsAreConsole;
/*
** Classify the three standard I/O streams according to whether
** they are connected to a console attached to the process.
**
** Returns the bit-wise OR of SAC_{In,Out,Err}Console values,
** or SAC_NoConsole if none of the streams reaches a console.
**
** This function should be called before any I/O is done with
** the given streams. As a side-effect, the given inputs are
** recorded so that later I/O operations on them may be done
** differently than the C library FILE* I/O would be done,
** iff the stream is used for the I/O functions that follow,
** and to support the ones that use an implicit stream.
**
** On some platforms, stream or console mode alteration (aka
** "Setup") may be made which is undone by consoleRestore().
*/
SQLITE_INTERNAL_LINKAGE StreamsAreConsole
consoleClassifySetup( FILE *pfIn, FILE *pfOut, FILE *pfErr );
/* A usual call for convenience: */
#define SQLITE_STD_CONSOLE_INIT() consoleClassifySetup(stdin,stdout,stderr)
/*
** After an initial call to consoleClassifySetup(...), renew
** the same setup it effected. (A call not after is an error.)
** This will restore state altered by consoleRestore();
**
** Applications which run an inferior (child) process which
** inherits the same I/O streams may call this function after
** such a process exits to guard against console mode changes.
*/
SQLITE_INTERNAL_LINKAGE void consoleRenewSetup(void);
/*
** Undo any side-effects left by consoleClassifySetup(...).
**
** This should be called after consoleClassifySetup() and
** before the process terminates normally. It is suitable
** for use with the atexit() C library procedure. After
** this call, no console I/O should be done until one of
** console{Classify or Renew}Setup(...) is called again.
**
** Applications which run an inferior (child) process that
** inherits the same I/O streams might call this procedure
** before so that said process will have a console setup
** however users have configured it or come to expect.
*/
SQLITE_INTERNAL_LINKAGE void SQLITE_CDECL consoleRestore( void );
#else /* defined(SQLITE_CIO_NO_CLASSIFY) */
# define consoleClassifySetup(i,o,e)
# define consoleRenewSetup()
# define consoleRestore()
#endif /* defined(SQLITE_CIO_NO_CLASSIFY) */
#ifndef SQLITE_CIO_NO_REDIRECT
/*
** Set stream to be used for the functions below which write
** to "the designated X stream", where X is Output or Error.
** Returns the previous value.
**
** Alternatively, pass the special value, invalidFileStream,
** to get the designated stream value without setting it.
**
** Before the designated streams are set, they default to
** those passed to consoleClassifySetup(...), and before
** that is called they default to stdout and stderr.
**
** It is error to close a stream so designated, then, without
** designating another, use the corresponding {o,e}Emit(...).
*/
SQLITE_INTERNAL_LINKAGE FILE *invalidFileStream;
SQLITE_INTERNAL_LINKAGE FILE *setOutputStream(FILE *pf);
# ifdef CONSIO_SET_ERROR_STREAM
SQLITE_INTERNAL_LINKAGE FILE *setErrorStream(FILE *pf);
# endif
#else
# define setOutputStream(pf)
# define setErrorStream(pf)
#endif /* !defined(SQLITE_CIO_NO_REDIRECT) */
#ifndef SQLITE_CIO_NO_TRANSLATE
/*
** Emit output like fprintf(). If the output is going to the
** console and translation from UTF-8 is necessary, perform
** the needed translation. Otherwise, write formatted output
** to the provided stream almost as-is, possibly with newline
** translation as specified by set{Binary,Text}Mode().
*/
SQLITE_INTERNAL_LINKAGE int fPrintfUtf8(FILE *pfO, const char *zFormat, ...);
/* Like fPrintfUtf8 except stream is always the designated output. */
SQLITE_INTERNAL_LINKAGE int oPrintfUtf8(const char *zFormat, ...);
/* Like fPrintfUtf8 except stream is always the designated error. */
SQLITE_INTERNAL_LINKAGE int ePrintfUtf8(const char *zFormat, ...);
/*
** Emit output like fputs(). If the output is going to the
** console and translation from UTF-8 is necessary, perform
** the needed translation. Otherwise, write given text to the
** provided stream almost as-is, possibly with newline
** translation as specified by set{Binary,Text}Mode().
*/
SQLITE_INTERNAL_LINKAGE int fPutsUtf8(const char *z, FILE *pfO);
/* Like fPutsUtf8 except stream is always the designated output. */
SQLITE_INTERNAL_LINKAGE int oPutsUtf8(const char *z);
/* Like fPutsUtf8 except stream is always the designated error. */
SQLITE_INTERNAL_LINKAGE int ePutsUtf8(const char *z);
/*
** Emit output like fPutsUtf8(), except that the length of the
** accepted char or character sequence is limited by nAccept.
**
** Returns the number of accepted char values.
*/
#ifdef CONSIO_SPUTB
SQLITE_INTERNAL_LINKAGE int
fPutbUtf8(FILE *pfOut, const char *cBuf, int nAccept);
/* Like fPutbUtf8 except stream is always the designated output. */
#endif
SQLITE_INTERNAL_LINKAGE int
oPutbUtf8(const char *cBuf, int nAccept);
/* Like fPutbUtf8 except stream is always the designated error. */
#ifdef CONSIO_EPUTB
SQLITE_INTERNAL_LINKAGE int
ePutbUtf8(const char *cBuf, int nAccept);
#endif
/*
** Collect input like fgets(...) with special provisions for input
** from the console on platforms that require same. Defers to the
** C library fgets() when input is not from the console. Newline
** translation may be done as set by set{Binary,Text}Mode(). As a
** convenience, pfIn==NULL is treated as stdin.
*/
SQLITE_INTERNAL_LINKAGE char* fGetsUtf8(char *cBuf, int ncMax, FILE *pfIn);
/* Like fGetsUtf8 except stream is always the designated input. */
/* SQLITE_INTERNAL_LINKAGE char* iGetsUtf8(char *cBuf, int ncMax); */
#endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */
#ifndef SQLITE_CIO_NO_SETMODE
/*
** Set given stream for binary mode, where newline translation is
** not done, or for text mode where, for some platforms, newlines
** are translated to the platform's conventional char sequence.
** If bFlush true, flush the stream.
**
** An additional side-effect is that if the stream is one passed
** to consoleClassifySetup() as an output, it is flushed first.
**
** Note that binary/text mode has no effect on console I/O
** translation. On all platforms, newline to the console starts
** a new line and CR,LF chars from the console become a newline.
*/
SQLITE_INTERNAL_LINKAGE void setBinaryMode(FILE *, short bFlush);
SQLITE_INTERNAL_LINKAGE void setTextMode(FILE *, short bFlush);
#endif
#ifdef SQLITE_CIO_PROMPTED_IN
typedef struct Prompts {
int numPrompts;
const char **azPrompts;
} Prompts;
/*
** Macros for use of a line editor.
**
** The following macros define operations involving use of a
** line-editing library or simple console interaction.
** A "T" argument is a text (char *) buffer or filename.
** A "N" argument is an integer.
**
** SHELL_ADD_HISTORY(T) // Record text as line(s) of history.
** SHELL_READ_HISTORY(T) // Read history from file named by T.
** SHELL_WRITE_HISTORY(T) // Write history to file named by T.
** SHELL_STIFLE_HISTORY(N) // Limit history to N entries.
**
** A console program which does interactive console input is
** expected to call:
** SHELL_READ_HISTORY(T) before collecting such input;
** SHELL_ADD_HISTORY(T) as record-worthy input is taken;
** SHELL_STIFLE_HISTORY(N) after console input ceases; then
** SHELL_WRITE_HISTORY(T) before the program exits.
*/
/*
** Retrieve a single line of input text from an input stream.
**
** If pfIn is the input stream passed to consoleClassifySetup(),
** and azPrompt is not NULL, then a prompt is issued before the
** line is collected, as selected by the isContinuation flag.
** Array azPrompt[{0,1}] holds the {main,continuation} prompt.
**
** If zBufPrior is not NULL then it is a buffer from a prior
** call to this routine that can be reused, or will be freed.
**
** The result is stored in space obtained from malloc() and
** must either be freed by the caller or else passed back to
** this function as zBufPrior for reuse.
**
** This function may call upon services of a line-editing
** library to interactively collect line edited input.
*/
SQLITE_INTERNAL_LINKAGE char *
shellGetLine(FILE *pfIn, char *zBufPrior, int nLen,
short isContinuation, Prompts azPrompt);
#endif /* defined(SQLITE_CIO_PROMPTED_IN) */
/*
** TBD: Define an interface for application(s) to generate
** completion candidates for use by the line-editor.
**
** This may be premature; the CLI is the only application
** that does this. Yet, getting line-editing melded into
** console I/O is desirable because a line-editing library
** may have to establish console operating mode, possibly
** in a way that interferes with the above functionality.
*/
#if !(defined(SQLITE_CIO_NO_UTF8SCAN)&&defined(SQLITE_CIO_NO_TRANSLATE))
/* Skip over as much z[] input char sequence as is valid UTF-8,
** limited per nAccept char's or whole characters and containing
** no char cn such that ((1<<cn) & ccm)!=0. On return, the
** sequence z:return (inclusive:exclusive) is validated UTF-8.
** Limit: nAccept>=0 => char count, nAccept<0 => character
*/
SQLITE_INTERNAL_LINKAGE const char*
zSkipValidUtf8(const char *z, int nAccept, long ccm);
#endif

19
ext/expert/expert1.test
View File

@ -464,4 +464,23 @@ do_execsql_test 5.3 {
t2 t2_idx_0001295b {100 20 5}
}
if 0 {
do_test expert1-6.0 {
catchcmd :memory: {
.expert
select base64('');
.expert
select name from pragma_collation_list order by name collate uint;
}
} {0 {(no new indexes)
SCAN CONSTANT ROW
(no new indexes)
SCAN pragma_collation_list VIRTUAL TABLE INDEX 0:
USE TEMP B-TREE FOR ORDER BY
}}
}
finish_test

101
ext/expert/sqlite3expert.c
View File

@ -662,6 +662,7 @@ static int idxRegisterVtab(sqlite3expert *p){
0, /* xRelease */
0, /* xRollbackTo */
0, /* xShadowName */
0, /* xIntegrity */
};
return sqlite3_create_module(p->dbv, "expert", &expertModule, (void*)p);
@ -1818,6 +1819,88 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){
return rc;
}
/*
** Define and possibly pretend to use a useless collation sequence.
** This pretense allows expert to accept SQL using custom collations.
*/
int dummyCompare(void *up1, int up2, const void *up3, int up4, const void *up5){
(void)up1;
(void)up2;
(void)up3;
(void)up4;
(void)up5;
assert(0); /* VDBE should never be run. */
return 0;
}
/* And a callback to register above upon actual need */
void useDummyCS(void *up1, sqlite3 *db, int etr, const char *zName){
(void)up1;
sqlite3_create_collation_v2(db, zName, etr, 0, dummyCompare, 0);
}
#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) \
&& !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS)
/*
** dummy functions for no-op implementation of UDFs during expert's work
*/
void dummyUDF(sqlite3_context *up1, int up2, sqlite3_value **up3){
(void)up1;
(void)up2;
(void)up3;
assert(0); /* VDBE should never be run. */
}
void dummyUDFvalue(sqlite3_context *up1){
(void)up1;
assert(0); /* VDBE should never be run. */
}
/*
** Register UDFs from user database with another.
*/
int registerUDFs(sqlite3 *dbSrc, sqlite3 *dbDst){
sqlite3_stmt *pStmt;
int rc = sqlite3_prepare_v2(dbSrc,
"SELECT name,type,enc,narg,flags "
"FROM pragma_function_list() "
"WHERE builtin==0", -1, &pStmt, 0);
if( rc==SQLITE_OK ){
while( SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){
int nargs = sqlite3_column_int(pStmt,3);
int flags = sqlite3_column_int(pStmt,4);
const char *name = (char*)sqlite3_column_text(pStmt,0);
const char *type = (char*)sqlite3_column_text(pStmt,1);
const char *enc = (char*)sqlite3_column_text(pStmt,2);
if( name==0 || type==0 || enc==0 ){
/* no-op. Only happens on OOM */
}else{
int ienc = SQLITE_UTF8;
int rcf = SQLITE_ERROR;
if( strcmp(enc,"utf16le")==0 ) ienc = SQLITE_UTF16LE;
else if( strcmp(enc,"utf16be")==0 ) ienc = SQLITE_UTF16BE;
ienc |= (flags & (SQLITE_DETERMINISTIC|SQLITE_DIRECTONLY));
if( strcmp(type,"w")==0 ){
rcf = sqlite3_create_window_function(dbDst,name,nargs,ienc,0,
dummyUDF,dummyUDFvalue,0,0,0);
}else if( strcmp(type,"a")==0 ){
rcf = sqlite3_create_function(dbDst,name,nargs,ienc,0,
0,dummyUDF,dummyUDFvalue);
}else if( strcmp(type,"s")==0 ){
rcf = sqlite3_create_function(dbDst,name,nargs,ienc,0,
dummyUDF,0,0);
}
if( rcf!=SQLITE_OK ){
rc = rcf;
break;
}
}
}
sqlite3_finalize(pStmt);
if( rc==SQLITE_DONE ) rc = SQLITE_OK;
}
return rc;
}
#endif
/*
** Allocate a new sqlite3expert object.
*/
@ -1845,6 +1928,20 @@ sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){
}
}
/* Allow custom collations to be dealt with through prepare. */
if( rc==SQLITE_OK ) rc = sqlite3_collation_needed(pNew->dbm,0,useDummyCS);
if( rc==SQLITE_OK ) rc = sqlite3_collation_needed(pNew->dbv,0,useDummyCS);
#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) \
&& !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS)
/* Register UDFs from database [db] with [dbm] and [dbv]. */
if( rc==SQLITE_OK ){
rc = registerUDFs(pNew->db, pNew->dbm);
}
if( rc==SQLITE_OK ){
rc = registerUDFs(pNew->db, pNew->dbv);
}
#endif
/* Copy the entire schema of database [db] into [dbm]. */
if( rc==SQLITE_OK ){
@ -1920,6 +2017,10 @@ int sqlite3_expert_sql(
while( rc==SQLITE_OK && zStmt && zStmt[0] ){
sqlite3_stmt *pStmt = 0;
/* Ensure that the provided statement compiles against user's DB. */
rc = idxPrepareStmt(p->db, &pStmt, pzErr, zStmt);
if( rc!=SQLITE_OK ) break;
sqlite3_finalize(pStmt);
rc = sqlite3_prepare_v2(p->dbv, zStmt, -1, &pStmt, &zStmt);
if( rc==SQLITE_OK ){
if( pStmt ){

91
ext/fts3/fts3.c
View File

@ -640,6 +640,7 @@ static void fts3DeclareVtab(int *pRc, Fts3Table *p){
zLanguageid = (p->zLanguageid ? p->zLanguageid : "__langid");
sqlite3_vtab_config(p->db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1);
sqlite3_vtab_config(p->db, SQLITE_VTAB_INNOCUOUS);
/* Create a list of user columns for the virtual table */
zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]);
@ -3889,6 +3890,8 @@ static int fts3RenameMethod(
rc = sqlite3Fts3PendingTermsFlush(p);
}
p->bIgnoreSavepoint = 1;
if( p->zContentTbl==0 ){
fts3DbExec(&rc, db,
"ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';",
@ -3916,6 +3919,8 @@ static int fts3RenameMethod(
"ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';",
p->zDb, p->zName, zName
);
p->bIgnoreSavepoint = 0;
return rc;
}
@ -3926,12 +3931,28 @@ static int fts3RenameMethod(
*/
static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
int rc = SQLITE_OK;
UNUSED_PARAMETER(iSavepoint);
assert( ((Fts3Table *)pVtab)->inTransaction );
assert( ((Fts3Table *)pVtab)->mxSavepoint <= iSavepoint );
TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint );
if( ((Fts3Table *)pVtab)->bIgnoreSavepoint==0 ){
rc = fts3SyncMethod(pVtab);
Fts3Table *pTab = (Fts3Table*)pVtab;
assert( pTab->inTransaction );
assert( pTab->mxSavepoint<=iSavepoint );
TESTONLY( pTab->mxSavepoint = iSavepoint );
if( pTab->bIgnoreSavepoint==0 ){
if( fts3HashCount(&pTab->aIndex[0].hPending)>0 ){
char *zSql = sqlite3_mprintf("INSERT INTO %Q.%Q(%Q) VALUES('flush')",
pTab->zDb, pTab->zName, pTab->zName
);
if( zSql ){
pTab->bIgnoreSavepoint = 1;
rc = sqlite3_exec(pTab->db, zSql, 0, 0, 0);
pTab->bIgnoreSavepoint = 0;
sqlite3_free(zSql);
}else{
rc = SQLITE_NOMEM;
}
}
if( rc==SQLITE_OK ){
pTab->iSavepoint = iSavepoint+1;
}
}
return rc;
}
@ -3942,12 +3963,11 @@ static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
** This is a no-op.
*/
static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
TESTONLY( Fts3Table *p = (Fts3Table*)pVtab );
UNUSED_PARAMETER(iSavepoint);
UNUSED_PARAMETER(pVtab);
assert( p->inTransaction );
assert( p->mxSavepoint >= iSavepoint );
TESTONLY( p->mxSavepoint = iSavepoint-1 );
Fts3Table *pTab = (Fts3Table*)pVtab;
assert( pTab->inTransaction );
assert( pTab->mxSavepoint >= iSavepoint );
TESTONLY( pTab->mxSavepoint = iSavepoint-1 );
pTab->iSavepoint = iSavepoint;
return SQLITE_OK;
}
@ -3957,11 +3977,13 @@ static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
** Discard the contents of the pending terms table.
*/
static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){
Fts3Table *p = (Fts3Table*)pVtab;
Fts3Table *pTab = (Fts3Table*)pVtab;
UNUSED_PARAMETER(iSavepoint);
assert( p->inTransaction );
TESTONLY( p->mxSavepoint = iSavepoint );
sqlite3Fts3PendingTermsClear(p);
assert( pTab->inTransaction );
TESTONLY( pTab->mxSavepoint = iSavepoint );
if( (iSavepoint+1)<=pTab->iSavepoint ){
sqlite3Fts3PendingTermsClear(pTab);
}
return SQLITE_OK;
}
@ -3980,8 +4002,42 @@ static int fts3ShadowName(const char *zName){
return 0;
}
/*
** Implementation of the xIntegrity() method on the FTS3/FTS4 virtual
** table.
*/
static int fts3IntegrityMethod(
sqlite3_vtab *pVtab, /* The virtual table to be checked */
const char *zSchema, /* Name of schema in which pVtab lives */
const char *zTabname, /* Name of the pVTab table */
int isQuick, /* True if this is a quick_check */
char **pzErr /* Write error message here */
){
Fts3Table *p = (Fts3Table*)pVtab;
int rc = SQLITE_OK;
int bOk = 0;
UNUSED_PARAMETER(isQuick);
rc = sqlite3Fts3IntegrityCheck(p, &bOk);
assert( rc!=SQLITE_CORRUPT_VTAB );
if( rc==SQLITE_ERROR || (rc&0xFF)==SQLITE_CORRUPT ){
*pzErr = sqlite3_mprintf("unable to validate the inverted index for"
" FTS%d table %s.%s: %s",
p->bFts4 ? 4 : 3, zSchema, zTabname, sqlite3_errstr(rc));
if( *pzErr ) rc = SQLITE_OK;
}else if( rc==SQLITE_OK && bOk==0 ){
*pzErr = sqlite3_mprintf("malformed inverted index for FTS%d table %s.%s",
p->bFts4 ? 4 : 3, zSchema, zTabname);
if( *pzErr==0 ) rc = SQLITE_NOMEM;
}
sqlite3Fts3SegmentsClose(p);
return rc;
}
static const sqlite3_module fts3Module = {
/* iVersion */ 3,
/* iVersion */ 4,
/* xCreate */ fts3CreateMethod,
/* xConnect */ fts3ConnectMethod,
/* xBestIndex */ fts3BestIndexMethod,
@ -4005,6 +4061,7 @@ static const sqlite3_module fts3Module = {
/* xRelease */ fts3ReleaseMethod,
/* xRollbackTo */ fts3RollbackToMethod,
/* xShadowName */ fts3ShadowName,
/* xIntegrity */ fts3IntegrityMethod,
};
/*

3
ext/fts3/fts3Int.h
View File

@ -265,6 +265,7 @@ struct Fts3Table {
int nPgsz; /* Page size for host database */
char *zSegmentsTbl; /* Name of %_segments table */
sqlite3_blob *pSegments; /* Blob handle open on %_segments table */
int iSavepoint;
/*
** The following array of hash tables is used to buffer pending index
@ -652,5 +653,7 @@ int sqlite3FtsUnicodeIsdiacritic(int);
int sqlite3Fts3ExprIterate(Fts3Expr*, int (*x)(Fts3Expr*,int,void*), void*);
int sqlite3Fts3IntegrityCheck(Fts3Table *p, int *pbOk);
#endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */
#endif /* _FTSINT_H */

3
ext/fts3/fts3_aux.c
View File

@ -545,7 +545,8 @@ int sqlite3Fts3InitAux(sqlite3 *db){
0, /* xSavepoint */
0, /* xRelease */
0, /* xRollbackTo */
0 /* xShadowName */
0, /* xShadowName */
0 /* xIntegrity */
};
int rc; /* Return code */

3
ext/fts3/fts3_term.c
View File

@ -362,7 +362,8 @@ int sqlite3Fts3InitTerm(sqlite3 *db){
0, /* xSavepoint */
0, /* xRelease */
0, /* xRollbackTo */
0 /* xShadowName */
0, /* xShadowName */
0 /* xIntegrity */
};
int rc; /* Return code */

3
ext/fts3/fts3_tokenize_vtab.c
View File

@ -445,7 +445,8 @@ int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash, void(*xDestroy)(void*)){
0, /* xSavepoint */
0, /* xRelease */
0, /* xRollbackTo */
0 /* xShadowName */
0, /* xShadowName */
0 /* xIntegrity */
};
int rc; /* Return code */

37
ext/fts3/fts3_write.c
View File

@ -3325,7 +3325,6 @@ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){
rc = fts3SegmentMerge(p, p->iPrevLangid, i, FTS3_SEGCURSOR_PENDING);
if( rc==SQLITE_DONE ) rc = SQLITE_OK;
}
sqlite3Fts3PendingTermsClear(p);
/* Determine the auto-incr-merge setting if unknown. If enabled,
** estimate the number of leaf blocks of content to be written
@ -3347,6 +3346,10 @@ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){
rc = sqlite3_reset(pStmt);
}
}
if( rc==SQLITE_OK ){
sqlite3Fts3PendingTermsClear(p);
}
return rc;
}
@ -3978,6 +3981,8 @@ static int fts3AppendToNode(
blobGrowBuffer(pPrev, nTerm, &rc);
if( rc!=SQLITE_OK ) return rc;
assert( pPrev!=0 );
assert( pPrev->a!=0 );
nPrefix = fts3PrefixCompress(pPrev->a, pPrev->n, zTerm, nTerm);
nSuffix = nTerm - nPrefix;
@ -4034,9 +4039,13 @@ static int fts3IncrmergeAppend(
nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist;
/* If the current block is not empty, and if adding this term/doclist
** to the current block would make it larger than Fts3Table.nNodeSize
** bytes, write this block out to the database. */
if( pLeaf->block.n>0 && (pLeaf->block.n + nSpace)>p->nNodeSize ){
** to the current block would make it larger than Fts3Table.nNodeSize bytes,
** and if there is still room for another leaf page, write this block out to
** the database. */
if( pLeaf->block.n>0
&& (pLeaf->block.n + nSpace)>p->nNodeSize
&& pLeaf->iBlock < (pWriter->iStart + pWriter->nLeafEst)
){
rc = fts3WriteSegment(p, pLeaf->iBlock, pLeaf->block.a, pLeaf->block.n);
pWriter->nWork++;
@ -5218,7 +5227,7 @@ static u64 fts3ChecksumIndex(
int rc;
u64 cksum = 0;
assert( *pRc==SQLITE_OK );
if( *pRc ) return 0;
memset(&filter, 0, sizeof(filter));
memset(&csr, 0, sizeof(csr));
@ -5285,7 +5294,7 @@ static u64 fts3ChecksumIndex(
** If an error occurs (e.g. an OOM or IO error), return an SQLite error
** code. The final value of *pbOk is undefined in this case.
*/
static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){
int sqlite3Fts3IntegrityCheck(Fts3Table *p, int *pbOk){
int rc = SQLITE_OK; /* Return code */
u64 cksum1 = 0; /* Checksum based on FTS index contents */
u64 cksum2 = 0; /* Checksum based on %_content contents */
@ -5363,7 +5372,12 @@ static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){
sqlite3_finalize(pStmt);
}
*pbOk = (cksum1==cksum2);
if( rc==SQLITE_CORRUPT_VTAB ){
rc = SQLITE_OK;
*pbOk = 0;
}else{
*pbOk = (rc==SQLITE_OK && cksum1==cksum2);
}
return rc;
}
@ -5403,7 +5417,7 @@ static int fts3DoIntegrityCheck(
){
int rc;
int bOk = 0;
rc = fts3IntegrityCheck(p, &bOk);
rc = sqlite3Fts3IntegrityCheck(p, &bOk);
if( rc==SQLITE_OK && bOk==0 ) rc = FTS_CORRUPT_VTAB;
return rc;
}
@ -5433,8 +5447,11 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){
rc = fts3DoIncrmerge(p, &zVal[6]);
}else if( nVal>10 && 0==sqlite3_strnicmp(zVal, "automerge=", 10) ){
rc = fts3DoAutoincrmerge(p, &zVal[10]);
}else if( nVal==5 && 0==sqlite3_strnicmp(zVal, "flush", 5) ){
rc = sqlite3Fts3PendingTermsFlush(p);
}
#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST)
}else{
else{
int v;
if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){
v = atoi(&zVal[9]);
@ -5452,8 +5469,8 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){
if( v>=4 && v<=FTS3_MERGE_COUNT && (v&1)==0 ) p->nMergeCount = v;
rc = SQLITE_OK;
}
#endif
}
#endif
return rc;
}

4
ext/fts5/extract_api_docs.tcl
View File

@ -223,10 +223,12 @@ proc main {data} {
Fts5ExtensionApi {
set struct [get_fts5_struct $data "^struct Fts5ExtensionApi" "^.;"]
set map [list]
set lKey [list]
foreach {k v} [get_struct_members $data] {
if {[string match x* $k]==0} continue
lappend map $k "<a href=#$k>$k</a>"
lappend lKey $k
}
foreach k [lsort -decr $lKey] { lappend map $k "<a href=#$k>$k</a>" }
output [string map $map $struct]
}

68
ext/fts5/fts5.h
View File

@ -88,8 +88,11 @@ struct Fts5PhraseIter {
** created with the "columnsize=0" option.
**
** xColumnText:
** This function attempts to retrieve the text of column iCol of the
** current document. If successful, (*pz) is set to point to a buffer
** If parameter iCol is less than zero, or greater than or equal to the
** number of columns in the table, SQLITE_RANGE is returned.
**
** Otherwise, this function attempts to retrieve the text of column iCol of
** the current document. If successful, (*pz) is set to point to a buffer
** containing the text in utf-8 encoding, (*pn) is set to the size in bytes
** (not characters) of the buffer and SQLITE_OK is returned. Otherwise,
** if an error occurs, an SQLite error code is returned and the final values
@ -99,8 +102,10 @@ struct Fts5PhraseIter {
** Returns the number of phrases in the current query expression.
**
** xPhraseSize:
** Returns the number of tokens in phrase iPhrase of the query. Phrases
** are numbered starting from zero.
** If parameter iCol is less than zero, or greater than or equal to the
** number of phrases in the current query, as returned by xPhraseCount,
** 0 is returned. Otherwise, this function returns the number of tokens in
** phrase iPhrase of the query. Phrases are numbered starting from zero.
**
** xInstCount:
** Set *pnInst to the total number of occurrences of all phrases within
@ -116,12 +121,13 @@ struct Fts5PhraseIter {
** Query for the details of phrase match iIdx within the current row.
** Phrase matches are numbered starting from zero, so the iIdx argument
** should be greater than or equal to zero and smaller than the value
** output by xInstCount().
** output by xInstCount(). If iIdx is less than zero or greater than
** or equal to the value returned by xInstCount(), SQLITE_RANGE is returned.
**
** Usually, output parameter *piPhrase is set to the phrase number, *piCol
** Otherwise, output parameter *piPhrase is set to the phrase number, *piCol
** to the column in which it occurs and *piOff the token offset of the
** first token of the phrase. Returns SQLITE_OK if successful, or an error
** code (i.e. SQLITE_NOMEM) if an error occurs.
** first token of the phrase. SQLITE_OK is returned if successful, or an
** error code (i.e. SQLITE_NOMEM) if an error occurs.
**
** This API can be quite slow if used with an FTS5 table created with the
** "detail=none" or "detail=column" option.
@ -147,6 +153,10 @@ struct Fts5PhraseIter {
** Invoking Api.xUserData() returns a copy of the pointer passed as
** the third argument to pUserData.
**
** If parameter iPhrase is less than zero, or greater than or equal to
** the number of phrases in the query, as returned by xPhraseCount(),
** this function returns SQLITE_RANGE.
**
** If the callback function returns any value other than SQLITE_OK, the
** query is abandoned and the xQueryPhrase function returns immediately.
** If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK.
@ -261,9 +271,42 @@ struct Fts5PhraseIter {
**
** xPhraseNextColumn()
** See xPhraseFirstColumn above.
**
** xQueryToken(pFts5, iPhrase, iToken, ppToken, pnToken)
** This is used to access token iToken of phrase iPhrase of the current
** query. Before returning, output parameter *ppToken is set to point
** to a buffer containing the requested token, and *pnToken to the
** size of this buffer in bytes.
**
** If iPhrase or iToken are less than zero, or if iPhrase is greater than
** or equal to the number of phrases in the query as reported by
** xPhraseCount(), or if iToken is equal to or greater than the number of
** tokens in the phrase, SQLITE_RANGE is returned and *ppToken and *pnToken
are both zeroed.
**
** The output text is not a copy of the query text that specified the
** token. It is the output of the tokenizer module. For tokendata=1
** tables, this includes any embedded 0x00 and trailing data.
**
** xInstToken(pFts5, iIdx, iToken, ppToken, pnToken)
** This is used to access token iToken of phrase hit iIdx within the
** current row. If iIdx is less than zero or greater than or equal to the
** value returned by xInstCount(), SQLITE_RANGE is returned. Otherwise,
** output variable (*ppToken) is set to point to a buffer containing the
** matching document token, and (*pnToken) to the size of that buffer in
** bytes. This API is not available if the specified token matches a
** prefix query term. In that case both output variables are always set
** to 0.
**
** The output text is not a copy of the document text that was tokenized.
** It is the output of the tokenizer module. For tokendata=1 tables, this
** includes any embedded 0x00 and trailing data.
**
** This API can be quite slow if used with an FTS5 table created with the
** "detail=none" or "detail=column" option.
*/
struct Fts5ExtensionApi {
int iVersion; /* Currently always set to 2 */
int iVersion; /* Currently always set to 3 */
void *(*xUserData)(Fts5Context*);
@ -298,6 +341,13 @@ struct Fts5ExtensionApi {
int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*);
void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol);
/* Below this point are iVersion>=3 only */
int (*xQueryToken)(Fts5Context*,
int iPhrase, int iToken,
const char **ppToken, int *pnToken
);
int (*xInstToken)(Fts5Context*, int iIdx, int iToken, const char**, int*);
};
/*

19
ext/fts5/fts5Int.h
View File

@ -196,6 +196,7 @@ struct Fts5Config {
char *zContent; /* content table */
char *zContentRowid; /* "content_rowid=" option value */
int bColumnsize; /* "columnsize=" option value (dflt==1) */
int bTokendata; /* "tokendata=" option value (dflt==0) */
int eDetail; /* FTS5_DETAIL_XXX value */
char *zContentExprlist;
Fts5Tokenizer *pTok;
@ -395,6 +396,8 @@ struct Fts5IndexIter {
#define FTS5INDEX_QUERY_SKIPEMPTY 0x0010
#define FTS5INDEX_QUERY_NOOUTPUT 0x0020
#define FTS5INDEX_QUERY_SKIPHASH 0x0040
#define FTS5INDEX_QUERY_NOTOKENDATA 0x0080
#define FTS5INDEX_QUERY_SCANONETERM 0x0100
/*
** Create/destroy an Fts5Index object.
@ -463,6 +466,10 @@ void *sqlite3Fts5StructureRef(Fts5Index*);
void sqlite3Fts5StructureRelease(void*);
int sqlite3Fts5StructureTest(Fts5Index*, void*);
/*
** Used by xInstToken():
*/
int sqlite3Fts5IterToken(Fts5IndexIter*, i64, int, int, const char**, int*);
/*
** Insert or remove data to or from the index. Each time a document is
@ -540,6 +547,13 @@ int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin);
int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid);
void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter*);
/* Used to populate hash tables for xInstToken in detail=none/column mode. */
int sqlite3Fts5IndexIterWriteTokendata(
Fts5IndexIter*, const char*, int, i64 iRowid, int iCol, int iOff
);
/*
** End of interface to code in fts5_index.c.
**************************************************************************/
@ -645,6 +659,7 @@ void sqlite3Fts5HashScanNext(Fts5Hash*);
int sqlite3Fts5HashScanEof(Fts5Hash*);
void sqlite3Fts5HashScanEntry(Fts5Hash *,
const char **pzTerm, /* OUT: term (nul-terminated) */
int *pnTerm, /* OUT: Size of term in bytes */
const u8 **ppDoclist, /* OUT: pointer to doclist */
int *pnDoclist /* OUT: size of doclist in bytes */
);
@ -771,6 +786,10 @@ int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**);
int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *);
int sqlite3Fts5ExprQueryToken(Fts5Expr*, int, int, const char**, int*);
int sqlite3Fts5ExprInstToken(Fts5Expr*, i64, int, int, int, int, const char**, int*);
void sqlite3Fts5ExprClearTokens(Fts5Expr*);
/*******************************************
** The fts5_expr.c API above this point is used by the other hand-written
** C code in this module. The interfaces below this point are called by

63
ext/fts5/fts5_aux.c
View File

@ -110,15 +110,19 @@ static int fts5CInstIterInit(
*/
typedef struct HighlightContext HighlightContext;
struct HighlightContext {
CInstIter iter; /* Coalesced Instance Iterator */
int iPos; /* Current token offset in zIn[] */
/* Constant parameters to fts5HighlightCb() */
int iRangeStart; /* First token to include */
int iRangeEnd; /* If non-zero, last token to include */
const char *zOpen; /* Opening highlight */
const char *zClose; /* Closing highlight */
const char *zIn; /* Input text */
int nIn; /* Size of input text in bytes */
int iOff; /* Current offset within zIn[] */
/* Variables modified by fts5HighlightCb() */
CInstIter iter; /* Coalesced Instance Iterator */
int iPos; /* Current token offset in zIn[] */
int iOff; /* Have copied up to this offset in zIn[] */
int bOpen; /* True if highlight is open */
char *zOut; /* Output value */
};
@ -151,8 +155,8 @@ static int fts5HighlightCb(
int tflags, /* Mask of FTS5_TOKEN_* flags */
const char *pToken, /* Buffer containing token */
int nToken, /* Size of token in bytes */
int iStartOff, /* Start offset of token */
int iEndOff /* End offset of token */
int iStartOff, /* Start byte offset of token */
int iEndOff /* End byte offset of token */
){
HighlightContext *p = (HighlightContext*)pContext;
int rc = SQLITE_OK;
@ -168,30 +172,55 @@ static int fts5HighlightCb(
if( p->iRangeStart && iPos==p->iRangeStart ) p->iOff = iStartOff;
}
if( iPos==p->iter.iStart ){
/* If the parenthesis is open, and this token is not part of the current
** phrase, and the starting byte offset of this token is past the point
** that has currently been copied into the output buffer, close the
** parenthesis. */
if( p->bOpen
&& (iPos<=p->iter.iStart || p->iter.iStart<0)
&& iStartOff>p->iOff
){
fts5HighlightAppend(&rc, p, p->zClose, -1);
p->bOpen = 0;
}
/* If this is the start of a new phrase, and the highlight is not open:
**
** * copy text from the input up to the start of the phrase, and
** * open the highlight.
*/
if( iPos==p->iter.iStart && p->bOpen==0 ){
fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iStartOff - p->iOff);
fts5HighlightAppend(&rc, p, p->zOpen, -1);
p->iOff = iStartOff;
p->bOpen = 1;
}
if( iPos==p->iter.iEnd ){
if( p->iRangeEnd>=0 && p->iter.iStart<p->iRangeStart ){
if( p->bOpen==0 ){
assert( p->iRangeEnd>=0 );
fts5HighlightAppend(&rc, p, p->zOpen, -1);
p->bOpen = 1;
}
fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
fts5HighlightAppend(&rc, p, p->zClose, -1);
p->iOff = iEndOff;
if( rc==SQLITE_OK ){
rc = fts5CInstIterNext(&p->iter);
}
}
if( p->iRangeEnd>=0 && iPos==p->iRangeEnd ){
if( iPos==p->iRangeEnd ){
if( p->bOpen ){
if( p->iter.iStart>=0 && iPos>=p->iter.iStart ){
fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
p->iOff = iEndOff;
if( iPos>=p->iter.iStart && iPos<p->iter.iEnd ){
fts5HighlightAppend(&rc, p, p->zClose, -1);
}
fts5HighlightAppend(&rc, p, p->zClose, -1);
p->bOpen = 0;
}
fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
p->iOff = iEndOff;
}
return rc;
@ -223,8 +252,10 @@ static void fts5HighlightFunction(
ctx.zClose = (const char*)sqlite3_value_text(apVal[2]);
ctx.iRangeEnd = -1;
rc = pApi->xColumnText(pFts, iCol, &ctx.zIn, &ctx.nIn);
if( ctx.zIn ){
if( rc==SQLITE_RANGE ){
sqlite3_result_text(pCtx, "", -1, SQLITE_STATIC);
rc = SQLITE_OK;
}else if( ctx.zIn ){
if( rc==SQLITE_OK ){
rc = fts5CInstIterInit(pApi, pFts, iCol, &ctx.iter);
}
@ -232,6 +263,9 @@ static void fts5HighlightFunction(
if( rc==SQLITE_OK ){
rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb);
}
if( ctx.bOpen ){
fts5HighlightAppend(&rc, &ctx, ctx.zClose, -1);
}
fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff);
if( rc==SQLITE_OK ){
@ -510,6 +544,9 @@ static void fts5SnippetFunction(
if( rc==SQLITE_OK ){
rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb);
}
if( ctx.bOpen ){
fts5HighlightAppend(&rc, &ctx, ctx.zClose, -1);
}
if( ctx.iRangeEnd>=(nColSize-1) ){
fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff);
}else{

3
ext/fts5/fts5_buffer.c
View File

@ -68,6 +68,7 @@ void sqlite3Fts5BufferAppendBlob(
){
if( nData ){
if( fts5BufferGrow(pRc, pBuf, nData) ) return;
assert( pBuf->p!=0 );
memcpy(&pBuf->p[pBuf->n], pData, nData);
pBuf->n += nData;
}
@ -169,6 +170,7 @@ int sqlite3Fts5PoslistNext64(
i64 *piOff /* IN/OUT: Current offset */
){
int i = *pi;
assert( a!=0 || i==0 );
if( i>=n ){
/* EOF */
*piOff = -1;
@ -176,6 +178,7 @@ int sqlite3Fts5PoslistNext64(
}else{
i64 iOff = *piOff;
u32 iVal;
assert( a!=0 );
fts5FastGetVarint32(a, i, iVal);
if( iVal<=1 ){
if( iVal==0 ){

10
ext/fts5/fts5_config.c
View File

@ -398,6 +398,16 @@ static int fts5ConfigParseSpecial(
return rc;
}
if( sqlite3_strnicmp("tokendata", zCmd, nCmd)==0 ){
if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
*pzErr = sqlite3_mprintf("malformed tokendata=... directive");
rc = SQLITE_ERROR;
}else{
pConfig->bTokendata = (zArg[0]=='1');
}
return rc;
}
*pzErr = sqlite3_mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd);
return SQLITE_ERROR;
}

174
ext/fts5/fts5_expr.c
View File

@ -100,7 +100,9 @@ struct Fts5ExprNode {
struct Fts5ExprTerm {
u8 bPrefix; /* True for a prefix term */
u8 bFirst; /* True if token must be first in column */
char *zTerm; /* nul-terminated term */
char *pTerm; /* Term data */
int nQueryTerm; /* Effective size of term in bytes */
int nFullTerm; /* Size of term in bytes incl. tokendata */
Fts5IndexIter *pIter; /* Iterator for this term */
Fts5ExprTerm *pSynonym; /* Pointer to first in list of synonyms */
};
@ -967,7 +969,7 @@ static int fts5ExprNearInitAll(
p->pIter = 0;
}
rc = sqlite3Fts5IndexQuery(
pExpr->pIndex, p->zTerm, (int)strlen(p->zTerm),
pExpr->pIndex, p->pTerm, p->nQueryTerm,
(pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) |
(pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0),
pNear->pColset,
@ -1604,7 +1606,7 @@ static void fts5ExprPhraseFree(Fts5ExprPhrase *pPhrase){
Fts5ExprTerm *pSyn;
Fts5ExprTerm *pNext;
Fts5ExprTerm *pTerm = &pPhrase->aTerm[i];
sqlite3_free(pTerm->zTerm);
sqlite3_free(pTerm->pTerm);
sqlite3Fts5IterClose(pTerm->pIter);
for(pSyn=pTerm->pSynonym; pSyn; pSyn=pNext){
pNext = pSyn->pSynonym;
@ -1702,6 +1704,7 @@ Fts5ExprNearset *sqlite3Fts5ParseNearset(
typedef struct TokenCtx TokenCtx;
struct TokenCtx {
Fts5ExprPhrase *pPhrase;
Fts5Config *pConfig;
int rc;
};
@ -1735,8 +1738,12 @@ static int fts5ParseTokenize(
rc = SQLITE_NOMEM;
}else{
memset(pSyn, 0, (size_t)nByte);
pSyn->zTerm = ((char*)pSyn) + sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer);
memcpy(pSyn->zTerm, pToken, nToken);
pSyn->pTerm = ((char*)pSyn) + sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer);
pSyn->nFullTerm = pSyn->nQueryTerm = nToken;
if( pCtx->pConfig->bTokendata ){
pSyn->nQueryTerm = (int)strlen(pSyn->pTerm);
}
memcpy(pSyn->pTerm, pToken, nToken);
pSyn->pSynonym = pPhrase->aTerm[pPhrase->nTerm-1].pSynonym;
pPhrase->aTerm[pPhrase->nTerm-1].pSynonym = pSyn;
}
@ -1761,7 +1768,11 @@ static int fts5ParseTokenize(
if( rc==SQLITE_OK ){
pTerm = &pPhrase->aTerm[pPhrase->nTerm++];
memset(pTerm, 0, sizeof(Fts5ExprTerm));
pTerm->zTerm = sqlite3Fts5Strndup(&rc, pToken, nToken);
pTerm->pTerm = sqlite3Fts5Strndup(&rc, pToken, nToken);
pTerm->nFullTerm = pTerm->nQueryTerm = nToken;
if( pCtx->pConfig->bTokendata && rc==SQLITE_OK ){
pTerm->nQueryTerm = (int)strlen(pTerm->pTerm);
}
}
}
@ -1828,6 +1839,7 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm(
memset(&sCtx, 0, sizeof(TokenCtx));
sCtx.pPhrase = pAppend;
sCtx.pConfig = pConfig;
rc = fts5ParseStringFromToken(pToken, &z);
if( rc==SQLITE_OK ){
@ -1875,12 +1887,15 @@ int sqlite3Fts5ExprClonePhrase(
Fts5Expr **ppNew
){
int rc = SQLITE_OK; /* Return code */
Fts5ExprPhrase *pOrig; /* The phrase extracted from pExpr */
Fts5ExprPhrase *pOrig = 0; /* The phrase extracted from pExpr */
Fts5Expr *pNew = 0; /* Expression to return via *ppNew */
TokenCtx sCtx = {0,0}; /* Context object for fts5ParseTokenize */
TokenCtx sCtx = {0,0,0}; /* Context object for fts5ParseTokenize */
if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){
rc = SQLITE_RANGE;
}else{
pOrig = pExpr->apExprPhrase[iPhrase];
pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr));
}
if( rc==SQLITE_OK ){
pNew->apExprPhrase = (Fts5ExprPhrase**)sqlite3Fts5MallocZero(&rc,
sizeof(Fts5ExprPhrase*));
@ -1893,7 +1908,7 @@ int sqlite3Fts5ExprClonePhrase(
pNew->pRoot->pNear = (Fts5ExprNearset*)sqlite3Fts5MallocZero(&rc,
sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*));
}
if( rc==SQLITE_OK ){
if( rc==SQLITE_OK && ALWAYS(pOrig!=0) ){
Fts5Colset *pColsetOrig = pOrig->pNode->pNear->pColset;
if( pColsetOrig ){
sqlite3_int64 nByte;
@ -1907,15 +1922,15 @@ int sqlite3Fts5ExprClonePhrase(
}
}
if( rc==SQLITE_OK ){
if( pOrig->nTerm ){
int i; /* Used to iterate through phrase terms */
sCtx.pConfig = pExpr->pConfig;
for(i=0; rc==SQLITE_OK && i<pOrig->nTerm; i++){
int tflags = 0;
Fts5ExprTerm *p;
for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){
const char *zTerm = p->zTerm;
rc = fts5ParseTokenize((void*)&sCtx, tflags, zTerm, (int)strlen(zTerm),
0, 0);
rc = fts5ParseTokenize((void*)&sCtx,tflags,p->pTerm,p->nFullTerm,0,0);
tflags = FTS5_TOKEN_COLOCATED;
}
if( rc==SQLITE_OK ){
@ -1928,6 +1943,7 @@ int sqlite3Fts5ExprClonePhrase(
** no token characters at all. (e.g ... MATCH '""'). */
sCtx.pPhrase = sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprPhrase));
}
}
if( rc==SQLITE_OK && ALWAYS(sCtx.pPhrase) ){
/* All the allocations succeeded. Put the expression object together. */
@ -2296,11 +2312,13 @@ static Fts5ExprNode *fts5ParsePhraseToAnd(
if( parseGrowPhraseArray(pParse) ){
fts5ExprPhraseFree(pPhrase);
}else{
Fts5ExprTerm *p = &pNear->apPhrase[0]->aTerm[ii];
Fts5ExprTerm *pTo = &pPhrase->aTerm[0];
pParse->apPhrase[pParse->nPhrase++] = pPhrase;
pPhrase->nTerm = 1;
pPhrase->aTerm[0].zTerm = sqlite3Fts5Strndup(
&pParse->rc, pNear->apPhrase[0]->aTerm[ii].zTerm, -1
);
pTo->pTerm = sqlite3Fts5Strndup(&pParse->rc, p->pTerm, p->nFullTerm);
pTo->nQueryTerm = p->nQueryTerm;
pTo->nFullTerm = p->nFullTerm;
pRet->apChild[ii] = sqlite3Fts5ParseNode(pParse, FTS5_STRING,
0, 0, sqlite3Fts5ParseNearset(pParse, 0, pPhrase)
);
@ -2485,16 +2503,17 @@ static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){
/* Determine the maximum amount of space required. */
for(p=pTerm; p; p=p->pSynonym){
nByte += (int)strlen(pTerm->zTerm) * 2 + 3 + 2;
nByte += pTerm->nQueryTerm * 2 + 3 + 2;
}
zQuoted = sqlite3_malloc64(nByte);
if( zQuoted ){
int i = 0;
for(p=pTerm; p; p=p->pSynonym){
char *zIn = p->zTerm;
char *zIn = p->pTerm;
char *zEnd = &zIn[p->nQueryTerm];
zQuoted[i++] = '"';
while( *zIn ){
while( zIn<zEnd ){
if( *zIn=='"' ) zQuoted[i++] = '"';
zQuoted[i++] = *zIn++;
}
@ -2572,8 +2591,10 @@ static char *fts5ExprPrintTcl(
zRet = fts5PrintfAppend(zRet, " {");
for(iTerm=0; zRet && iTerm<pPhrase->nTerm; iTerm++){
char *zTerm = pPhrase->aTerm[iTerm].zTerm;
zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" ", zTerm);
Fts5ExprTerm *p = &pPhrase->aTerm[iTerm];
zRet = fts5PrintfAppend(zRet, "%s%.*s", iTerm==0?"":" ",
p->nQueryTerm, p->pTerm
);
if( pPhrase->aTerm[iTerm].bPrefix ){
zRet = fts5PrintfAppend(zRet, "*");
}
@ -2974,6 +2995,17 @@ static int fts5ExprColsetTest(Fts5Colset *pColset, int iCol){
return 0;
}
/*
** pToken is a buffer nToken bytes in size that may or may not contain
** an embedded 0x00 byte. If it does, return the number of bytes in
** the buffer before the 0x00. If it does not, return nToken.
*/
static int fts5QueryTerm(const char *pToken, int nToken){
int ii;
for(ii=0; ii<nToken && pToken[ii]; ii++){}
return ii;
}
static int fts5ExprPopulatePoslistsCb(
void *pCtx, /* Copy of 2nd argument to xTokenize() */
int tflags, /* Mask of FTS5_TOKEN_* flags */
@ -2985,22 +3017,33 @@ static int fts5ExprPopulatePoslistsCb(
Fts5ExprCtx *p = (Fts5ExprCtx*)pCtx;
Fts5Expr *pExpr = p->pExpr;
int i;
int nQuery = nToken;
i64 iRowid = pExpr->pRoot->iRowid;
UNUSED_PARAM2(iUnused1, iUnused2);
if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE;
if( nQuery>FTS5_MAX_TOKEN_SIZE ) nQuery = FTS5_MAX_TOKEN_SIZE;
if( pExpr->pConfig->bTokendata ){
nQuery = fts5QueryTerm(pToken, nQuery);
}
if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++;
for(i=0; i<pExpr->nPhrase; i++){
Fts5ExprTerm *pTerm;
Fts5ExprTerm *pT;
if( p->aPopulator[i].bOk==0 ) continue;
for(pTerm=&pExpr->apExprPhrase[i]->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){
int nTerm = (int)strlen(pTerm->zTerm);
if( (nTerm==nToken || (nTerm<nToken && pTerm->bPrefix))
&& memcmp(pTerm->zTerm, pToken, nTerm)==0
for(pT=&pExpr->apExprPhrase[i]->aTerm[0]; pT; pT=pT->pSynonym){
if( (pT->nQueryTerm==nQuery || (pT->nQueryTerm<nQuery && pT->bPrefix))
&& memcmp(pT->pTerm, pToken, pT->nQueryTerm)==0
){
int rc = sqlite3Fts5PoslistWriterAppend(
&pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff
);
if( rc==SQLITE_OK && pExpr->pConfig->bTokendata && !pT->bPrefix ){
int iCol = p->iOff>>32;
int iTokOff = p->iOff & 0x7FFFFFFF;
rc = sqlite3Fts5IndexIterWriteTokendata(
pT->pIter, pToken, nToken, iRowid, iCol, iTokOff
);
}
if( rc ) return rc;
break;
}
@ -3135,3 +3178,80 @@ int sqlite3Fts5ExprPhraseCollist(
return rc;
}
/*
** Does the work of the fts5_api.xQueryToken() API method.
*/
int sqlite3Fts5ExprQueryToken(
Fts5Expr *pExpr,
int iPhrase,
int iToken,
const char **ppOut,
int *pnOut
){
Fts5ExprPhrase *pPhrase = 0;
if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){
return SQLITE_RANGE;
}
pPhrase = pExpr->apExprPhrase[iPhrase];
if( iToken<0 || iToken>=pPhrase->nTerm ){
return SQLITE_RANGE;
}
*ppOut = pPhrase->aTerm[iToken].pTerm;
*pnOut = pPhrase->aTerm[iToken].nFullTerm;
return SQLITE_OK;
}
/*
** Does the work of the fts5_api.xInstToken() API method.
*/
int sqlite3Fts5ExprInstToken(
Fts5Expr *pExpr,
i64 iRowid,
int iPhrase,
int iCol,
int iOff,
int iToken,
const char **ppOut,
int *pnOut
){
Fts5ExprPhrase *pPhrase = 0;
Fts5ExprTerm *pTerm = 0;
int rc = SQLITE_OK;
if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){
return SQLITE_RANGE;
}
pPhrase = pExpr->apExprPhrase[iPhrase];
if( iToken<0 || iToken>=pPhrase->nTerm ){
return SQLITE_RANGE;
}
pTerm = &pPhrase->aTerm[iToken];
if( pTerm->bPrefix==0 ){
if( pExpr->pConfig->bTokendata ){
rc = sqlite3Fts5IterToken(
pTerm->pIter, iRowid, iCol, iOff+iToken, ppOut, pnOut
);
}else{
*ppOut = pTerm->pTerm;
*pnOut = pTerm->nFullTerm;
}
}
return rc;
}
/*
** Clear the token mappings for all Fts5IndexIter objects mannaged by
** the expression passed as the only argument.
*/
void sqlite3Fts5ExprClearTokens(Fts5Expr *pExpr){
int ii;
for(ii=0; ii<pExpr->nPhrase; ii++){
Fts5ExprTerm *pT;
for(pT=&pExpr->apExprPhrase[ii]->aTerm[0]; pT; pT=pT->pSynonym){
sqlite3Fts5IndexIterClearTokendata(pT->pIter);
}
}
}

55
ext/fts5/fts5_hash.c
View File

@ -36,10 +36,15 @@ struct Fts5Hash {
/*
** Each entry in the hash table is represented by an object of the
** following type. Each object, its key (a nul-terminated string) and
** its current data are stored in a single memory allocation. The
** key immediately follows the object in memory. The position list
** data immediately follows the key data in memory.
** following type. Each object, its key, and its current data are stored
** in a single memory allocation. The key immediately follows the object
** in memory. The position list data immediately follows the key data
** in memory.
**
** The key is Fts5HashEntry.nKey bytes in size. It consists of a single
** byte identifying the index (either the main term index or a prefix-index),
** followed by the term data. For example: "0token". There is no
** nul-terminator - in this case nKey=6.
**
** The data that follows the key is in a similar, but not identical format
** to the doclist data stored in the database. It is:
@ -174,8 +179,7 @@ static int fts5HashResize(Fts5Hash *pHash){
unsigned int iHash;
Fts5HashEntry *p = apOld[i];
apOld[i] = p->pHashNext;
iHash = fts5HashKey(nNew, (u8*)fts5EntryKey(p),
(int)strlen(fts5EntryKey(p)));
iHash = fts5HashKey(nNew, (u8*)fts5EntryKey(p), p->nKey);
p->pHashNext = apNew[iHash];
apNew[iHash] = p;
}
@ -259,7 +263,7 @@ int sqlite3Fts5HashWrite(
for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
char *zKey = fts5EntryKey(p);
if( zKey[0]==bByte
&& p->nKey==nToken
&& p->nKey==nToken+1
&& memcmp(&zKey[1], pToken, nToken)==0
){
break;
@ -289,9 +293,9 @@ int sqlite3Fts5HashWrite(
zKey[0] = bByte;
memcpy(&zKey[1], pToken, nToken);
assert( iHash==fts5HashKey(pHash->nSlot, (u8*)zKey, nToken+1) );
p->nKey = nToken;
p->nKey = nToken+1;
zKey[nToken+1] = '\0';
p->nData = nToken+1 + 1 + sizeof(Fts5HashEntry);
p->nData = nToken+1 + sizeof(Fts5HashEntry);
p->pHashNext = pHash->aSlot[iHash];
pHash->aSlot[iHash] = p;
pHash->nEntry++;
@ -408,12 +412,17 @@ static Fts5HashEntry *fts5HashEntryMerge(
*ppOut = p1;
p1 = 0;
}else{
int i = 0;
char *zKey1 = fts5EntryKey(p1);
char *zKey2 = fts5EntryKey(p2);
while( zKey1[i]==zKey2[i] ) i++;
int nMin = MIN(p1->nKey, p2->nKey);
if( ((u8)zKey1[i])>((u8)zKey2[i]) ){
int cmp = memcmp(zKey1, zKey2, nMin);
if( cmp==0 ){
cmp = p1->nKey - p2->nKey;
}
assert( cmp!=0 );
if( cmp>0 ){
/* p2 is smaller */
*ppOut = p2;
ppOut = &p2->pScanNext;
@ -432,10 +441,8 @@ static Fts5HashEntry *fts5HashEntryMerge(
}
/*
** Extract all tokens from hash table iHash and link them into a list
** in sorted order. The hash table is cleared before returning. It is
** the responsibility of the caller to free the elements of the returned
** list.
** Link all tokens from hash table iHash into a list in sorted order. The
** tokens are not removed from the hash table.
*/
static int fts5HashEntrySort(
Fts5Hash *pHash,
@ -457,7 +464,7 @@ static int fts5HashEntrySort(
Fts5HashEntry *pIter;
for(pIter=pHash->aSlot[iSlot]; pIter; pIter=pIter->pHashNext){
if( pTerm==0
|| (pIter->nKey+1>=nTerm && 0==memcmp(fts5EntryKey(pIter), pTerm, nTerm))
|| (pIter->nKey>=nTerm && 0==memcmp(fts5EntryKey(pIter), pTerm, nTerm))
){
Fts5HashEntry *pEntry = pIter;
pEntry->pScanNext = 0;
@ -496,12 +503,11 @@ int sqlite3Fts5HashQuery(
for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
zKey = fts5EntryKey(p);
assert( p->nKey+1==(int)strlen(zKey) );
if( nTerm==p->nKey+1 && memcmp(zKey, pTerm, nTerm)==0 ) break;
if( nTerm==p->nKey && memcmp(zKey, pTerm, nTerm)==0 ) break;
}
if( p ){
int nHashPre = sizeof(Fts5HashEntry) + nTerm + 1;
int nHashPre = sizeof(Fts5HashEntry) + nTerm;
int nList = p->nData - nHashPre;
u8 *pRet = (u8*)(*ppOut = sqlite3_malloc64(nPre + nList + 10));
if( pRet ){
@ -562,19 +568,22 @@ int sqlite3Fts5HashScanEof(Fts5Hash *p){
void sqlite3Fts5HashScanEntry(
Fts5Hash *pHash,
const char **pzTerm, /* OUT: term (nul-terminated) */
int *pnTerm, /* OUT: Size of term in bytes */
const u8 **ppDoclist, /* OUT: pointer to doclist */
int *pnDoclist /* OUT: size of doclist in bytes */
){
Fts5HashEntry *p;
if( (p = pHash->pScan) ){
char *zKey = fts5EntryKey(p);
int nTerm = (int)strlen(zKey);
int nTerm = p->nKey;
fts5HashAddPoslistSize(pHash, p, 0);
*pzTerm = zKey;
*ppDoclist = (const u8*)&zKey[nTerm+1];
*pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm + 1);
*pnTerm = nTerm;
*ppDoclist = (const u8*)&zKey[nTerm];
*pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm);
}else{
*pzTerm = 0;
*pnTerm = 0;
*ppDoclist = 0;
*pnDoclist = 0;
}

985
ext/fts5/fts5_index.c
View File

File diff suppressed because it is too large Load Diff

191
ext/fts5/fts5_main.c
View File

@ -117,6 +117,8 @@ struct Fts5FullTable {
Fts5Storage *pStorage; /* Document store */
Fts5Global *pGlobal; /* Global (connection wide) data */
Fts5Cursor *pSortCsr; /* Sort data from this cursor */
int iSavepoint; /* Successful xSavepoint()+1 */
#ifdef SQLITE_DEBUG
struct Fts5TransactionState ts;
#endif
@ -405,6 +407,13 @@ static int fts5InitVtab(
pConfig->pzErrmsg = 0;
}
if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){
rc = sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, (int)1);
}
if( rc==SQLITE_OK ){
rc = sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS);
}
if( rc!=SQLITE_OK ){
fts5FreeVtab(pTab);
pTab = 0;
@ -647,12 +656,15 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
}
idxStr[iIdxStr] = '\0';
/* Set idxFlags flags for the ORDER BY clause */
/* Set idxFlags flags for the ORDER BY clause
**
** Note that tokendata=1 tables cannot currently handle "ORDER BY rowid DESC".
*/
if( pInfo->nOrderBy==1 ){
int iSort = pInfo->aOrderBy[0].iColumn;
if( iSort==(pConfig->nCol+1) && bSeenMatch ){
idxFlags |= FTS5_BI_ORDER_RANK;
}else if( iSort==-1 ){
}else if( iSort==-1 && (!pInfo->aOrderBy[0].desc || !pConfig->bTokendata) ){
idxFlags |= FTS5_BI_ORDER_ROWID;
}
if( BitFlagTest(idxFlags, FTS5_BI_ORDER_RANK|FTS5_BI_ORDER_ROWID) ){
@ -904,6 +916,16 @@ static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){
);
assert( !CsrFlagTest(pCsr, FTS5CSR_EOF) );
/* If this cursor uses FTS5_PLAN_MATCH and this is a tokendata=1 table,
** clear any token mappings accumulated at the fts5_index.c level. In
** other cases, specifically FTS5_PLAN_SOURCE and FTS5_PLAN_SORTED_MATCH,
** we need to retain the mappings for the entire query. */
if( pCsr->ePlan==FTS5_PLAN_MATCH
&& ((Fts5Table*)pCursor->pVtab)->pConfig->bTokendata
){
sqlite3Fts5ExprClearTokens(pCsr->pExpr);
}
if( pCsr->ePlan<3 ){
int bSkip = 0;
if( (rc = fts5CursorReseek(pCsr, &bSkip)) || bSkip ) return rc;
@ -1329,6 +1351,9 @@ static int fts5FilterMethod(
pCsr->iFirstRowid = fts5GetRowidLimit(pRowidGe, SMALLEST_INT64);
}
rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex);
if( rc!=SQLITE_OK ) goto filter_out;
if( pTab->pSortCsr ){
/* If pSortCsr is non-NULL, then this call is being made as part of
** processing for a "... MATCH <expr> ORDER BY rank" query (ePlan is
@ -1351,6 +1376,7 @@ static int fts5FilterMethod(
pCsr->pExpr = pTab->pSortCsr->pExpr;
rc = fts5CursorFirst(pTab, pCsr, bDesc);
}else if( pCsr->pExpr ){
assert( rc==SQLITE_OK );
rc = fts5CursorParseRank(pConfig, pCsr, pRank);
if( rc==SQLITE_OK ){
if( bOrderByRank ){
@ -1522,6 +1548,7 @@ static int fts5SpecialInsert(
Fts5Config *pConfig = pTab->p.pConfig;
int rc = SQLITE_OK;
int bError = 0;
int bLoadConfig = 0;
if( 0==sqlite3_stricmp("delete-all", zCmd) ){
if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
@ -1533,6 +1560,7 @@ static int fts5SpecialInsert(
}else{
rc = sqlite3Fts5StorageDeleteAll(pTab->pStorage);
}
bLoadConfig = 1;
}else if( 0==sqlite3_stricmp("rebuild", zCmd) ){
if( pConfig->eContent==FTS5_CONTENT_NONE ){
fts5SetVtabError(pTab,
@ -1542,6 +1570,7 @@ static int fts5SpecialInsert(
}else{
rc = sqlite3Fts5StorageRebuild(pTab->pStorage);
}
bLoadConfig = 1;
}else if( 0==sqlite3_stricmp("optimize", zCmd) ){
rc = sqlite3Fts5StorageOptimize(pTab->pStorage);
}else if( 0==sqlite3_stricmp("merge", zCmd) ){
@ -1554,8 +1583,13 @@ static int fts5SpecialInsert(
}else if( 0==sqlite3_stricmp("prefix-index", zCmd) ){
pConfig->bPrefixIndex = sqlite3_value_int(pVal);
#endif
}else if( 0==sqlite3_stricmp("flush", zCmd) ){
rc = sqlite3Fts5FlushToDisk(&pTab->p);
}else{
rc = sqlite3Fts5FlushToDisk(&pTab->p);
if( rc==SQLITE_OK ){
rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex);
}
if( rc==SQLITE_OK ){
rc = sqlite3Fts5ConfigSetValue(pTab->p.pConfig, zCmd, pVal, &bError);
}
@ -1567,6 +1601,12 @@ static int fts5SpecialInsert(
}
}
}
if( rc==SQLITE_OK && bLoadConfig ){
pTab->p.pConfig->iCookie--;
rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex);
}
return rc;
}
@ -1685,7 +1725,7 @@ static int fts5UpdateMethod(
assert( nArg!=1 || eType0==SQLITE_INTEGER );
/* Filter out attempts to run UPDATE or DELETE on contentless tables.
** This is not suported. Except - DELETE is supported if the CREATE
** This is not suported. Except - they are both supported if the CREATE
** VIRTUAL TABLE statement contained "contentless_delete=1". */
if( eType0==SQLITE_INTEGER
&& pConfig->eContent==FTS5_CONTENT_NONE
@ -1714,7 +1754,8 @@ static int fts5UpdateMethod(
}
else if( eType0!=SQLITE_INTEGER ){
/* If this is a REPLACE, first remove the current entry (if any) */
/* An INSERT statement. If the conflict-mode is REPLACE, first remove
** the current entry (if any). */
if( eConflict==SQLITE_REPLACE && eType1==SQLITE_INTEGER ){
i64 iNew = sqlite3_value_int64(apVal[1]); /* Rowid to delete */
rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0);
@ -1873,7 +1914,10 @@ static int fts5ApiColumnText(
){
int rc = SQLITE_OK;
Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
if( fts5IsContentless((Fts5FullTable*)(pCsr->base.pVtab))
Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
if( iCol<0 || iCol>=pTab->pConfig->nCol ){
rc = SQLITE_RANGE;
}else if( fts5IsContentless((Fts5FullTable*)(pCsr->base.pVtab))
|| pCsr->ePlan==FTS5_PLAN_SPECIAL
){
*pz = 0;
@ -1898,8 +1942,9 @@ static int fts5CsrPoslist(
int rc = SQLITE_OK;
int bLive = (pCsr->pSorter==0);
if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){
if( iPhrase<0 || iPhrase>=sqlite3Fts5ExprPhraseCount(pCsr->pExpr) ){
rc = SQLITE_RANGE;
}else if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){
if( pConfig->eDetail!=FTS5_DETAIL_FULL ){
Fts5PoslistPopulator *aPopulator;
int i;
@ -1923,6 +1968,7 @@ static int fts5CsrPoslist(
CsrFlagClear(pCsr, FTS5CSR_REQUIRE_POSLIST);
}
if( rc==SQLITE_OK ){
if( pCsr->pSorter && pConfig->eDetail==FTS5_DETAIL_FULL ){
Fts5Sorter *pSorter = pCsr->pSorter;
int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]);
@ -1931,6 +1977,11 @@ static int fts5CsrPoslist(
}else{
*pn = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa);
}
}else{
*pa = 0;
*pn = 0;
}
return rc;
}
@ -2038,12 +2089,6 @@ static int fts5ApiInst(
){
if( iIdx<0 || iIdx>=pCsr->nInstCount ){
rc = SQLITE_RANGE;
#if 0
}else if( fts5IsOffsetless((Fts5Table*)pCsr->base.pVtab) ){
*piPhrase = pCsr->aInst[iIdx*3];
*piCol = pCsr->aInst[iIdx*3 + 2];
*piOff = -1;
#endif
}else{
*piPhrase = pCsr->aInst[iIdx*3];
*piCol = pCsr->aInst[iIdx*3 + 1];
@ -2298,13 +2343,56 @@ static int fts5ApiPhraseFirstColumn(
return rc;
}
/*
** xQueryToken() API implemenetation.
*/
static int fts5ApiQueryToken(
Fts5Context* pCtx,
int iPhrase,
int iToken,
const char **ppOut,
int *pnOut
){
Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
return sqlite3Fts5ExprQueryToken(pCsr->pExpr, iPhrase, iToken, ppOut, pnOut);
}
/*
** xInstToken() API implemenetation.
*/
static int fts5ApiInstToken(
Fts5Context *pCtx,
int iIdx,
int iToken,
const char **ppOut, int *pnOut
){
Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
int rc = SQLITE_OK;
if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)==0
|| SQLITE_OK==(rc = fts5CacheInstArray(pCsr))
){
if( iIdx<0 || iIdx>=pCsr->nInstCount ){
rc = SQLITE_RANGE;
}else{
int iPhrase = pCsr->aInst[iIdx*3];
int iCol = pCsr->aInst[iIdx*3 + 1];
int iOff = pCsr->aInst[iIdx*3 + 2];
i64 iRowid = fts5CursorRowid(pCsr);
rc = sqlite3Fts5ExprInstToken(
pCsr->pExpr, iRowid, iPhrase, iCol, iOff, iToken, ppOut, pnOut
);
}
}
return rc;
}
static int fts5ApiQueryPhrase(Fts5Context*, int, void*,
int(*)(const Fts5ExtensionApi*, Fts5Context*, void*)
);
static const Fts5ExtensionApi sFts5Api = {
2, /* iVersion */
3, /* iVersion */
fts5ApiUserData,
fts5ApiColumnCount,
fts5ApiRowCount,
@ -2324,6 +2412,8 @@ static const Fts5ExtensionApi sFts5Api = {
fts5ApiPhraseNext,
fts5ApiPhraseFirstColumn,
fts5ApiPhraseNextColumn,
fts5ApiQueryToken,
fts5ApiInstToken
};
/*
@ -2588,8 +2678,10 @@ static int fts5RenameMethod(
sqlite3_vtab *pVtab, /* Virtual table handle */
const char *zName /* New name of table */
){
int rc;
Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
return sqlite3Fts5StorageRename(pTab->pStorage, zName);
rc = sqlite3Fts5StorageRename(pTab->pStorage, zName);
return rc;
}
int sqlite3Fts5FlushToDisk(Fts5Table *pTab){
@ -2603,9 +2695,15 @@ int sqlite3Fts5FlushToDisk(Fts5Table *pTab){
** Flush the contents of the pending-terms table to disk.
*/
static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
UNUSED_PARAM(iSavepoint); /* Call below is a no-op for NDEBUG builds */
fts5CheckTransactionState((Fts5FullTable*)pVtab, FTS5_SAVEPOINT, iSavepoint);
return sqlite3Fts5FlushToDisk((Fts5Table*)pVtab);
Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
int rc = SQLITE_OK;
fts5CheckTransactionState(pTab, FTS5_SAVEPOINT, iSavepoint);
rc = sqlite3Fts5FlushToDisk((Fts5Table*)pVtab);
if( rc==SQLITE_OK ){
pTab->iSavepoint = iSavepoint+1;
}
return rc;
}
/*
@ -2614,9 +2712,16 @@ static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
** This is a no-op.
*/
static int fts5ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
UNUSED_PARAM(iSavepoint); /* Call below is a no-op for NDEBUG builds */
fts5CheckTransactionState((Fts5FullTable*)pVtab, FTS5_RELEASE, iSavepoint);
return sqlite3Fts5FlushToDisk((Fts5Table*)pVtab);
Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
int rc = SQLITE_OK;
fts5CheckTransactionState(pTab, FTS5_RELEASE, iSavepoint);
if( (iSavepoint+1)<pTab->iSavepoint ){
rc = sqlite3Fts5FlushToDisk(&pTab->p);
if( rc==SQLITE_OK ){
pTab->iSavepoint = iSavepoint;
}
}
return rc;
}
/*
@ -2626,11 +2731,14 @@ static int fts5ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
*/
static int fts5RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){
Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
UNUSED_PARAM(iSavepoint); /* Call below is a no-op for NDEBUG builds */
int rc = SQLITE_OK;
fts5CheckTransactionState(pTab, FTS5_ROLLBACKTO, iSavepoint);
fts5TripCursors(pTab);
if( (iSavepoint+1)<=pTab->iSavepoint ){
pTab->p.pConfig->pgsz = 0;
return sqlite3Fts5StorageRollback(pTab->pStorage);
rc = sqlite3Fts5StorageRollback(pTab->pStorage);
}
return rc;
}
/*
@ -2850,9 +2958,41 @@ static int fts5ShadowName(const char *zName){
return 0;
}
/*
** Run an integrity check on the FTS5 data structures. Return a string
** if anything is found amiss. Return a NULL pointer if everything is
** OK.
*/
static int fts5IntegrityMethod(
sqlite3_vtab *pVtab, /* the FTS5 virtual table to check */
const char *zSchema, /* Name of schema in which this table lives */
const char *zTabname, /* Name of the table itself */
int isQuick, /* True if this is a quick-check */
char **pzErr /* Write error message here */
){
Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
int rc;
assert( pzErr!=0 && *pzErr==0 );
UNUSED_PARAM(isQuick);
rc = sqlite3Fts5StorageIntegrity(pTab->pStorage, 0);
if( (rc&0xff)==SQLITE_CORRUPT ){
*pzErr = sqlite3_mprintf("malformed inverted index for FTS5 table %s.%s",
zSchema, zTabname);
rc = (*pzErr) ? SQLITE_OK : SQLITE_NOMEM;
}else if( rc!=SQLITE_OK ){
*pzErr = sqlite3_mprintf("unable to validate the inverted index for"
" FTS5 table %s.%s: %s",
zSchema, zTabname, sqlite3_errstr(rc));
}
sqlite3Fts5IndexCloseReader(pTab->p.pIndex);
return rc;
}
static int fts5Init(sqlite3 *db){
static const sqlite3_module fts5Mod = {
/* iVersion */ 3,
/* iVersion */ 4,
/* xCreate */ fts5CreateMethod,
/* xConnect */ fts5ConnectMethod,
/* xBestIndex */ fts5BestIndexMethod,
@ -2875,7 +3015,8 @@ static int fts5Init(sqlite3 *db){
/* xSavepoint */ fts5SavepointMethod,
/* xRelease */ fts5ReleaseMethod,
/* xRollbackTo */ fts5RollbackToMethod,
/* xShadowName */ fts5ShadowName
/* xShadowName */ fts5ShadowName,
/* xIntegrity */ fts5IntegrityMethod
};
int rc;

4
ext/fts5/fts5_storage.c
View File

@ -673,7 +673,7 @@ int sqlite3Fts5StorageRebuild(Fts5Storage *p){
}
if( rc==SQLITE_OK ){
rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, 0);
rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, pConfig->pzErrmsg);
}
while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pScan) ){
@ -1184,8 +1184,10 @@ int sqlite3Fts5StorageSync(Fts5Storage *p){
i64 iLastRowid = sqlite3_last_insert_rowid(p->pConfig->db);
if( p->bTotalsValid ){
rc = fts5StorageSaveTotals(p);
if( rc==SQLITE_OK ){
p->bTotalsValid = 0;
}
}
if( rc==SQLITE_OK ){
rc = sqlite3Fts5IndexSync(p->pIndex);
}

208
ext/fts5/fts5_tcl.c
View File

@ -244,6 +244,9 @@ static int SQLITE_TCLAPI xF5tApi(
{ "xGetAuxdataInt", 1, "CLEAR" }, /* 15 */
{ "xPhraseForeach", 4, "IPHRASE COLVAR OFFVAR SCRIPT" }, /* 16 */
{ "xPhraseColumnForeach", 3, "IPHRASE COLVAR SCRIPT" }, /* 17 */
{ "xQueryToken", 2, "IPHRASE ITERM" }, /* 18 */
{ "xInstToken", 2, "IDX ITERM" }, /* 19 */
{ 0, 0, 0}
};
@ -500,6 +503,38 @@ static int SQLITE_TCLAPI xF5tApi(
break;
}
CASE(18, "xQueryToken") {
const char *pTerm = 0;
int nTerm = 0;
int iPhrase = 0;
int iTerm = 0;
if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR;
if( Tcl_GetIntFromObj(interp, objv[3], &iTerm) ) return TCL_ERROR;
rc = p->pApi->xQueryToken(p->pFts, iPhrase, iTerm, &pTerm, &nTerm);
if( rc==SQLITE_OK ){
Tcl_SetObjResult(interp, Tcl_NewStringObj(pTerm, nTerm));
}
break;
}
CASE(19, "xInstToken") {
const char *pTerm = 0;
int nTerm = 0;
int iIdx = 0;
int iTerm = 0;
if( Tcl_GetIntFromObj(interp, objv[2], &iIdx) ) return TCL_ERROR;
if( Tcl_GetIntFromObj(interp, objv[3], &iTerm) ) return TCL_ERROR;
rc = p->pApi->xInstToken(p->pFts, iIdx, iTerm, &pTerm, &nTerm);
if( rc==SQLITE_OK ){
Tcl_SetObjResult(interp, Tcl_NewStringObj(pTerm, nTerm));
}
break;
}
default:
assert( 0 );
break;
@ -1117,6 +1152,176 @@ static int SQLITE_TCLAPI f5tRegisterTok(
return TCL_OK;
}
typedef struct OriginTextCtx OriginTextCtx;
struct OriginTextCtx {
sqlite3 *db;
fts5_api *pApi;
};
typedef struct OriginTextTokenizer OriginTextTokenizer;
struct OriginTextTokenizer {
Fts5Tokenizer *pTok; /* Underlying tokenizer object */
fts5_tokenizer tokapi; /* API implementation for pTok */
};
/*
** Delete the OriginTextCtx object indicated by the only argument.
*/
static void f5tOrigintextTokenizerDelete(void *pCtx){
OriginTextCtx *p = (OriginTextCtx*)pCtx;
ckfree((char*)p);
}
static int f5tOrigintextCreate(
void *pCtx,
const char **azArg,
int nArg,
Fts5Tokenizer **ppOut
){
OriginTextCtx *p = (OriginTextCtx*)pCtx;
OriginTextTokenizer *pTok = 0;
void *pTokCtx = 0;
int rc = SQLITE_OK;
pTok = (OriginTextTokenizer*)sqlite3_malloc(sizeof(OriginTextTokenizer));
if( pTok==0 ){
rc = SQLITE_NOMEM;
}else if( nArg<1 ){
rc = SQLITE_ERROR;
}else{
/* Locate the underlying tokenizer */
rc = p->pApi->xFindTokenizer(p->pApi, azArg[0], &pTokCtx, &pTok->tokapi);
}
/* Create the new tokenizer instance */
if( rc==SQLITE_OK ){
rc = pTok->tokapi.xCreate(pTokCtx, &azArg[1], nArg-1, &pTok->pTok);
}
if( rc!=SQLITE_OK ){
sqlite3_free(pTok);
pTok = 0;
}
*ppOut = (Fts5Tokenizer*)pTok;
return rc;
}
static void f5tOrigintextDelete(Fts5Tokenizer *pTokenizer){
OriginTextTokenizer *p = (OriginTextTokenizer*)pTokenizer;
if( p->pTok ){
p->tokapi.xDelete(p->pTok);
}
sqlite3_free(p);
}
typedef struct OriginTextCb OriginTextCb;
struct OriginTextCb {
void *pCtx;
const char *pText;
int nText;
int (*xToken)(void *, int, const char *, int, int, int);
char *aBuf; /* Buffer to use */
int nBuf; /* Allocated size of aBuf[] */
};
static int xOriginToken(
void *pCtx, /* Copy of 2nd argument to xTokenize() */
int tflags, /* Mask of FTS5_TOKEN_* flags */
const char *pToken, /* Pointer to buffer containing token */
int nToken, /* Size of token in bytes */
int iStart, /* Byte offset of token within input text */
int iEnd /* Byte offset of end of token within input */
){
OriginTextCb *p = (OriginTextCb*)pCtx;
int ret = 0;
if( nToken==(iEnd-iStart) && 0==memcmp(pToken, &p->pText[iStart], nToken) ){
/* Token exactly matches document text. Pass it through as is. */
ret = p->xToken(p->pCtx, tflags, pToken, nToken, iStart, iEnd);
}else{
int nReq = nToken + 1 + (iEnd-iStart);
if( nReq>p->nBuf ){
sqlite3_free(p->aBuf);
p->aBuf = sqlite3_malloc(nReq*2);
if( p->aBuf==0 ) return SQLITE_NOMEM;
p->nBuf = nReq*2;
}
memcpy(p->aBuf, pToken, nToken);
p->aBuf[nToken] = '\0';
memcpy(&p->aBuf[nToken+1], &p->pText[iStart], iEnd-iStart);
ret = p->xToken(p->pCtx, tflags, p->aBuf, nReq, iStart, iEnd);
}
return ret;
}
static int f5tOrigintextTokenize(
Fts5Tokenizer *pTokenizer,
void *pCtx,
int flags, /* Mask of FTS5_TOKENIZE_* flags */
const char *pText, int nText,
int (*xToken)(void *, int, const char *, int, int, int)
){
OriginTextTokenizer *p = (OriginTextTokenizer*)pTokenizer;
OriginTextCb cb;
int ret;
memset(&cb, 0, sizeof(cb));
cb.pCtx = pCtx;
cb.pText = pText;
cb.nText = nText;
cb.xToken = xToken;
ret = p->tokapi.xTokenize(p->pTok,(void*)&cb,flags,pText,nText,xOriginToken);
sqlite3_free(cb.aBuf);
return ret;
}
/*
** sqlite3_fts5_register_origintext DB
**
** Description...
*/
static int SQLITE_TCLAPI f5tRegisterOriginText(
void * clientData,
Tcl_Interp *interp,
int objc,
Tcl_Obj *CONST objv[]
){
sqlite3 *db = 0;
fts5_api *pApi = 0;
int rc;
fts5_tokenizer tok = {0, 0, 0};
OriginTextCtx *pCtx = 0;
if( objc!=2 ){
Tcl_WrongNumArgs(interp, 1, objv, "DB");
return TCL_ERROR;
}
if( f5tDbAndApi(interp, objv[1], &db, &pApi) ) return TCL_ERROR;
pCtx = (OriginTextCtx*)ckalloc(sizeof(OriginTextCtx));
pCtx->db = db;
pCtx->pApi = pApi;
tok.xCreate = f5tOrigintextCreate;
tok.xDelete = f5tOrigintextDelete;
tok.xTokenize = f5tOrigintextTokenize;
rc = pApi->xCreateTokenizer(
pApi, "origintext", (void*)pCtx, &tok, f5tOrigintextTokenizerDelete
);
Tcl_ResetResult(interp);
if( rc!=SQLITE_OK ){
Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
return TCL_ERROR;
}
return TCL_OK;
}
/*
** Entry point.
*/
@ -1133,7 +1338,8 @@ int Fts5tcl_Init(Tcl_Interp *interp){
{ "sqlite3_fts5_may_be_corrupt", f5tMayBeCorrupt, 0 },
{ "sqlite3_fts5_token_hash", f5tTokenHash, 0 },
{ "sqlite3_fts5_register_matchinfo", f5tRegisterMatchinfo, 0 },
{ "sqlite3_fts5_register_fts5tokenize", f5tRegisterTok, 0 }
{ "sqlite3_fts5_register_fts5tokenize", f5tRegisterTok, 0 },
{ "sqlite3_fts5_register_origintext",f5tRegisterOriginText, 0 }
};
int i;
F5tTokenizerContext *pContext;

3
ext/fts5/fts5_test_tok.c
View File

@ -472,7 +472,8 @@ int sqlite3Fts5TestRegisterTok(sqlite3 *db, fts5_api *pApi){
0, /* xSavepoint */
0, /* xRelease */
0, /* xRollbackTo */
0 /* xShadowName */
0, /* xShadowName */
0 /* xIntegrity */
};
int rc; /* Return code */

87
ext/fts5/fts5_tokenize.c
View File

@ -229,6 +229,12 @@ static const unsigned char sqlite3Utf8Trans1[] = {
#endif /* ifndef SQLITE_AMALGAMATION */
#define FTS5_SKIP_UTF8(zIn) { \
if( ((unsigned char)(*(zIn++)))>=0xc0 ){ \
while( (((unsigned char)*zIn) & 0xc0)==0x80 ){ zIn++; } \
} \
}
typedef struct Unicode61Tokenizer Unicode61Tokenizer;
struct Unicode61Tokenizer {
unsigned char aTokenChar[128]; /* ASCII range token characters */
@ -1264,6 +1270,7 @@ static int fts5PorterTokenize(
typedef struct TrigramTokenizer TrigramTokenizer;
struct TrigramTokenizer {
int bFold; /* True to fold to lower-case */
int iFoldParam; /* Parameter to pass to Fts5UnicodeFold() */
};
/*
@ -1290,6 +1297,7 @@ static int fts5TriCreate(
}else{
int i;
pNew->bFold = 1;
pNew->iFoldParam = 0;
for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
const char *zArg = azArg[i+1];
if( 0==sqlite3_stricmp(azArg[i], "case_sensitive") ){
@ -1298,10 +1306,21 @@ static int fts5TriCreate(
}else{
pNew->bFold = (zArg[0]=='0');
}
}else if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
rc = SQLITE_ERROR;
}else{
pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0;
}
}else{
rc = SQLITE_ERROR;
}
}
if( pNew->iFoldParam!=0 && pNew->bFold==0 ){
rc = SQLITE_ERROR;
}
if( rc!=SQLITE_OK ){
fts5TriDelete((Fts5Tokenizer*)pNew);
pNew = 0;
@ -1324,40 +1343,62 @@ static int fts5TriTokenize(
TrigramTokenizer *p = (TrigramTokenizer*)pTok;
int rc = SQLITE_OK;
char aBuf[32];
char *zOut = aBuf;
int ii;
const unsigned char *zIn = (const unsigned char*)pText;
const unsigned char *zEof = &zIn[nText];
u32 iCode;
int aStart[3]; /* Input offset of each character in aBuf[] */
UNUSED_PARAM(unusedFlags);
while( 1 ){
char *zOut = aBuf;
int iStart = zIn - (const unsigned char*)pText;
const unsigned char *zNext;
/* Populate aBuf[] with the characters for the first trigram. */
for(ii=0; ii<3; ii++){
do {
aStart[ii] = zIn - (const unsigned char*)pText;
READ_UTF8(zIn, zEof, iCode);
if( iCode==0 ) break;
zNext = zIn;
if( zIn<zEof ){
if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, 0);
if( iCode==0 ) return SQLITE_OK;
if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, p->iFoldParam);
}while( iCode==0 );
WRITE_UTF8(zOut, iCode);
READ_UTF8(zIn, zEof, iCode);
if( iCode==0 ) break;
}else{
break;
}
if( zIn<zEof ){
if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, 0);
WRITE_UTF8(zOut, iCode);
/* At the start of each iteration of this loop:
**
** aBuf: Contains 3 characters. The 3 characters of the next trigram.
** zOut: Points to the byte following the last character in aBuf.
** aStart[3]: Contains the byte offset in the input text corresponding
** to the start of each of the three characters in the buffer.
*/
assert( zIn<=zEof );
while( 1 ){
int iNext; /* Start of character following current tri */
const char *z1;
/* Read characters from the input up until the first non-diacritic */
do {
iNext = zIn - (const unsigned char*)pText;
READ_UTF8(zIn, zEof, iCode);
if( iCode==0 ) break;
if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, 0);
if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, p->iFoldParam);
}while( iCode==0 );
/* Pass the current trigram back to fts5 */
rc = xToken(pCtx, 0, aBuf, zOut-aBuf, aStart[0], iNext);
if( iCode==0 || rc!=SQLITE_OK ) break;
/* Remove the first character from buffer aBuf[]. Append the character
** with codepoint iCode. */
z1 = aBuf;
FTS5_SKIP_UTF8(z1);
memmove(aBuf, z1, zOut - z1);
zOut -= (z1 - aBuf);
WRITE_UTF8(zOut, iCode);
}else{
break;
}
rc = xToken(pCtx, 0, aBuf, zOut-aBuf, iStart, iStart + zOut-aBuf);
if( rc!=SQLITE_OK ) break;
zIn = zNext;
/* Update the aStart[] array */
aStart[0] = aStart[1];
aStart[1] = aStart[2];
aStart[2] = iNext;
}
return rc;
@ -1380,8 +1421,10 @@ int sqlite3Fts5TokenizerPattern(
){
if( xCreate==fts5TriCreate ){
TrigramTokenizer *p = (TrigramTokenizer*)pTok;
if( p->iFoldParam==0 ){
return p->bFold ? FTS5_PATTERN_LIKE : FTS5_PATTERN_GLOB;
}
}
return FTS5_PATTERN_NONE;
}

5
ext/fts5/fts5_vocab.c
View File

@ -629,7 +629,7 @@ static int fts5VocabFilterMethod(
if( pEq ){
zTerm = (const char *)sqlite3_value_text(pEq);
nTerm = sqlite3_value_bytes(pEq);
f = 0;
f = FTS5INDEX_QUERY_NOTOKENDATA;
}else{
if( pGe ){
zTerm = (const char *)sqlite3_value_text(pGe);
@ -783,7 +783,8 @@ int sqlite3Fts5VocabInit(Fts5Global *pGlobal, sqlite3 *db){
/* xSavepoint */ 0,
/* xRelease */ 0,
/* xRollbackTo */ 0,
/* xShadowName */ 0
/* xShadowName */ 0,
/* xIntegrity */ 0
};
void *p = (void*)pGlobal;

34
ext/fts5/test/fts5_common.tcl
View File

@ -61,6 +61,12 @@ proc fts5_test_collist {cmd} {
set res
}
proc fts5_collist {cmd iPhrase} {
set res [list]
$cmd xPhraseColumnForeach $iPhrase c { lappend res $c }
set res
}
proc fts5_test_columnsize {cmd} {
set res [list]
for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
@ -69,6 +75,10 @@ proc fts5_test_columnsize {cmd} {
set res
}
proc fts5_columntext {cmd iCol} {
$cmd xColumnText $iCol
}
proc fts5_test_columntext {cmd} {
set res [list]
for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
@ -125,6 +135,13 @@ proc fts5_test_queryphrase {cmd} {
set res
}
proc fts5_queryphrase {cmd iPhrase} {
set cnt [list]
for {set j 0} {$j < [$cmd xColumnCount]} {incr j} { lappend cnt 0 }
$cmd xQueryPhrase $iPhrase [list test_queryphrase_cb cnt]
set cnt
}
proc fts5_test_phrasecount {cmd} {
$cmd xPhraseCount
}
@ -154,6 +171,9 @@ proc fts5_aux_test_functions {db} {
fts5_test_queryphrase
fts5_test_phrasecount
fts5_columntext
fts5_queryphrase
fts5_collist
} {
sqlite3_fts5_create_function $db $f $f
}
@ -438,6 +458,20 @@ proc detail_is_none {} { detail_check ; expr {$::detail == "none"} }
proc detail_is_col {} { detail_check ; expr {$::detail == "col" } }
proc detail_is_full {} { detail_check ; expr {$::detail == "full"} }
proc foreach_tokenizer_mode {prefix script} {
set saved $::testprefix
foreach {d mapping} {
"" {}
"-origintext" {, tokenize="origintext unicode61", tokendata=1}
} {
set s [string map [list %TOKENIZER% $mapping] $script]
set ::testprefix "$prefix$d"
reset_db
sqlite3_fts5_register_origintext db
uplevel $s
}
set ::testprefix $saved
}
#-------------------------------------------------------------------------
# Convert a poslist of the type returned by fts5_test_poslist() to a

75
ext/fts5/test/fts5aa.test
View File

@ -22,6 +22,7 @@ ifcapable !fts5 {
}
foreach_detail_mode $::testprefix {
foreach_tokenizer_mode $::testprefix {
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, c);
@ -44,7 +45,7 @@ do_execsql_test 1.1 {
#
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%);
CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL% %TOKENIZER%);
}
do_execsql_test 2.1 {
INSERT INTO t1 VALUES('a b c', 'd e f');
@ -65,14 +66,17 @@ foreach w {a b c d e f} {
do_execsql_test 2.4 {
INSERT INTO t1(t1) VALUES('integrity-check');
}
PRAGMA integrity_check;
PRAGMA integrity_check(t1);
} {ok ok}
#-------------------------------------------------------------------------
#
reset_db
sqlite3_fts5_register_origintext db
do_execsql_test 3.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
}
foreach {i x y} {
1 {g f d b f} {h h e i a}
@ -88,14 +92,16 @@ foreach {i x y} {
} {
do_execsql_test 3.$i.1 { INSERT INTO t1 VALUES($x, $y) }
do_execsql_test 3.$i.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
do_execsql_test 3.$i.3 { PRAGMA integrity_check(t1) } ok
if {[set_test_counter errors]} break
}
#-------------------------------------------------------------------------
#
reset_db
sqlite3_fts5_register_origintext db
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
foreach {i x y} {
@ -118,8 +124,9 @@ foreach {i x y} {
#-------------------------------------------------------------------------
#
reset_db
sqlite3_fts5_register_origintext db
do_execsql_test 5.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
foreach {i x y} {
@ -135,15 +142,16 @@ foreach {i x y} {
10 {ddd abcde dddd dd c} {dddd c c d abcde}
} {
do_execsql_test 5.$i.1 { INSERT INTO t1 VALUES($x, $y) }
do_execsql_test 5.$i.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
do_execsql_test 5.$i.2 { PRAGMA integrity_check(t1) } ok
if {[set_test_counter errors]} break
}
#-------------------------------------------------------------------------
#
reset_db
sqlite3_fts5_register_origintext db
do_execsql_test 6.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
@ -178,6 +186,7 @@ do_execsql_test 6.6 {
#-------------------------------------------------------------------------
#
reset_db
sqlite3_fts5_register_origintext db
expr srand(0)
do_execsql_test 7.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x,y,z);
@ -219,6 +228,7 @@ for {set i 1} {$i <= 10} {incr i} {
#-------------------------------------------------------------------------
#
reset_db
sqlite3_fts5_register_origintext db
do_execsql_test 8.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x, prefix="1,2,3");
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
@ -233,6 +243,7 @@ do_execsql_test 8.1 {
#-------------------------------------------------------------------------
#
reset_db
sqlite3_fts5_register_origintext db
expr srand(0)
@ -277,8 +288,9 @@ for {set i 1} {$i <= 10} {incr i} {
#-------------------------------------------------------------------------
#
reset_db
sqlite3_fts5_register_origintext db
do_execsql_test 10.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
}
set d10 {
1 {g f d b f} {h h e i a}
@ -311,19 +323,19 @@ do_execsql_test 10.4.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
#-------------------------------------------------------------------------
#
do_catchsql_test 11.1 {
CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank, detail=%DETAIL%);
CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank, detail=%DETAIL% %TOKENIZER%);
} {1 {reserved fts5 column name: rank}}
do_catchsql_test 11.2 {
CREATE VIRTUAL TABLE rank USING fts5(a, b, c, detail=%DETAIL%);
CREATE VIRTUAL TABLE rank USING fts5(a, b, c, detail=%DETAIL% %TOKENIZER%);
} {1 {reserved fts5 table name: rank}}
do_catchsql_test 11.3 {
CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rowid, detail=%DETAIL%);
CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rowid, detail=%DETAIL% %TOKENIZER%);
} {1 {reserved fts5 column name: rowid}}
#-------------------------------------------------------------------------
#
do_execsql_test 12.1 {
CREATE VIRTUAL TABLE t2 USING fts5(x,y, detail=%DETAIL%);
CREATE VIRTUAL TABLE t2 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
} {}
do_catchsql_test 12.2 {
@ -338,8 +350,9 @@ do_test 12.3 {
#-------------------------------------------------------------------------
#
reset_db
sqlite3_fts5_register_origintext db
do_execsql_test 13.1 {
CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
INSERT INTO t1(rowid, x) VALUES(1, 'o n e'), (2, 't w o');
} {}
@ -362,8 +375,9 @@ do_execsql_test 13.6 {
#-------------------------------------------------------------------------
#
reset_db
sqlite3_fts5_register_origintext db
do_execsql_test 14.1 {
CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%);
CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL% %TOKENIZER%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
WITH d(x,y) AS (
SELECT NULL, 'xyz xyz xyz xyz xyz xyz'
@ -446,8 +460,9 @@ do_catchsql_test 16.2 {
#-------------------------------------------------------------------------
#
reset_db
sqlite3_fts5_register_origintext db
do_execsql_test 17.1 {
CREATE VIRTUAL TABLE b2 USING fts5(x, detail=%DETAIL%);
CREATE VIRTUAL TABLE b2 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
INSERT INTO b2 VALUES('a');
INSERT INTO b2 VALUES('b');
INSERT INTO b2 VALUES('c');
@ -463,8 +478,9 @@ do_test 17.2 {
if {[string match n* %DETAIL%]==0} {
reset_db
sqlite3_fts5_register_origintext db
do_execsql_test 17.3 {
CREATE VIRTUAL TABLE c2 USING fts5(x, y, detail=%DETAIL%);
CREATE VIRTUAL TABLE c2 USING fts5(x, y, detail=%DETAIL% %TOKENIZER%);
INSERT INTO c2 VALUES('x x x', 'x x x');
SELECT rowid FROM c2 WHERE c2 MATCH 'y:x';
} {1}
@ -473,8 +489,9 @@ if {[string match n* %DETAIL%]==0} {
#-------------------------------------------------------------------------
#
reset_db
sqlite3_fts5_register_origintext db
do_execsql_test 17.1 {
CREATE VIRTUAL TABLE uio USING fts5(ttt, detail=%DETAIL%);
CREATE VIRTUAL TABLE uio USING fts5(ttt, detail=%DETAIL% %TOKENIZER%);
INSERT INTO uio VALUES(NULL);
INSERT INTO uio SELECT NULL FROM uio;
INSERT INTO uio SELECT NULL FROM uio;
@ -521,8 +538,8 @@ do_execsql_test 17.9 {
#--------------------------------------------------------------------
#
do_execsql_test 18.1 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
CREATE VIRTUAL TABLE t2 USING fts5(c, d, detail=%DETAIL%);
CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL% %TOKENIZER%);
CREATE VIRTUAL TABLE t2 USING fts5(c, d, detail=%DETAIL% %TOKENIZER%);
INSERT INTO t1 VALUES('abc*', NULL);
INSERT INTO t2 VALUES(1, 'abcdefg');
}
@ -537,8 +554,9 @@ do_execsql_test 18.3 {
# fts5 table in the temp schema.
#
reset_db
sqlite3_fts5_register_origintext db
do_execsql_test 19.0 {
CREATE VIRTUAL TABLE temp.t1 USING fts5(x, detail=%DETAIL%);
CREATE VIRTUAL TABLE temp.t1 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
INSERT INTO t1 VALUES('x y z');
INSERT INTO t1 VALUES('w x 1');
SELECT rowid FROM t1 WHERE t1 MATCH 'x';
@ -548,8 +566,9 @@ do_execsql_test 19.0 {
# Test that 6 and 7 byte varints can be read.
#
reset_db
sqlite3_fts5_register_origintext db
do_execsql_test 20.0 {
CREATE VIRTUAL TABLE temp.tmp USING fts5(x, detail=%DETAIL%);
CREATE VIRTUAL TABLE temp.tmp USING fts5(x, detail=%DETAIL% %TOKENIZER%);
}
set ::ids [list \
0 [expr 1<<36] [expr 2<<36] [expr 1<<43] [expr 2<<43]
@ -567,7 +586,7 @@ do_test 20.1 {
#
do_execsql_test 21.0 {
CREATE TEMP TABLE t8(a, b);
CREATE VIRTUAL TABLE ft USING fts5(x, detail=%DETAIL%);
CREATE VIRTUAL TABLE ft USING fts5(x, detail=%DETAIL% %TOKENIZER%);
}
do_execsql_test 21.1 {
@ -578,7 +597,7 @@ do_execsql_test 21.1 {
}
do_execsql_test 22.0 {
CREATE VIRTUAL TABLE t9 USING fts5(x, detail=%DETAIL%);
CREATE VIRTUAL TABLE t9 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
INSERT INTO t9(rowid, x) VALUES(2, 'bbb');
BEGIN;
INSERT INTO t9(rowid, x) VALUES(1, 'aaa');
@ -593,7 +612,7 @@ do_execsql_test 22.1 {
#-------------------------------------------------------------------------
do_execsql_test 23.0 {
CREATE VIRTUAL TABLE t10 USING fts5(x, detail=%DETAIL%);
CREATE VIRTUAL TABLE t10 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
CREATE TABLE t11(x);
}
do_execsql_test 23.1 {
@ -605,7 +624,7 @@ do_execsql_test 23.2 {
#-------------------------------------------------------------------------
do_execsql_test 24.0 {
CREATE VIRTUAL TABLE t12 USING fts5(x, detail=%DETAIL%);
CREATE VIRTUAL TABLE t12 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
INSERT INTO t12 VALUES('aaaa');
}
do_execsql_test 24.1 {
@ -615,6 +634,9 @@ do_execsql_test 24.1 {
INSERT INTO t12 VALUES('aaaa');
END;
}
execsql_pp {
SELECT rowid, hex(block) FROM t12_data
}
do_execsql_test 24.2 {
INSERT INTO t12(t12) VALUES('integrity-check');
}
@ -624,7 +646,7 @@ do_execsql_test 24.3 {
#-------------------------------------------------------------------------
do_execsql_test 25.0 {
CREATE VIRTUAL TABLE t13 USING fts5(x, detail=%DETAIL%);
CREATE VIRTUAL TABLE t13 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
}
do_execsql_test 25.1 {
BEGIN;
@ -635,6 +657,7 @@ SELECT * FROM t13('BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB
}
}
}
expand_all_sql db

71
ext/fts5/test/fts5aux.test
View File

@ -307,5 +307,74 @@ do_catchsql_test 10.1.4 {
SELECT group_concat(firstcol(t1), '.') FROM t1 GROUP BY rowid
} {1 {unable to use function firstcol in the requested context}}
finish_test
#-------------------------------------------------------------------------
# Test that xInstCount() works from within an xPhraseQuery() callback.
#
reset_db
proc xCallback {cmd} {
incr ::hitcount [$cmd xInstCount]
return SQLITE_OK
}
proc fts5_hitcount {cmd} {
set ::hitcount 0
$cmd xQueryPhrase 0 xCallback
return $::hitcount
}
sqlite3_fts5_create_function db fts5_hitcount fts5_hitcount
do_execsql_test 11.1 {
CREATE VIRTUAL TABLE x1 USING fts5(z);
INSERT INTO x1 VALUES('one two three');
INSERT INTO x1 VALUES('one two one three one');
INSERT INTO x1 VALUES('one two three');
}
do_execsql_test 11.2 {
SELECT fts5_hitcount(x1) FROM x1('one') LIMIT 1;
} {5}
#-------------------------------------------------------------------------
# Test that xColumnText returns SQLITE_RANGE when it should.
#
reset_db
fts5_aux_test_functions db
do_execsql_test 12.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, c);
INSERT INTO t1 VALUES('one', 'two', 'three');
INSERT INTO t1 VALUES('one', 'one', 'one');
INSERT INTO t1 VALUES('two', 'two', 'two');
INSERT INTO t1 VALUES('three', 'three', 'three');
}
do_catchsql_test 12.1.1 {
SELECT fts5_columntext(t1, -1) FROM t1('two');
} {1 SQLITE_RANGE}
do_catchsql_test 12.1.2 {
SELECT fts5_columntext(t1, 3) FROM t1('two');
} {1 SQLITE_RANGE}
do_catchsql_test 12.1.2 {
SELECT fts5_columntext(t1, 1) FROM t1('one AND two');
} {0 two}
do_catchsql_test 12.2.1 {
SELECT fts5_queryphrase(t1, -1) FROM t1('one AND two');
} {1 SQLITE_RANGE}
do_catchsql_test 12.2.2 {
SELECT fts5_queryphrase(t1, 2) FROM t1('one AND two');
} {1 SQLITE_RANGE}
do_catchsql_test 12.2.3 {
SELECT fts5_queryphrase(t1, 1) FROM t1('one AND two');
} {0 {{1 2 1}}}
do_catchsql_test 12.3.1 {
SELECT fts5_collist(t1, -1) FROM t1('one AND two');
} {1 SQLITE_RANGE}
do_catchsql_test 12.3.2 {
SELECT fts5_collist(t1, 2) FROM t1('one AND two');
} {1 SQLITE_RANGE}
do_catchsql_test 12.3.3 {
SELECT fts5_collist(t1, 1) FROM t1('one AND two');
} {0 1}
finish_test

40
ext/fts5/test/fts5conflict.test
View File

@ -65,4 +65,44 @@ do_execsql_test 2.1 {
INSERT INTO fts_idx(fts_idx) VALUES('integrity-check');
}
#-------------------------------------------------------------------------
# Tests for OR IGNORE conflict handling.
#
reset_db
foreach_detail_mode $::testprefix {
do_execsql_test 3.0 {
CREATE VIRTUAL TABLE t1 USING fts5(xyz, detail=%DETAIL%);
BEGIN;
INSERT INTO t1(rowid, xyz) VALUES(13, 'thirteen documents');
INSERT INTO t1(rowid, xyz) VALUES(14, 'fourteen documents');
INSERT INTO t1(rowid, xyz) VALUES(15, 'fifteen documents');
COMMIT;
}
set db_cksum [cksum]
foreach {tn sql} {
1 {
INSERT OR IGNORE INTO t1(rowid, xyz) VALUES(14, 'new text');
}
2 {
UPDATE OR IGNORE t1 SET rowid=13 WHERE rowid=15;
}
3 {
INSERT OR IGNORE INTO t1(rowid, xyz)
SELECT 13, 'some text'
UNION ALL
SELECT 14, 'some text'
UNION ALL
SELECT 15, 'some text'
}
} {
do_execsql_test 3.1.$tn.1 $sql
do_test 3.1.$tn.2 { cksum } $db_cksum
}
}
finish_test

36
ext/fts5/test/fts5content.test
View File

@ -293,5 +293,39 @@ do_catchsql_test 7.2.5 {
SELECT * FROM t1('abc') ORDER BY rank;
} {1 {recursively defined fts5 content table}}
finish_test
#---------------------------------------------------------------------------
# Check that if the content table is a view, and that view contains an
# error, a reasonable error message is returned if the user tries to
# read from the view via the fts5 table.
#
reset_db
do_execsql_test 8.1 {
CREATE VIEW a1 AS
SELECT 1 AS r, text_value(1) AS t
UNION ALL
SELECT 2 AS r, text_value(2) AS t;
CREATE VIRTUAL TABLE t1 USING fts5(t, content='a1', content_rowid='r');
}
foreach {tn sql} {
1 "SELECT * FROM t1"
2 "INSERT INTO t1(t1) VALUES('rebuild')"
3 "SELECT * FROM t1 WHERE rowid=1"
} {
do_catchsql_test 8.2.$tn $sql {1 {no such function: text_value}}
}
proc text_value {i} {
if {$i==1} { return "one" }
if {$i==2} { return "two" }
return "many"
}
db func text_value text_value
do_execsql_test 8.3.1 { SELECT * FROM t1 } {one two}
do_execsql_test 8.3.2 { INSERT INTO t1(t1) VALUES('rebuild') }
do_execsql_test 8.3.3 { SELECT * FROM t1 WHERE rowid=1 } {one}
do_execsql_test 8.3.4 { SELECT rowid FROM t1('two') } {2}
finish_test

1
ext/fts5/test/fts5contentless.test
View File

@ -268,4 +268,3 @@ do_execsql_test 8.2 {
} {}
finish_test

1
ext/fts5/test/fts5contentless2.test
View File

@ -205,4 +205,3 @@ foreach {tn step} {
finish_test

1
ext/fts5/test/fts5contentless3.test
View File

@ -193,4 +193,3 @@ do_execsql_test 3.7 {
finish_test

1
ext/fts5/test/fts5contentless4.test
View File

@ -245,4 +245,3 @@ do_execsql_test 4.3 {
}
finish_test

1
ext/fts5/test/fts5contentless5.test
View File

@ -56,4 +56,3 @@ foreach {tn up err} {
}
finish_test

4
ext/fts5/test/fts5corrupt.test
View File

@ -48,6 +48,10 @@ do_test 1.3 {
}
catchsql { INSERT INTO t1(t1) VALUES('integrity-check') }
} {1 {database disk image is malformed}}
do_execsql_test 1.3b {
PRAGMA integrity_check(t1);
} {{malformed inverted index for FTS5 table main.t1}}
do_test 1.4 {
db_restore_and_reopen

3
ext/fts5/test/fts5corrupt2.test
View File

@ -167,6 +167,9 @@ foreach {tn hdr} {
do_test 3.$tn.$tn2.2 {
catchsql { INSERT INTO x3(x3) VALUES('integrity-check') }
} {1 {database disk image is malformed}}
do_execsql_test 3.$tn.$tn2.3 {
PRAGMA integrity_check(x3);
} {{malformed inverted index for FTS5 table main.x3}}
}
execsql ROLLBACK

570
ext/fts5/test/fts5corrupt5.test
View File

@ -880,6 +880,576 @@ do_catchsql_test 5.4 {
UPDATE t1 SET content=randomblob(500);
} {1 {database disk image is malformed}}
#-------------------------------------------------------------------------
reset_db
do_test 6.0 {
sqlite3 db {}
db deserialize [decode_hexdb {
.open --hexdb
| size 32768 pagesize 4096 filename crash-42fa37b694d45a.db
| page 1 offset 0
| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3.
| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 07 .....@ ........
| 96: 00 00 00 00 0d 00 00 00 07 0d d2 00 0f c4 0f 6d ...............m
| 112: 0f 02 0e ab 0e 4e 0d f6 0d d2 00 00 00 00 00 00 .....N..........
| 3536: 00 00 22 07 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet
| 3552: 32 74 32 07 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE
| 3568: 20 74 32 28 78 29 56 06 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta
| 3584: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c
| 3600: 6f 6e 66 69 67 06 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB
| 3616: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k
| 3632: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v)
| 3648: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 05 WITHOUT ROWID[.
| 3664: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d
| 3680: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize
| 3696: 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't
| 3712: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN
| 3728: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE
| 3744: 59 2c 20 73 7a 20 42 4c 4f 42 29 55 04 06 17 21 Y, sz BLOB)U...!
| 3760: 21 01 77 74 61 62 6c 65 74 31 5f 63 6f 6e 74 65 !.wtablet1_conte
| 3776: 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 04 43 52 45 ntt1_content.CRE
| 3792: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f ATE TABLE 't1_co
| 3808: 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 45 ntent'(id INTEGE
| 3824: 52 20 50 52 49 4d 41 52 49 20 4b 45 59 2c 20 63 R PRIMARI KEY, c
| 3840: 30 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 0)i.......-table
| 3856: 74 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 t1_idxt1_idx.CRE
| 3872: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 ATE TABLE 't1_id
| 3888: 78 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 x'(segid, term,
| 3904: 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 pgno, PRIMARY KE
| 3920: 59 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 Y(segid, term))
| 3936: 57 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 WITHOUT ROWIDU..
| 3952: 17 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 ......tablet1_da
| 3968: 74 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 tat1_data.CREATE
| 3984: 20 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 TABLE 't1_data'
| 4000: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM
| 4016: 41 52 b9 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 AR. KEY, block B
| 4032: 4c 4f 42 29 3a 01 06 17 11 11 08 63 74 61 62 6c LOB):......ctabl
| 4048: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54 et1t1CREATE VIRT
| 4064: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49 UAL TABLE t1 USI
| 4080: 4e 47 20 66 74 73 35 28 63 6f 6e 74 65 6e 74 29 NG fts5(content)
| page 2 offset 4096
| 0: 0d 00 00 00 03 0f bd 00 0f e8 0f ef 0f bd f0 00 ................
| 16: 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 24 84 80 .............$..
| 4032: 80 80 80 01 03 00 4e 00 10 00 1e 06 30 61 62 61 ......N.....0aba
| 4048: 63 6c 01 02 02 04 02 66 74 02 5f 02 04 04 6e 64 cl.....ft._...nd
| 4064: 6f 6e 02 02 02 04 0a 07 05 01 03 00 10 03 03 0f on..............
| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 11 ...$............
| page 3 offset 8192
| 0: 0a 00 00 00 01 0f 00 01 00 00 00 00 00 00 00 00 ................
| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................
| page 4 offset 12288
| 0: 0d 00 00 00 03 0f e0 00 0f f6 0f ec 0f e0 00 00 ................
| 4064: 0a 03 03 00 1b 61 62 61 6e 64 6f 6e 08 02 03 00 .....abandon....
| 4080: 17 61 62 61 66 74 08 01 03 00 17 61 62 61 63 6b .abaft.....aback
| page 5 offset 16384
| 0: 0d 00 00 00 03 0f ee 00 0f fa 0f 00 00 00 00 00 ................
| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 03 ................
| 4080: 03 00 0e 01 04 02 03 00 0e 01 04 01 03 00 0e 01 ................
| page 6 offset 20480
| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................
| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version.
| page 7 offset 24576
| 0: 0d 00 00 10 03 0f d6 00 0f 00 00 00 00 00 00 00 ................
| 4048: 00 00 00 00 00 00 09 03 02 1b 72 65 62 75 69 6c ..........rebuil
| 4064: 64 11 02 02 2b 69 6e 74 65 67 72 69 74 79 2d 63 d...+integrity-c
| 4080: 68 65 63 6b 0a 01 02 1d 6f 70 74 69 6d 00 00 00 heck....optim...
| page 8 offset 28672
| 0: 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
| end crash-42fa37b694d45a.db
}]} {}
do_execsql_test 6.1 {
INSERT INTO t1(t1,rank) VALUES('secure-delete',1);
}
do_catchsql_test 6.2 {
UPDATE t1 SET content=randomblob(500) WHERE t1;
} {1 {constraint failed}}
#-------------------------------------------------------------------------
reset_db
do_test 7.0 {
sqlite3 db {}
db deserialize [decode_hexdb {
.open --hexdb
| size 40960 pagesize 4096 filename crash-d8b4a99207c10b.db
| page 1 offset 0
| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3.
| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 0a .....@ ........
| 32: 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 04 ................
| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................
| 96: 00 00 00 00 0d 00 00 00 0d 0b 62 00 0f 97 0f 40 ..........b....@
| 112: 0e d5 0e 75 0e 18 0d c0 0d 66 0d 0f 0c a4 0c 44 ...u.....f.....D
| 128: 0b ec 0b a7 0b 62 00 00 00 00 00 00 00 00 00 00 .....b..........
| 2912: 00 00 43 0d 06 17 11 11 08 75 74 61 62 6c 65 74 ..C......utablet
| 2928: 34 74 34 43 52 45 41 54 45 20 56 49 52 54 55 41 4t4CREATE VIRTUA
| 2944: 4c 20 54 41 42 4c 45 20 74 34 20 55 53 49 4e 47 L TABLE t4 USING
| 2960: 20 66 74 73 35 76 6f 63 61 62 28 27 74 32 27 2c fts5vocab('t2',
| 2976: 20 27 72 6f 77 27 29 43 0c 06 17 11 11 08 75 74 'row')C......ut
| 2992: 61 62 6c 65 74 33 74 33 43 52 45 41 54 45 20 56 ablet3t3CREATE V
| 3008: 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 33 20 IRTUAL TABLE t3
| 3024: 55 53 49 4e 47 20 66 74 73 35 76 6f 63 61 62 28 USING fts5vocab(
| 3040: 27 74 31 27 2c 20 27 72 6f 77 27 29 56 0b 06 17 't1', 'row')V...
| 3056: 1f 1f 01 7d 74 61 62 6c 65 74 32 5f 63 6f 6e 66 ....tablet2_conf
| 3072: 69 67 74 32 5f 63 6f 6e 66 69 67 0a 43 52 45 41 igt2_config.CREA
| 3088: 54 45 20 54 41 42 4c 45 20 27 74 32 5f 63 6f 6e TE TABLE 't2_con
| 3104: 66 69 67 27 28 6b 20 50 52 49 4d 41 52 59 20 4b fig'(k PRIMARY K
| 3120: 45 59 2c 20 76 29 20 57 49 54 48 4f 55 54 20 52 EY, v) WITHOUT R
| 3136: 4f 57 49 44 5e 0a 07 17 21 21 01 81 07 74 61 62 OWID^...!!...tab
| 3152: 6c 65 74 32 5f 63 6f 6e 74 65 6e 74 74 32 5f 63 let2_contentt2_c
| 3168: 6f 6e 74 65 6e 74 09 43 52 45 41 54 45 20 54 41 ontent.CREATE TA
| 3184: 42 4c 45 20 27 74 32 5f 63 6f 6e 74 65 6e 74 27 BLE 't2_content'
| 3200: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM
| 3216: 41 52 59 20 4b 45 59 2c 20 63 30 2c 20 63 31 2c ARY KEY, c0, c1,
| 3232: 20 63 32 29 69 09 07 17 19 19 01 81 2d 74 61 62 c2)i.......-tab
| 3248: 6c 65 74 32 5f 69 64 78 74 32 5f 69 64 78 08 43 let2_idxt2_idx.C
| 3264: 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 32 5f REATE TABLE 't2_
| 3280: 69 64 78 27 28 73 65 67 69 64 2c 20 74 65 72 6d idx'(segid, term
| 3296: 2c 20 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 , pgno, PRIMARY
| 3312: 4b 45 59 28 73 65 67 69 64 2c 20 74 65 72 6d 29 KEY(segid, term)
| 3328: 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 55 ) WITHOUT ROWIDU
| 3344: 08 07 17 1b 1b 01 81 01 74 61 62 6c 65 74 32 5f ........tablet2_
| 3360: 64 61 74 61 74 32 5f 64 61 74 61 07 43 52 45 41 datat2_data.CREA
| 3376: 54 45 20 54 41 42 4c 45 20 27 74 32 5f 64 61 74 TE TABLE 't2_dat
| 3392: 61 27 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 a'(id INTEGER PR
| 3408: 49 4d 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b IMARY KEY, block
| 3424: 20 42 4c 4f 42 29 58 07 07 17 11 11 08 81 1d 74 BLOB)X........t
| 3440: 61 62 6c 65 74 32 74 32 43 52 45 41 54 45 20 56 ablet2t2CREATE V
| 3456: 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 32 20 IRTUAL TABLE t2
| 3472: 55 53 49 4e 47 20 66 74 73 35 28 27 61 27 2c 5b USING fts5('a',[
| 3488: 62 5d 2c 22 63 22 2c 64 65 74 61 69 6c 3d 6e 6f b],.c.,detail=no
| 3504: 6e 65 2c 63 6f 6c 75 6d 6e 73 69 7a 65 3d 30 29 ne,columnsize=0)
| 3520: 56 06 06 17 1f 1f 01 7d 74 61 62 6c 65 74 31 5f V.......tablet1_
| 3536: 63 6f 6e 66 69 67 74 31 5f 63 6f 6e 66 69 67 06 configt1_config.
| 3552: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 CREATE TABLE 't1
| 3568: 5f 63 6f 6e 66 69 67 27 28 6b 20 50 52 49 4d 41 _config'(k PRIMA
| 3584: 52 59 20 4b 45 59 2c 20 76 29 20 57 49 54 48 4f RY KEY, v) WITHO
| 3600: 55 54 20 52 4f 57 49 44 5b 05 07 17 21 21 01 81 UT ROWID[...!!..
| 3616: 01 74 61 62 6c 65 74 31 5f 64 6f 63 73 69 7a 65 .tablet1_docsize
| 3632: 74 31 5f 64 6f 63 73 69 7a 65 05 43 52 45 41 54 t1_docsize.CREAT
| 3648: 45 20 54 41 42 4c 45 20 27 74 31 5f 64 6f 63 73 E TABLE 't1_docs
| 3664: 69 7a 65 27 28 69 64 20 49 4e 54 45 47 45 52 20 ize'(id INTEGER
| 3680: 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 73 7a 20 PRIMARY KEY, sz
| 3696: 42 4c 4f 42 29 5e 04 07 17 21 21 01 81 07 74 61 BLOB)^...!!...ta
| 3712: 62 6c 65 74 31 5f 63 6f 6e 74 65 6e 74 74 31 5f blet1_contentt1_
| 3728: 63 6f 6e 74 65 6e 74 04 43 52 45 41 54 45 20 54 content.CREATE T
| 3744: 41 42 4c 45 20 27 74 31 5f 63 6f 6e 74 65 6e 74 ABLE 't1_content
| 3760: 27 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 '(id INTEGER PRI
| 3776: 4d 41 52 59 20 4b 45 59 2c 20 63 30 2c 20 63 31 MARY KEY, c0, c1
| 3792: 2c 20 63 32 29 69 03 07 17 19 19 01 81 2d 74 61 , c2)i.......-ta
| 3808: 62 6c 65 74 31 5f 69 64 78 74 31 5f 69 64 78 03 blet1_idxt1_idx.
| 3824: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 CREATE TABLE 't1
| 3840: 5f 69 64 78 27 28 73 65 67 69 64 2c 20 74 65 72 _idx'(segid, ter
| 3856: 6d 2c 20 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 m, pgno, PRIMARY
| 3872: 20 4b 45 59 28 73 65 67 69 64 2c 20 74 65 72 6d KEY(segid, term
| 3888: 29 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 )) WITHOUT ROWID
| 3904: 55 02 07 17 1b 1b 01 81 01 74 61 62 6c 65 74 31 U........tablet1
| 3920: 5f 64 61 74 61 74 31 5f 64 61 74 61 02 43 52 45 _datat1_data.CRE
| 3936: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 64 61 ATE TABLE 't1_da
| 3952: 74 61 27 28 69 64 20 49 4e 54 45 47 45 52 20 50 ta'(id INTEGER P
| 3968: 52 49 4d 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 RIMARY KEY, bloc
| 3984: 6b 20 42 4c 4f 42 29 67 01 07 17 11 11 08 81 3b k BLOB)g.......;
| 4000: 74 61 62 6c 65 74 31 74 31 43 52 45 41 54 45 20 tablet1t1CREATE
| 4016: 56 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 31 VIRTUAL TABLE t1
| 4032: 20 55 53 49 4e 47 20 66 74 73 35 28 61 2c 62 20 USING fts5(a,b
| 4048: 75 6e 69 6e 64 65 78 65 64 2c 63 2c 74 6f 6b 65 unindexed,c,toke
| 4064: 6e 69 7a 65 3d 22 70 6f 72 74 65 72 20 61 73 63 nize=.porter asc
| 4080: 69 69 22 2c 74 6f 6b 65 6e 64 61 74 61 3d 31 29 ii.,tokendata=1)
| page 2 offset 4096
| 0: 0d 0f 68 00 05 0f 13 00 0f e6 0f 13 0f a8 0f 7c ..h............|
| 16: 0f 2a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 .*..............
| 3856: 00 00 00 15 0a 03 00 30 00 00 00 00 01 03 03 00 .......0........
| 3872: 03 01 01 01 02 01 01 03 01 01 37 8c 80 80 80 80 ..........7.....
| 3888: 01 03 00 74 00 00 00 2e 02 30 61 03 02 02 01 01 ...t.....0a.....
| 3904: 62 03 02 03 01 01 63 03 02 04 01 01 67 03 06 01 b.....c.....g...
| 3920: 02 02 01 01 68 03 06 01 02 03 01 01 69 03 06 01 ....h.......i...
| 3936: 02 04 04 06 06 06 08 08 0f ef 00 14 2a 00 00 00 ............*...
| 3952: 00 01 02 02 00 02 01 01 01 02 01 01 25 88 80 80 ............%...
| 3968: 80 80 01 03 00 50 00 00 00 1f 02 30 67 02 08 02 .....P.....0g...
| 3984: 01 02 02 01 01 68 02 08 03 01 02 03 01 01 69 02 .....h........i.
| 4000: 08 04 01 02 04 04 09 09 37 84 80 80 80 7f f1 03 ........7.......
| 4016: 00 74 00 00 00 2e 02 30 61 01 02 02 01 01 62 01 .t.....0a.....b.
| 4032: 02 03 01 01 63 01 02 04 01 01 67 01 06 01 02 02 ....c.....g.....
| 4048: 01 01 68 01 06 01 02 03 01 01 69 01 06 01 02 04 ..h.......i.....
| 4064: 04 06 06 06 08 08 07 01 03 00 14 03 09 00 09 00 ................
| 4080: 00 00 11 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............
| page 3 offset 8192
| 0: 0a 00 00 00 03 0f ec 00 0f fa 0f f3 0f ec 00 00 ................
| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 06 04 01 0c ................
| 4080: 01 03 02 06 04 01 0c 01 02 02 05 04 09 0c 01 02 ................
| page 4 offset 12288
| 0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00 ................
| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03 ................
| 4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67 .....a b cg h ig
| 4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69 h i.......g h i
| 4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17 a b cg h i......
| 4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69 .a b cd e fg h i
| page 5 offset 16384
| 0: 0d 00 00 00 03 0f e8 00 0f f8 0f f0 0f e8 00 00 ................
| 4064: 00 00 00 00 00 00 00 00 06 03 03 00 12 03 00 03 ................
| 4080: 06 02 03 00 12 03 00 03 06 01 03 00 12 03 00 03 ................
| page 6 offset 20480
| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................
| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version.
| page 7 offset 24576
| 0: 0d 00 00 00 03 0f 9e 00 0f e6 0f ef 0f 9e 00 00 ................
| 3984: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 84 ..............A.
| 4000: 80 80 80 80 01 04 00 81 06 00 00 00 34 02 30 61 ............4.0a
| 4016: 01 01 01 01 01 62 01 01 01 01 01 63 01 01 01 01 .....b.....c....
| 4032: 01 64 01 01 01 65 01 01 01 66 01 01 01 67 01 01 .d...e...f...g..
| 4048: 01 01 01 68 01 01 01 01 01 69 01 01 01 04 06 06 ...h.....i......
| 4064: 06 04 04 04 06 06 07 01 03 00 14 03 09 09 09 0f ................
| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............
| page 8 offset 28672
| 0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00 ................
| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................
| page 9 offset 32768
| 0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00 ................
| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03 ................
| 4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67 .....a b cg h ig
| 4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69 h i.......g h i
| 4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17 a b cg h i......
| 4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69 .a b cd e fg h i
| page 10 offset 36864
| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................
| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version.
| end crash-d8b4a99207c10b.db
}]} {}
do_catchsql_test 7.1 {
SELECT snippet(t1, -1, '.', '..', '[', ']'),
highlight(t1, 2, '[', ']')
FROM t1('g + h')
WHERE rank MATCH 'bm25(1.0, 1.0)' ORDER BY rank;
} {1 {database disk image is malformed}}
#-------------------------------------------------------------------------
reset_db
do_test 8.0 {
sqlite3 db {}
db deserialize [decode_hexdb {
.open --hexdb
| size 20480 pagesize 4096 filename crash-d57c01958e48ab.db
| page 1 offset 0
| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3.
| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 05 .....@ ........
| 32: 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 04 ................
| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................
| 96: 00 00 00 00 0d 00 00 00 05 0e 10 00 0f 97 0f 40 ...............@
| 112: 0e d5 0e 68 0e 10 01 00 00 00 00 00 00 00 00 00 ...h............
| 3600: 56 05 06 17 1f 1f 01 7d 74 61 62 6c 65 74 31 5f V.......tablet1_
| 3616: 63 6f 6e 66 69 67 74 31 5f 63 6f 6e 66 69 67 05 configt1_config.
| 3632: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 CREATE TABLE 't1
| 3648: 5f 63 6f 6e 66 69 67 27 28 6b 20 50 52 49 4d 41 _config'(k PRIMA
| 3664: 52 59 20 4b 45 59 2c 20 76 29 20 57 49 54 48 4f RY KEY, v) WITHO
| 3680: 55 54 20 52 4f 57 49 44 6b 04 07 17 21 21 01 81 UT ROWIDk...!!..
| 3696: 21 74 61 62 6c 65 74 31 5f 64 6f 63 73 69 7a 65 !tablet1_docsize
| 3712: 74 31 5f 64 6f 63 73 69 7a 65 04 43 52 45 41 54 t1_docsize.CREAT
| 3728: 45 20 54 41 42 4c 45 20 27 74 31 5f 64 6f 63 73 E TABLE 't1_docs
| 3744: 69 7a 65 27 28 69 64 20 49 4e 54 45 47 45 52 20 ize'(id INTEGER
| 3760: 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 73 7a 20 PRIMARY KEY, sz
| 3776: 42 4c 4f 42 2c 20 6f 72 69 67 69 6e 20 49 4e 54 BLOB, origin INT
| 3792: 45 47 45 52 29 69 03 07 17 19 19 01 81 2d 74 61 EGER)i.......-ta
| 3808: 62 6c 65 74 31 5f 69 64 78 74 31 5f 69 64 78 03 blet1_idxt1_idx.
| 3824: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 CREATE TABLE 't1
| 3840: 5f 69 64 78 27 28 73 65 67 69 64 2c 20 74 65 72 _idx'(segid, ter
| 3856: 6d 2c 20 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 m, pgno, PRIMARY
| 3872: 20 4b 45 59 28 73 65 67 69 64 2c 20 74 65 72 6d KEY(segid, term
| 3888: 29 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 )) WITHOUT ROWID
| 3904: 55 02 07 17 1b 1b 01 81 01 74 61 62 6c 65 74 31 U........tablet1
| 3920: 5f 64 61 74 61 74 31 5f 64 61 74 61 02 43 52 45 _datat1_data.CRE
| 3936: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 64 61 ATE TABLE 't1_da
| 3952: 74 61 27 28 69 64 20 49 4e 54 45 47 45 52 20 50 ta'(id INTEGER P
| 3968: 52 49 4d 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 RIMARY KEY, bloc
| 3984: 6b 20 42 4c 4f 42 29 67 01 07 17 11 11 08 81 3b k BLOB)g.......;
| 4000: 74 61 62 6c 65 74 31 74 31 43 52 45 41 54 45 20 tablet1t1CREATE
| 4016: 56 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 31 VIRTUAL TABLE t1
| 4032: 20 55 53 49 4e 47 20 66 74 73 35 28 61 2c 20 62 USING fts5(a, b
| 4048: 2c 20 63 6f 6e 74 65 6e 74 3d 27 27 2c 20 63 6f , content='', co
| 4064: 6e 74 65 6e 74 6c 65 73 73 5f 64 65 6c 65 74 65 ntentless_delete
| 4080: 3d 31 2c 20 74 6f 6b 65 6e 64 61 74 61 3d 31 29 =1, tokendata=1)
| page 2 offset 4096
| 0: 0d 0f eb 00 03 0e 17 00 0f e2 0e 17 0e 31 00 00 .............1..
| 16: 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
| 3600: 00 00 00 00 00 00 00 18 0a 03 00 36 00 00 00 00 ...........6....
| 3616: ff 00 00 01 01 01 01 00 01 01 01 01 01 01 00 00 ................
| 3632: 07 83 29 84 80 80 80 80 01 04 00 86 56 00 00 01 ..).........V...
| 3648: 96 04 30 61 61 61 01 02 02 01 04 02 04 01 08 02 ..0aaa..........
| 3664: 04 04 04 01 10 02 04 04 04 04 04 04 04 01 20 02 .............. .
| 3680: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 01 ................
| 3696: 40 02 04 04 04 04 04 04 04 04 04 04 04 04 04 04 @...............
| 3712: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 ................
| 3728: 04 01 81 00 02 04 04 04 04 04 04 04 04 04 04 04 ................
| 3744: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 ................
| 3760: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 ................
| 3776: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 ................
| 3792: 04 04 04 04 02 02 62 63 01 06 01 01 02 01 03 62 ......bc.......b
| 3808: 62 62 02 02 03 01 04 03 06 01 08 03 06 06 06 01 bb..............
| 3824: 10 03 06 06 06 06 06 06 06 01 20 03 06 06 06 06 .......... .....
| 3840: 06 06 06 06 06 06 06 06 06 06 06 01 40 03 06 06 ............@...
| 3856: 06 06 06 06 06 06 06 06 06 06 06 06 06 06 06 06 ................
| 3872: 06 06 06 06 06 06 06 06 06 06 16 06 06 02 02 63 ...............c
| 3888: 64 02 06 01 01 02 01 03 63 63 63 03 02 05 01 04 d.......ccc.....
| 3904: 05 0a 01 08 05 0a 0a 0a 01 10 05 0a 0a 0a 0a 0a ................
| 3920: 0a 0a 01 20 05 0a 0a 0a 0a 0a 0a 0a 0a 0a 0a 0a ... ............
| 3936: 0a 0a 0a 0a 02 02 64 65 03 06 01 01 02 01 03 64 ......de.......d
| 3952: 64 64 04 02 09 01 04 09 12 01 08 09 12 12 12 01 dd..............
| 3968: 10 09 12 12 12 12 12 12 12 02 02 65 66 04 06 01 ...........ef...
| 3984: 01 02 01 03 65 65 65 05 02 11 01 04 11 22 01 08 ....eee.........
| 4000: 11 22 22 22 02 02 66 67 05 06 01 01 02 01 03 66 ......fg.......f
| 4016: 56 66 06 02 21 01 04 21 42 02 02 67 68 06 06 01 Vf..!..!B..gh...
| 4032: 01 02 cb 03 67 67 67 07 02 41 02 02 68 69 07 06 ....ggg..A..hi..
| 4048: 01 01 02 04 81 13 09 50 09 2e 09 1c 09 12 09 0c .......P........
| 4064: 09 08 07 01 03 00 14 07 81 77 07 00 00 00 15 22 .........w......
| 4080: 00 00 00 00 ff 00 00 01 00 00 00 00 00 00 05 0c ................
| page 3 offset 8192
| 0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00 ................
| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................
| page 4 offset 12288
| 0: 0d 00 00 00 07 0f c8 00 0f f8 0f f0 0f e8 0f e0 ................
| 16: 0f d8 0f d0 0f c8 00 00 00 00 00 00 00 00 00 00 ................
| 4032: 00 00 00 00 00 00 00 00 06 07 04 00 10 09 7f 01 ................
| 4048: 06 06 04 00 10 09 3f 01 06 05 04 00 10 09 1f 01 ......?.........
| 4064: 06 04 04 00 10 09 0f 01 06 03 04 00 10 09 07 01 ................
| 4080: 06 02 04 00 10 09 03 01 06 01 04 00 10 09 01 01 ................
| page 5 offset 16384
| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................
| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version.
| end crash-d57c01958e48ab.db
}]} {}
do_catchsql_test 8.1 {
SELECT rowid FROM t1('a* NOT ý‘') ;
} {0 {1 2 3 4 5 6 7}}
#-------------------------------------------------------------------------
reset_db
do_test 9.0 {
sqlite3 db {}
db deserialize [decode_hexdb {
.open --hexdb
| size 32768 pagesize 4096 filename crash-c76a16c24c8ba6.db
| page 1 offset 0
| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3.
| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 08 .....@ ........
| 32: 00 00 00 02 00 00 00 01 00 00 00 09 00 00 00 04 ................
| 96: 00 00 00 00 0d 0f c7 00 07 0d 92 00 0f 8d 0f 36 ...............6
| 112: 0e cb 0e 6b 0e 0e 0d b6 0d 92 0d 92 00 00 00 00 ...k............
| 3472: 00 00 22 08 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet
| 3488: 32 74 32 08 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE
| 3504: 20 74 32 28 78 29 56 07 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta
| 3520: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c
| 3536: 6f 6e 66 69 67 07 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB
| 3552: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k
| 3568: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v)
| 3584: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 06 WITHOUT ROWID[.
| 3600: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d
| 3616: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize
| 3632: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't
| 3648: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN
| 3664: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE
| 3680: 59 2c 20 73 7a 20 42 4c 4f 42 29 5e 05 07 17 21 Y, sz BLOB)^...!
| 3696: 21 01 81 07 74 61 62 6c 65 74 31 5f 63 6f 6e 74 !...tablet1_cont
| 3712: 65 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 05 43 52 entt1_content.CR
| 3728: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 EATE TABLE 't1_c
| 3744: 6f 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 ontent'(id INTEG
| 3760: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY,
| 3776: 63 30 2c 20 63 31 2c 20 63 32 29 69 04 07 17 19 c0, c1, c2)i....
| 3792: 19 01 81 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 ...-tablet1_idxt
| 3808: 31 5f 69 64 78 04 43 52 45 41 54 45 20 54 41 42 1_idx.CREATE TAB
| 3824: 4c 45 20 27 74 31 5f 69 64 78 27 28 73 65 67 69 LE 't1_idx'(segi
| 3840: 64 2c 20 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 d, term, pgno, P
| 3856: 52 49 4d 41 52 59 20 4b 45 59 28 73 65 67 69 64 RIMARY KEY(segid
| 3872: 2c 20 74 65 72 6d 29 29 20 57 49 54 48 4f 55 54 , term)) WITHOUT
| 3888: 20 52 4f 57 49 44 55 03 07 17 1b 1b 01 81 01 74 ROWIDU........t
| 3904: 61 62 6c 65 74 31 5f 64 61 74 61 74 31 5f 64 61 ablet1_datat1_da
| 3920: 74 61 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 ta.CREATE TABLE
| 3936: 27 74 31 5f 64 61 74 61 27 28 69 64 20 49 4e 54 't1_data'(id INT
| 3952: 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 EGER PRIMARY KEY
| 3968: 2c 20 62 6c 6f 63 6b 20 42 4c 4f 42 29 38 02 06 , block BLOB)8..
| 3984: 17 11 11 08 5f 74 61 62 6c 65 74 31 74 31 43 52 ...._tablet1t1CR
| 4000: 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 EATE VIRTUAL TAB
| 4016: 4c 45 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 LE t1 USING fts5
| 4032: 28 61 2c 62 2c 63 29 00 00 00 00 00 00 00 00 00 (a,b,c).........
| page 3 offset 8192
| 0: 0d 00 00 00 03 0c 94 00 0f e6 0f ef 0c 94 00 00 ................
| 3216: 00 00 00 00 86 4a 84 80 80 80 80 01 04 00 8d 18 .....J..........
| 3232: 00 00 03 2b 02 30 30 01 02 06 01 02 06 01 02 06 ...+.00.........
| 3248: 1f 02 03 01 02 03 01 02 03 01 08 32 30 31 36 30 ...........20160
| 3264: 36 30 39 01 02 07 01 02 07 01 02 07 01 01 34 01 609...........4.
| 3280: 02 05 01 02 05 01 02 05 01 01 35 01 02 04 01 02 ..........5.....
| 3296: 04 01 02 04 02 07 30 30 30 30 30 30 30 1c 02 04 ......0000000...
| 3312: 01 02 04 01 02 04 01 06 62 69 6e 61 72 79 03 06 ........binary..
| 3328: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................
| 3344: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................
| 3360: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................
| 3376: 03 06 01 02 02 03 06 01 02 02 01 08 63 6f 6d 70 ............comp
| 3392: 69 6c 65 72 01 02 02 01 02 02 01 02 02 01 06 64 iler...........d
| 3408: 62 73 74 61 74 07 02 03 01 02 03 01 02 03 02 04 bstat...........
| 3424: 65 62 75 67 04 02 02 01 02 02 01 02 02 01 06 65 ebug...........e
| 3440: 6e 61 62 6c 65 07 02 02 01 02 02 01 02 02 01 02 nable...........
| 3456: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 ................
| 3472: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................
| 3488: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................
| 3504: 02 01 02 02 02 08 78 74 65 6e 73 69 6f 6e 1f 02 ......xtension..
| 3520: 04 01 02 04 01 02 04 01 04 66 74 73 34 0a 02 03 .........fts4...
| 3536: 01 02 03 01 02 03 04 01 35 0d 02 03 01 02 03 01 ........5.......
| 3552: 02 03 01 03 67 63 63 01 02 03 01 02 03 01 02 03 ....gcc.........
| 3568: 02 06 65 6f 70 6f 6c 79 10 02 03 01 02 03 01 02 ..eopoly........
| 3584: 03 01 05 6a 73 6f 6e 31 13 02 03 01 02 03 01 02 ...json1........
| 3600: 03 01 04 6c 6f 61 64 1f 02 03 01 02 03 01 02 03 ...load.........
| 3616: 01 03 6d 61 78 1c 02 02 01 02 02 01 02 02 02 05 ..max...........
| 3632: 65 6d 6f 72 79 1c 02 03 01 02 03 01 02 03 04 04 emory...........
| 3648: 73 79 73 35 16 02 03 01 02 03 01 02 03 01 06 6e sys5...........n
| 3664: 6f 63 61 73 65 02 06 01 02 02 03 06 01 02 02 03 ocase...........
| 3680: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................
| 3696: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................
| 3712: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................
| 3728: 02 01 04 6f 6d 69 74 1f 02 02 01 02 02 01 02 02 ...omit.........
| 3744: 01 05 72 74 72 65 65 19 02 03 01 02 03 01 02 03 ..rtree.........
| 3760: 04 02 69 6d 01 06 01 02 02 03 06 01 02 02 03 06 ..im............
| 3776: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................
| 3792: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................
| 3808: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................
| 3824: 01 0a 74 68 72 65 61 64 73 61 66 65 03 57 34 56 ..threadsafe.W4V
| 3840: 94 64 91 46 85 84 04 76 74 61 62 07 02 04 01 02 .d.F...vtab.....
| 3856: 04 01 02 04 01 01 78 01 06 01 01 02 01 06 01 01 ......x.........
| 3872: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 10 02 ................
| 3888: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................
| 3904: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................
| 3920: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................
| 3936: 01 02 01 06 01 01 10 01 06 01 01 02 01 06 01 01 ................
| 3952: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................
| 3968: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................
| 3984: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................
| 4000: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................
| 4016: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................
| 4032: 02 01 06 01 01 02 01 06 01 01 02 04 15 13 0c 0c ................
| 4048: 12 44 13 11 0f 47 13 0f 0c 0e 11 10 0f 0e 10 0f .D...G..........
| 4064: 44 0f 10 40 15 0f 07 01 03 00 14 24 5a 24 24 0f D..@.......$Z$$.
| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............
| page 4 offset 12288
| 0: 0a 00 00 00 01 0f fa 00 00 00 00 00 00 00 00 00 ................
| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................
| page 5 offset 16384
| 0: 0d 00 00 00 24 0c 0a 00 0f d8 0f af 0f 86 0f 74 ....$..........t
| 16: 0f 61 0f 4e 0f 2f 0f 0f 0e ef 0e d7 0e be 0e a5 .a.N./..........
| 32: 0e 8d 0e 74 0e 5b 0e 40 0e 24 0e 08 0d ef 0d d5 ...t.[.@.$......
| 48: 0d bb 0d a0 0d 84 0d 68 0d 4f 0d 35 0d 1b 0c fb .......h.O.5....
| 64: 0c da 0c b9 0c 99 0c 78 0c 57 0c 3e 0c 24 0c 0a .......x.W.>.$..
| 3072: 00 00 00 00 00 00 00 00 00 00 18 24 05 00 25 0f ...........$..%.
| 3088: 19 54 48 52 45 41 44 53 41 46 45 3d 30 58 42 49 .THREADSAFE=0XBI
| 3104: 4e 41 52 59 18 23 05 00 25 0f 19 54 48 52 45 41 NARY.#..%..THREA
| 3120: 44 53 41 46 45 3d 30 58 4e 4f 43 41 53 45 17 22 DSAFE=0XNOCASE..
| 3136: 05 00 25 0f 17 54 48 52 45 41 44 53 31 46 45 3d ..%..THREADS1FE=
| 3152: 30 58 52 64 52 49 4d 1f 21 05 00 33 0f 19 4f 4d 0XRdRIM.!..3..OM
| 3168: 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 4f IT LOAD EXTENSIO
| 3184: 4e 58 42 49 4e 41 52 59 1f 20 05 00 33 0f 19 4f NXBINARY. ..3..O
| 3200: 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 MIT LOAD EXTENSI
| 3216: 4f 4e 58 4e 4f 43 41 53 45 1e 1f 05 00 33 0f 17 ONXNOCASE....3..
| 3232: 4f 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 OMIT LOAD EXTENS
| 3248: 49 4f 4e 58 52 54 52 49 4d 1f 1e 05 00 33 0f 19 IONXRTRIM....3..
| 3264: 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30 30 MAX MEMORY=50000
| 3280: 30 30 30 58 42 49 4e 41 52 59 1f 1d 05 00 33 0f 000XBINARY....3.
| 3296: 19 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30 .MAX MEMORY=5000
| 3312: 30 30 30 30 58 4e 4f 43 41 53 45 1e 1c 05 00 33 0000XNOCASE....3
| 3328: 0f 17 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 ..MAX MEMORY=500
| 3344: 30 30 30 30 30 58 52 54 52 49 4d 18 1b 05 00 25 00000XRTRIM....%
| 3360: 0f 19 45 4e 41 42 4c 45 20 52 54 52 45 45 58 42 ..ENABLE RTREEXB
| 3376: 49 4e 41 52 59 18 1a 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB
| 3392: 4c 45 20 52 54 52 45 45 58 4e 4f 43 41 53 45 17 LE RTREEXNOCASE.
| 3408: 19 05 00 25 0f 17 45 4e 41 42 4c 45 20 52 54 52 ...%..ENABLE RTR
| 3424: 45 45 58 52 54 52 49 4d 1a 18 05 00 29 0f 19 45 EEXRTRIM....)..E
| 3440: 4e 41 42 4b 45 20 4d 45 4d 53 59 53 35 58 42 49 NABKE MEMSYS5XBI
| 3456: 4e 41 52 59 1a 17 05 00 29 0f 19 45 4e 41 42 4c NARY....)..ENABL
| 3472: 42 60 2d 45 4d 53 59 53 35 58 4e 4f 43 41 53 45 B`-EMSYS5XNOCASE
| 3488: 19 16 05 00 29 0f 17 45 4e 41 42 4c 45 20 4d 45 ....)..ENABLE ME
| 3504: 4d 53 59 53 35 58 52 54 52 49 4d 18 15 05 00 25 MSYS5XRTRIM....%
| 3520: 0f 19 45 4e 41 42 4c 45 20 4a 53 4f 4e 31 58 42 ..ENABLE JSON1XB
| 3536: 49 4e 41 52 59 18 14 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB
| 3552: 4c 45 20 4a 53 4f 4e 31 58 4e 4f 43 41 53 45 17 LE JSON1XNOCASE.
| 3568: 13 05 00 25 0f 17 45 4e 41 42 4c 45 20 4a 53 4f ...%..ENABLE JSO
| 3584: 4e 31 58 52 54 52 49 4d 1a 12 05 00 29 0f 19 45 N1XRTRIM....)..E
| 3600: 4e 41 42 4c 45 20 47 45 4f 50 4f 4c 59 58 42 49 NABLE GEOPOLYXBI
| 3616: 4e 41 52 59 1a 11 05 00 39 0f 19 45 4e 41 42 4c NARY....9..ENABL
| 3632: 45 20 47 45 4f 50 4f 4c 59 58 4e 4f 43 41 53 45 E GEOPOLYXNOCASE
| 3648: 19 10 05 00 29 0f 17 45 4e 41 42 4c 45 20 47 45 ....)..ENABLE GE
| 3664: 4f 50 4f 4c 59 58 52 54 52 49 4d 17 0f 05 00 23 OPOLYXRTRIM....#
| 3680: 0f 19 45 4e 41 42 4c 45 20 46 54 53 35 58 42 49 ..ENABLE FTS5XBI
| 3696: 4e 41 52 59 17 0e 05 00 23 0f 19 45 4e 41 42 4c NARY....#..ENABL
| 3712: 45 20 46 54 53 35 58 4e 4f 43 41 53 45 16 0d 05 E FTS5XNOCASE...
| 3728: 00 23 0f 17 45 4e 41 42 4c 45 20 46 54 53 35 58 .#..ENABLE FTS5X
| 3744: 52 54 52 49 4d 17 0c 05 00 23 0f 19 45 4e 41 42 RTRIM....#..ENAB
| 3760: 4c 45 20 46 54 53 34 58 42 49 4e 41 52 59 17 0b LE FTS4XBINARY..
| 3776: 05 00 23 0f 19 45 4e 41 42 4c 45 20 46 54 53 34 ..#..ENABLE FTS4
| 3792: 58 4e 4f 43 41 53 45 16 0a 05 00 23 0f 17 45 4e XNOCASE....#..EN
| 3808: 41 42 4c 45 20 46 54 53 34 58 52 54 52 49 4d 1e ABLE FTS4XRTRIM.
| 3824: 09 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS
| 3840: 54 41 54 20 56 54 41 42 58 42 49 4e 41 52 59 1e TAT VTABXBINARY.
| 3856: 08 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS
| 3872: 54 41 54 20 56 54 24 15 48 4e 4f 43 41 53 45 1d TAT VT$.HNOCASE.
| 3888: 07 05 00 31 0f 17 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS
| 3904: 54 41 54 20 56 54 41 42 58 52 54 52 49 4d 11 06 TAT VTABXRTRIM..
| 3920: 05 00 17 0f 19 44 45 42 55 47 58 42 49 4e 41 52 .....DEBUGXBINAR
| 3936: 59 11 05 05 00 17 0f 19 44 45 42 55 47 58 4e 4f Y.......DEBUGXNO
| 3952: 43 41 53 45 10 04 05 00 17 0f 17 44 45 42 55 47 CASE.......DEBUG
| 3968: 58 52 54 52 49 4d 27 03 05 00 43 0f 19 43 4f 4d XRTRIM'...C..COM
| 3984: 50 49 4c 45 52 3d 67 63 63 2d 35 2e 34 2e 30 20 PILER=gcc-5.4.0
| 4000: 32 30 31 36 30 36 30 39 58 42 49 4e 41 52 59 27 20160609XBINARY'
| 4016: 02 05 00 43 0f 19 43 4f 4d 50 49 4c 45 52 3c 67 ...C..COMPILER<g
| 4032: 63 63 2d 35 2e 34 2e 30 20 32 30 31 36 30 36 30 cc-5.4.0 2016060
| 4048: 39 58 4e 4f 43 41 53 45 26 01 05 00 43 0f 17 43 9XNOCASE&...C..C
| 4064: 4f 4d 50 49 4c 45 52 3d 67 63 63 2d 35 2e 34 2e OMPILER=gcc-5.4.
| 4080: 30 20 32 30 31 36 30 36 30 39 58 52 54 52 49 4d 0 20160609XRTRIM
| page 6 offset 20480
| 0: 0d 00 00 00 24 0e e0 00 0f f8 0f f0 0f e8 0f e0 ....$...........
| 16: 0f d8 0f d0 0f c8 0f c0 0f b8 0f b0 0f a8 0f a0 ................
| 32: 1f 98 0f 90 0f 88 0f 80 0f 78 0f 70 0f 68 0f 60 .........x.p.h.`
| 48: 0f 58 0f 50 0f 48 0f 40 0f 38 0f 30 0f 28 0f 20 .X.P.H.@.8.0.(.
| 64: 0f 18 0f 10 0f 08 0f 00 0e f8 0e f0 0e e8 0e e0 ................
| 3808: 06 24 03 00 12 02 01 01 06 23 03 00 12 02 01 01 .$.......#......
| 3824: 06 22 03 00 12 02 01 01 06 21 03 00 12 03 01 01 .........!......
| 3840: 06 20 03 00 12 03 01 01 06 1f 03 00 12 03 01 01 . ..............
| 3856: 06 1e 03 00 12 03 01 01 06 1d 03 00 12 03 01 01 ................
| 3872: 06 1c 03 00 12 03 01 01 06 1b 03 00 12 02 01 01 ................
| 3888: 06 1a 03 00 12 02 01 01 06 19 03 00 12 02 01 01 ................
| 3904: 06 18 03 00 12 02 01 01 06 17 03 00 12 02 01 01 ................
| 3920: 06 15 f3 00 12 02 01 01 06 15 03 00 12 02 01 01 ................
| 3936: 06 14 03 00 12 02 01 01 06 13 03 00 12 02 01 01 ................
| 3952: 06 12 03 00 12 02 01 01 06 11 03 00 12 02 01 01 ................
| 3968: 06 10 03 00 12 02 01 01 06 0f 03 00 12 02 01 01 ................
| 3984: 06 0e 03 00 12 02 01 01 06 0d 03 00 12 02 01 01 ................
| 4000: 06 0c 03 00 12 02 01 01 06 0b 03 00 12 02 01 01 ................
| 4016: 06 0a 03 00 12 02 01 01 06 09 03 00 12 03 01 01 ................
| 4032: 06 08 03 00 12 03 01 01 06 07 03 00 12 03 01 01 ................
| 4048: 06 06 03 00 12 01 01 01 06 05 03 00 12 01 01 01 ................
| 4064: 06 04 03 00 12 01 01 01 06 03 03 00 12 06 01 01 ................
| 4080: 06 02 03 00 12 06 01 01 06 01 03 00 12 06 01 01 ................
| page 7 offset 24576
| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................
| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version.
| page 8 offset 28672
| 0: 0d 00 00 00 03 0f d6 00 0f f4 0f e9 0f d6 00 00 ................
| 4048: 00 00 00 00 00 00 11 03 02 2b 69 6e 74 65 67 72 .........+integr
| 4064: 69 74 79 2d 63 68 65 63 6b 09 02 02 1b 72 65 62 ity-check....reb
| 4080: 75 69 6c 64 0a 01 02 1d 6f 70 74 69 00 00 00 00 uild....opti....
| end crash-c76a16c24c8ba6.db
}]} {}
#.testctrl prng_seed 1 db
#.testctrl internal_functions
#.testctrl json_selfcheck on
#
do_execsql_test 9.1 {
UPDATE t1 SET b=quote(zeroblob(current_date)) WHERE t1 MATCH 't*';
SAVEPOINT a;
UPDATE t1 SET b=quote(zeroblob(current_date)) WHERE t1 MATCH 't*';
INSERT INTO t1(t1,rank) VALUES('secure-delete',1);
}
do_catchsql_test 9.2 {
DELETE FROM t1;
} {1 {database disk image is malformed}}
sqlite3_fts5_may_be_corrupt 0
finish_test

14
ext/fts5/test/fts5fault8.test
View File

@ -57,7 +57,6 @@ foreach_detail_mode $testprefix {
} ;# foreach_detail_mode...
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE x2 USING fts5(a);
INSERT INTO x2(x2, rank) VALUES('crisismerge', 2);
@ -80,5 +79,18 @@ do_faultsim_test 4 -faults oom-* -prep {
faultsim_test_result {0 {}} {1 SQLITE_NOMEM}
}
set TMPDBERROR {1 {unable to open a temporary database file for storing temporary tables}}
do_faultsim_test 5 -faults oom-t* -prep {
faultsim_restore_and_reopen
execsql { PRAGMA temp_store = memory }
} -body {
execsql { PRAGMA integrity_check }
} -test {
if {[string match {*error code=7*} $testresult]==0} {
faultsim_test_result {0 ok} {1 SQLITE_NOMEM} $::TMPDBERROR
}
}
finish_test

76
ext/fts5/test/fts5faultG.test Normal file
View File

@ -0,0 +1,76 @@
# 2010 June 15
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
source [file join [file dirname [info script]] fts5_common.tcl]
source $testdir/malloc_common.tcl
set testprefix fts5faultG
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
set ::testprefix fts5faultG
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a);
INSERT INTO t1 VALUES('test renaming the table');
INSERT INTO t1 VALUES(' after it has been written');
INSERT INTO t1 VALUES(' actually other stuff instead');
}
faultsim_save_and_close
do_faultsim_test 1 -faults oom* -prep {
faultsim_restore_and_reopen
execsql {
BEGIN;
DELETE FROM t1 WHERE rowid=2;
}
} -body {
execsql {
DELETE FROM t1;
}
} -test {
catchsql { COMMIT }
faultsim_integrity_check
faultsim_test_result {0 {}}
}
reset_db
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, content=, contentless_delete=1);
BEGIN;
INSERT INTO t1 VALUES('here''s some text');
INSERT INTO t1 VALUES('useful stuff, text');
INSERT INTO t1 VALUES('what would we do without text!');
COMMIT;
}
faultsim_save_and_close
do_faultsim_test 2 -faults oom* -prep {
faultsim_restore_and_reopen
execsql {
BEGIN;
DELETE FROM t1 WHERE rowid=2;
}
} -body {
execsql {
INSERT INTO t1(t1) VALUES('optimize');
}
} -test {
faultsim_integrity_check
faultsim_test_result {0 {}}
}
finish_test

150
ext/fts5/test/fts5faultH.test Normal file
View File

@ -0,0 +1,150 @@
# 2010 June 15
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
source [file join [file dirname [info script]] fts5_common.tcl]
source $testdir/malloc_common.tcl
set testprefix fts5faultG
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
set ::testprefix fts5faultH
sqlite3_fts5_register_origintext db
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(
x, tokenize="origintext unicode61", tokendata=1
);
BEGIN;
INSERT INTO t1 VALUES('oNe tWo thRee');
INSERT INTO t1 VALUES('One Two Three');
INSERT INTO t1 VALUES('onE twO threE');
COMMIT;
BEGIN;
INSERT INTO t1 VALUES('one two three');
INSERT INTO t1 VALUES('one two three');
INSERT INTO t1 VALUES('one two three');
COMMIT;
}
do_faultsim_test 1 -faults oom* -prep {
} -body {
execsql {
SELECT rowid FROM t1('three');
}
} -test {
faultsim_integrity_check
faultsim_test_result {0 {1 2 3 4 5 6}}
}
reset_db
sqlite3_fts5_register_origintext db
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE t1 USING fts5(
x, tokenize="origintext unicode61", tokendata=1
);
INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
BEGIN;
INSERT INTO t1(rowid, x) VALUES(10, 'aaa bbb BBB');
INSERT INTO t1(rowid, x) VALUES(12, 'bbb bbb bbb');
INSERT INTO t1(rowid, x) VALUES(13, 'bbb bbb bbb');
INSERT INTO t1(rowid, x) VALUES(14, 'bbb BBB bbb');
INSERT INTO t1(rowid, x) VALUES(15, 'bbb bbb bbb');
INSERT INTO t1(rowid, x) VALUES(16, 'bbb bbb bbb');
INSERT INTO t1(rowid, x) VALUES(17, 'bbb bbb bbb');
INSERT INTO t1(rowid, x) VALUES(18, 'bbb bbb bbb');
INSERT INTO t1(rowid, x) VALUES(19, 'bbb bbb bbb');
INSERT INTO t1(rowid, x) VALUES(20, 'bbb bbb bbb');
INSERT INTO t1(rowid, x) VALUES(21, 'bbb bbb bbb');
INSERT INTO t1(rowid, x) VALUES(22, 'bbb bbb bbb');
INSERT INTO t1(rowid, x) VALUES(23, 'bbb bbb bbb');
INSERT INTO t1(rowid, x) VALUES(24, 'aaa bbb BBB');
COMMIT;
}
do_faultsim_test 2 -faults oom* -prep {
} -body {
execsql {
SELECT rowid FROM t1('BBB AND AAA');
}
} -test {
faultsim_integrity_check
faultsim_test_result {0 {10 24}}
}
reset_db
sqlite3_fts5_register_origintext db
do_execsql_test 3.0 {
CREATE VIRTUAL TABLE t1 USING fts5(
x, tokenize="origintext unicode61", tokendata=1
);
INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
INSERT INTO t1(rowid, x) VALUES(9, 'bbb Bbb BBB');
BEGIN;
INSERT INTO t1(rowid, x) VALUES(10, 'aaa bbb BBB');
INSERT INTO t1(rowid, x) VALUES(11, 'bbb Bbb BBB');
INSERT INTO t1(rowid, x) VALUES(12, 'bbb Bbb BBB');
INSERT INTO t1(rowid, x) VALUES(13, 'bbb Bbb BBB');
INSERT INTO t1(rowid, x) VALUES(14, 'bbb Bbb BBB');
INSERT INTO t1(rowid, x) VALUES(15, 'bbb Bbb BBB');
INSERT INTO t1(rowid, x) VALUES(16, 'bbb Bbb BBB');
INSERT INTO t1(rowid, x) VALUES(17, 'bbb Bbb BBB');
INSERT INTO t1(rowid, x) VALUES(18, 'bbb Bbb BBB');
INSERT INTO t1(rowid, x) VALUES(19, 'bbb Bbb BBB');
INSERT INTO t1(rowid, x) VALUES(20, 'bbb Bbb BBB');
INSERT INTO t1(rowid, x) VALUES(21, 'bbb Bbb BBB');
INSERT INTO t1(rowid, x) VALUES(22, 'bbb Bbb BBB');
INSERT INTO t1(rowid, x) VALUES(23, 'bbb Bbb BBB');
INSERT INTO t1(rowid, x) VALUES(24, 'bbb Bbb BBB');
INSERT INTO t1(rowid, x) VALUES(25, 'bbb Bbb BBB');
INSERT INTO t1(rowid, x) VALUES(26, 'bbb Bbb BBB');
INSERT INTO t1(rowid, x) VALUES(27, 'bbb Bbb BBB');
INSERT INTO t1(rowid, x) VALUES(28, 'bbb Bbb BBB');
INSERT INTO t1(rowid, x) VALUES(29, 'bbb Bbb BBB');
INSERT INTO t1(rowid, x) VALUES(30, 'bbb Bbb BBB');
INSERT INTO t1(rowid, x) VALUES(31, 'bbb Bbb BBB');
INSERT INTO t1(rowid, x) VALUES(32, 'bbb Bbb BBB');
INSERT INTO t1(rowid, x) VALUES(33, 'bbb Bbb BBB');
INSERT INTO t1(rowid, x) VALUES(34, 'bbb Bbb BBB');
INSERT INTO t1(rowid, x) VALUES(35, 'aaa bbb BBB');
COMMIT;
}
do_faultsim_test 3.1 -faults oom* -prep {
} -body {
execsql {
SELECT rowid FROM t1('BBB AND AAA');
}
} -test {
faultsim_integrity_check
faultsim_test_result {0 {10 35}}
}
do_faultsim_test 3.2 -faults oom* -prep {
} -body {
execsql {
SELECT count(*) FROM t1('BBB');
}
} -test {
faultsim_integrity_check
faultsim_test_result {0 27}
}
finish_test

64
ext/fts5/test/fts5integrity.test
View File

@ -77,6 +77,9 @@ do_catchsql_test 4.2 {
UPDATE aa_docsize SET sz = X'44' WHERE rowid = 3;
INSERT INTO aa(aa) VALUES('integrity-check');
} {1 {database disk image is malformed}}
do_execsql_test 4.2.1 {
PRAGMA integrity_check(aa);
} {{malformed inverted index for FTS5 table main.aa}}
do_catchsql_test 4.3 {
ROLLBACK;
@ -317,4 +320,65 @@ do_catchsql_test 10.5.3 {
INSERT INTO vt0(vt0) VALUES('integrity-check');
} {0 {}}
reset_db
proc slang {in} {return [string map {th d e eh} $in]}
db function slang -deterministic -innocuous slang
do_execsql_test 11.0 {
CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT, c TEXT AS (slang(b)));
INSERT INTO t1(b) VALUES('the quick fox jumps over the lazy brown dog');
SELECT c FROM t1;
} {{deh quick fox jumps ovehr deh lazy brown dog}}
do_execsql_test 11.1 {
CREATE VIRTUAL TABLE t2 USING fts5(content="t1", c);
INSERT INTO t2(t2) VALUES('rebuild');
SELECT rowid FROM t2 WHERE t2 MATCH 'deh';
} {1}
do_execsql_test 11.2 {
PRAGMA integrity_check(t2);
} {ok}
db close
sqlite3 db test.db
# FIX ME?
#
# FTS5 integrity-check does not care if the content table is unreadable or
# does not exist. It only looks for internal inconsistencies in the
# inverted index.
#
do_execsql_test 11.3 {
PRAGMA integrity_check(t2);
} {ok}
do_execsql_test 11.4 {
DROP TABLE t1;
PRAGMA integrity_check(t2);
} {ok}
#-------------------------------------------------------------------
reset_db
do_execsql_test 12.1 {
CREATE VIRTUAL TABLE x1 USING fts5(a, b);
INSERT INTO x1 VALUES('one', 'two');
INSERT INTO x1 VALUES('three', 'four');
INSERT INTO x1 VALUES('five', 'six');
}
do_execsql_test 12.2 {
PRAGMA integrity_check
} {ok}
db close
sqlite3 db test.db -readonly 1
explain_i {
PRAGMA integrity_check
}
do_execsql_test 12.3 {
PRAGMA integrity_check
} {ok}
finish_test

46
ext/fts5/test/fts5misc.test
View File

@ -44,12 +44,12 @@ do_catchsql_test 1.2.2 {
do_catchsql_test 1.3.1 {
SELECT highlight(t1, 4, '<b>', '</b>') FROM t1('*reads');
} {1 {no such cursor: 1}}
} {1 {no such cursor: 2}}
do_catchsql_test 1.3.2 {
SELECT a FROM t1
WHERE rank = (SELECT highlight(t1, 4, '<b>', '</b>') FROM t1('*reads'));
} {1 {no such cursor: 1}}
} {1 {no such cursor: 2}}
db close
sqlite3 db test.db
@ -91,7 +91,6 @@ do_execsql_test 2.2.1 {
INSERT INTO vt0(c0) VALUES ('xyz');
}
breakpoint
do_execsql_test 2.2.2 {
ALTER TABLE t0 RENAME TO t1;
}
@ -424,10 +423,12 @@ do_execsql_test -db db2 15.3 {
SAVEPOINT one;
} {}
do_execsql_test 15.4 END
do_test 15.4 {
do_test 15.5 {
list [catch { db2 eval COMMIT } msg] $msg
} {0 {}}
db2 close
#-------------------------------------------------------------------------
reset_db
forcedelete test.db2
@ -469,6 +470,8 @@ do_execsql_test -db db2 16.6 {
SELECT * FROM x1
} {abc def}
db2 close
#-------------------------------------------------------------------------
reset_db
do_execsql_test 17.1 {
@ -496,6 +499,41 @@ do_execsql_test 17.5 {
SELECT c0 FROM t0 WHERE c0 GLOB '*faul*';
} {assertionfaultproblem}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 18.0 {
BEGIN;
CREATE VIRTUAL TABLE t1 USING fts5(text);
ALTER TABLE t1 RENAME TO t2;
}
do_execsql_test 18.1 {
DROP TABLE t2;
}
do_execsql_test 18.2 {
COMMIT;
}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 19.0 {
CREATE VIRTUAL TABLE t1 USING fts5(text);
CREATE TABLE t2(text);
BEGIN;
INSERT INTO t1 VALUES('one');
INSERT INTO t1 VALUES('two');
INSERT INTO t1 VALUES('three');
INSERT INTO t1 VALUES('one');
INSERT INTO t1 VALUES('two');
INSERT INTO t1 VALUES('three');
SAVEPOINT one;
INSERT INTO t2 VALUES('one');
INSERT INTO t2 VALUES('two');
INSERT INTO t2 VALUES('three');
ROLLBACK TO one;
COMMIT;
}
finish_test

297
ext/fts5/test/fts5origintext.test Normal file
View File

@ -0,0 +1,297 @@
# 2014 Jan 08
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# Tests focused on phrase queries.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5origintext
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
foreach_detail_mode $testprefix {
sqlite3_fts5_register_origintext db
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE ft USING fts5(
x, tokenize="origintext unicode61", detail=%DETAIL%
);
CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance);
}
do_execsql_test 1.1 {
INSERT INTO ft VALUES('Hello world');
}
do_execsql_test 1.2 {
INSERT INTO ft(ft) VALUES('integrity-check');
}
proc b {x} { string map [list "\0" "."] $x }
db func b b
do_execsql_test 1.3 {
select b(term) from vocab;
} {
hello.Hello
world
}
do_execsql_test 1.4 {
SELECT rowid FROM ft('Hello');
} {1}
#-------------------------------------------------------------------------
reset_db
# Return a random integer between 0 and n-1.
#
proc random {n} {
expr {abs(int(rand()*$n))}
}
proc select_one {list} {
set n [llength $list]
lindex $list [random $n]
}
proc term {} {
set first_letter {
a b c d e f g h i j k l m n o p q r s t u v w x y z
A B C D E F G H I J K L M N O P Q R S T U V W X Y Z
}
set term [select_one $first_letter]
append term [random 100]
}
proc document {} {
set nTerm [expr [random 5] + 5]
set doc ""
for {set ii 0} {$ii < $nTerm} {incr ii} {
lappend doc [term]
}
set doc
}
db func document document
sqlite3_fts5_register_origintext db
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE ft USING fts5(
x, tokenize="origintext unicode61", detail=%DETAIL%
);
INSERT INTO ft(ft, rank) VALUES('pgsz', 128);
CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance);
}
do_test 2.1 {
for {set ii 0} {$ii < 500} {incr ii} {
execsql { INSERT INTO ft VALUES( document() ) }
}
} {}
do_execsql_test 2.2 {
INSERT INTO ft(ft) VALUES('integrity-check');
}
do_execsql_test 2.3 {
INSERT INTO ft(ft, rank) VALUES('merge', 16);
}
do_execsql_test 2.4 {
INSERT INTO ft(ft) VALUES('integrity-check');
}
do_execsql_test 2.5 {
INSERT INTO ft(ft) VALUES('optimize');
}
#-------------------------------------------------------------------------
reset_db
sqlite3_fts5_register_origintext db
do_execsql_test 3.0 {
CREATE VIRTUAL TABLE ft USING fts5(
x, tokenize="origintext unicode61", detail=%DETAIL%
);
CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance);
INSERT INTO ft(rowid, x) VALUES(1, 'hello');
INSERT INTO ft(rowid, x) VALUES(2, 'Hello');
INSERT INTO ft(rowid, x) VALUES(3, 'HELLO');
}
#proc b {x} { string map [list "\0" "."] $x }
#db func b b
#execsql_pp { SELECT b(term) FROM vocab }
do_execsql_test 3.1.1 { SELECT rowid FROM ft('hello') } 1
do_execsql_test 3.1.2 { SELECT rowid FROM ft('Hello') } 2
do_execsql_test 3.1.3 { SELECT rowid FROM ft('HELLO') } 3
do_execsql_test 3.2 {
CREATE VIRTUAL TABLE ft2 USING fts5(x,
tokenize="origintext unicode61",
tokendata=1,
detail=%DETAIL%
);
CREATE VIRTUAL TABLE vocab2 USING fts5vocab(ft2, instance);
INSERT INTO ft2(rowid, x) VALUES(1, 'hello');
INSERT INTO ft2(rowid, x) VALUES(2, 'Hello');
INSERT INTO ft2(rowid, x) VALUES(3, 'HELLO');
INSERT INTO ft2(rowid, x) VALUES(10, 'helloooo');
}
#proc b {x} { string map [list "\0" "."] $x }
#db func b b
#execsql_pp { SELECT b(term) FROM vocab }
do_execsql_test 3.3.1 { SELECT rowid FROM ft2('hello') } {1 2 3}
do_execsql_test 3.3.2 { SELECT rowid FROM ft2('Hello') } {1 2 3}
do_execsql_test 3.3.3 { SELECT rowid FROM ft2('HELLO') } {1 2 3}
do_execsql_test 3.3.4 { SELECT rowid FROM ft2('hello*') } {1 2 3 10}
#-------------------------------------------------------------------------
#
reset_db
sqlite3_fts5_register_origintext db
proc querytoken {cmd iPhrase iToken} {
set txt [$cmd xQueryToken $iPhrase $iToken]
string map [list "\0" "."] $txt
}
sqlite3_fts5_create_function db querytoken querytoken
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE ft USING fts5(
x, tokenize='origintext unicode61', tokendata=1, detail=%DETAIL%
);
INSERT INTO ft VALUES('one two three four');
}
do_execsql_test 4.1 {
SELECT rowid, querytoken(ft, 0, 0) FROM ft('TwO')
} {1 two.TwO}
do_execsql_test 4.2 {
SELECT rowid, querytoken(ft, 0, 0) FROM ft('one TWO ThreE')
} {1 one}
do_execsql_test 4.3 {
SELECT rowid, querytoken(ft, 1, 0) FROM ft('one TWO ThreE')
} {1 two.TWO}
if {"%DETAIL%"=="full"} {
# Phrase queries are only supported for detail=full.
#
do_execsql_test 4.4 {
SELECT rowid, querytoken(ft, 0, 2) FROM ft('"one TWO ThreE"')
} {1 three.ThreE}
do_catchsql_test 4.5 {
SELECT rowid, querytoken(ft, 0, 3) FROM ft('"one TWO ThreE"')
} {1 SQLITE_RANGE}
do_catchsql_test 4.6 {
SELECT rowid, querytoken(ft, 1, 0) FROM ft('"one TWO ThreE"')
} {1 SQLITE_RANGE}
do_catchsql_test 4.7 {
SELECT rowid, querytoken(ft, -1, 0) FROM ft('"one TWO ThreE"')
} {1 SQLITE_RANGE}
}
#-------------------------------------------------------------------------
#
reset_db
sqlite3_fts5_register_origintext db
proc insttoken {cmd iIdx iToken} {
set txt [$cmd xInstToken $iIdx $iToken]
string map [list "\0" "."] $txt
}
sqlite3_fts5_create_function db insttoken insttoken
fts5_aux_test_functions db
do_execsql_test 5.0 {
CREATE VIRTUAL TABLE ft USING fts5(
x, tokenize='origintext unicode61', tokendata=1, detail=%DETAIL%
);
INSERT INTO ft VALUES('one ONE One oNe oNE one');
}
do_execsql_test 5.1 {
SELECT insttoken(ft, 0, 0),
insttoken(ft, 1, 0),
insttoken(ft, 2, 0),
insttoken(ft, 3, 0),
insttoken(ft, 4, 0),
insttoken(ft, 5, 0)
FROM ft('one');
} {
one one.ONE one.One one.oNe one.oNE one
}
do_execsql_test 5.2 {
SELECT insttoken(ft, 1, 0) FROM ft('one');
} {
one.ONE
}
do_execsql_test 5.3 {
SELECT fts5_test_poslist(ft) FROM ft('one');
} {
{0.0.0 0.0.1 0.0.2 0.0.3 0.0.4 0.0.5}
}
#-------------------------------------------------------------------------
# Test the xInstToken() API with:
#
# * a non tokendata=1 table.
# * prefix queries.
#
reset_db
sqlite3_fts5_register_origintext db
do_execsql_test 6.0 {
CREATE VIRTUAL TABLE ft USING fts5(
x, y, tokenize='origintext unicode61', detail=%DETAIL%
);
INSERT INTO ft VALUES('One Two', 'Three two');
INSERT INTO ft VALUES('three Three', 'one One');
}
proc tokens {cmd} {
set ret [list]
for {set iTok 0} {$iTok < [$cmd xInstCount]} {incr iTok} {
set txt [$cmd xInstToken $iTok 0]
set txt [string map [list "\0" "."] $txt]
lappend ret $txt
}
set ret
}
sqlite3_fts5_create_function db tokens tokens
do_execsql_test 6.1 {
SELECT rowid, tokens(ft) FROM ft('One');
} {1 one.One 2 one.One}
do_execsql_test 6.2 {
SELECT rowid, tokens(ft) FROM ft('on*');
} {1 {{}} 2 {{} {}}}
do_execsql_test 6.3 {
SELECT rowid, tokens(ft) FROM ft('Three*');
} {1 {{}} 2 {{}}}
}
finish_test

146
ext/fts5/test/fts5origintext2.test Normal file
View File

@ -0,0 +1,146 @@
# 2014 Jan 08
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# Tests focused on phrase queries.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5origintext2
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
sqlite3_fts5_register_origintext db
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE ft USING fts5(
x, tokenize="origintext unicode61", tokendata=1
);
}
do_execsql_test 1.1 {
BEGIN;
INSERT INTO ft VALUES('Hello');
INSERT INTO ft VALUES('hello');
INSERT INTO ft VALUES('HELLO');
INSERT INTO ft VALUES('today');
INSERT INTO ft VALUES('today');
INSERT INTO ft VALUES('today');
INSERT INTO ft VALUES('World');
INSERT INTO ft VALUES('world');
INSERT INTO ft VALUES('WORLD');
COMMIT;
}
do_execsql_test 1.2 { SELECT rowid FROM ft('hello'); } {1 2 3}
do_execsql_test 1.3 { SELECT rowid FROM ft('today'); } {4 5 6}
do_execsql_test 1.4 { SELECT rowid FROM ft('world'); } {7 8 9}
do_execsql_test 1.5 {
SELECT count(*) FROM ft_data
} 3
do_execsql_test 1.6 {
DELETE FROM ft;
INSERT INTO ft(ft, rank) VALUES('pgsz', 64);
BEGIN;
WITH s(i) AS (
SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100
)
INSERT INTO ft SELECT 'Hello Hello Hello Hello Hello Hello Hello' FROM s;
INSERT INTO ft VALUES ('hELLO hELLO hELLO');
INSERT INTO ft VALUES('today today today today today today today');
INSERT INTO ft VALUES('today today today today today today today');
INSERT INTO ft VALUES('today today today today today today today');
INSERT INTO ft VALUES('today today today today today today today');
INSERT INTO ft VALUES('today today today today today today today');
INSERT INTO ft VALUES('today today today today today today today');
INSERT INTO ft VALUES('World World World World World World World');
INSERT INTO ft VALUES('world world world world world world world');
INSERT INTO ft VALUES('WORLD WORLD WORLD WORLD WORLD WORLD WORLD');
INSERT INTO ft VALUES('World World World World World World World');
INSERT INTO ft VALUES('world world world world world world world');
INSERT INTO ft VALUES('WORLD WORLD WORLD WORLD WORLD WORLD WORLD');
COMMIT;
}
do_execsql_test 1.7 {
SELECT count(*) FROM ft_data;
} 23
do_execsql_test 1.8 { SELECT rowid FROM ft('hello') WHERE rowid>100; } {101}
do_execsql_test 1.9 {
DELETE FROM ft;
INSERT INTO ft(ft) VALUES('optimize');
SELECT count(*) FROM ft_data;
} {2}
do_execsql_test 1.10 {
BEGIN;
INSERT INTO ft VALUES('Hello');
INSERT INTO ft VALUES('hello');
INSERT INTO ft VALUES('HELLO');
INSERT INTO ft VALUES('today');
INSERT INTO ft VALUES('today');
INSERT INTO ft VALUES('today');
INSERT INTO ft VALUES('World');
INSERT INTO ft VALUES('world');
INSERT INTO ft VALUES('WORLD');
}
do_execsql_test 1.11 { SELECT rowid FROM ft('hello'); } {1 2 3}
do_execsql_test 1.12 { SELECT rowid FROM ft('today'); } {4 5 6}
do_execsql_test 1.13 { SELECT rowid FROM ft('world'); } {7 8 9}
do_execsql_test 1.14 { SELECT rowid FROM ft('hello') ORDER BY rank; } {1 2 3}
#------------------------------------------------------------------------
reset_db
sqlite3_fts5_register_origintext db
proc tokens {cmd} {
set ret [list]
for {set iTok 0} {$iTok < [$cmd xInstCount]} {incr iTok} {
set txt [$cmd xInstToken $iTok 0]
set txt [string map [list "\0" "."] $txt]
lappend ret $txt
}
set ret
}
sqlite3_fts5_create_function db tokens tokens
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE x1 USING fts5(
v, tokenize="origintext unicode61", tokendata=1, detail=none
);
INSERT INTO x1 VALUES('xxx Xxx XXX yyy YYY yyy');
INSERT INTO x1 VALUES('xxx yyy xxx yyy yyy yyy');
}
do_execsql_test 2.1 {
SELECT tokens(x1) FROM x1('xxx');
} {
{xxx xxx.Xxx xxx.XXX} {xxx xxx}
}
do_execsql_test 2.2 {
UPDATE x1_content SET c0 = 'xxx xxX xxx yyy yyy yyy' WHERE id=1;
}
do_execsql_test 2.3 {
SELECT tokens(x1) FROM x1('xxx');
} {
{xxx {} xxx} {xxx xxx}
}
finish_test

101
ext/fts5/test/fts5origintext3.test Normal file
View File

@ -0,0 +1,101 @@
# 2023 November 22
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# Tests focused on phrase queries.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5origintext3
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
foreach_detail_mode $testprefix {
reset_db
sqlite3_fts5_register_origintext db
fts5_aux_test_functions db
proc insttoken {cmd iIdx iToken} {
set txt [$cmd xInstToken $iIdx $iToken]
string map [list "\0" "."] $txt
}
sqlite3_fts5_create_function db insttoken insttoken
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE ft USING fts5(
x, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL%
);
}
do_execsql_test 1.1 {
INSERT INTO ft VALUES('Hello world HELLO WORLD hello');
}
do_execsql_test 1.2 {
SELECT fts5_test_poslist(ft) FROM ft('hello');
} {{0.0.0 0.0.2 0.0.4}}
do_execsql_test 1.3 {
SELECT
insttoken(ft, 0, 0),
insttoken(ft, 1, 0),
insttoken(ft, 2, 0)
FROM ft('hello');
} {hello.Hello hello.HELLO hello}
do_execsql_test 1.4 {
SELECT
insttoken(ft, 0, 0),
insttoken(ft, 1, 0),
insttoken(ft, 2, 0)
FROM ft('hello') ORDER BY rank;
} {hello.Hello hello.HELLO hello}
do_execsql_test 1.5 {
CREATE VIRTUAL TABLE ft2 USING fts5(
x, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL%
);
INSERT INTO ft2(rowid, x) VALUES(1, 'ONE one two three ONE');
INSERT INTO ft2(rowid, x) VALUES(2, 'TWO one two three TWO');
INSERT INTO ft2(rowid, x) VALUES(3, 'THREE one two three THREE');
}
do_execsql_test 1.6 {
SELECT insttoken(ft2, 0, 0), rowid FROM ft2('three') ORDER BY rank;
} {three.THREE 3 three 1 three 2}
do_execsql_test 1.7 {
INSERT INTO ft2(rowid, x) VALUES(10, 'aaa bbb BBB');
INSERT INTO ft2(rowid, x) VALUES(12, 'bbb bbb bbb');
INSERT INTO ft2(rowid, x) VALUES(13, 'bbb bbb bbb');
INSERT INTO ft2(rowid, x) VALUES(14, 'bbb BBB bbb');
INSERT INTO ft2(rowid, x) VALUES(15, 'bbb bbb bbb');
INSERT INTO ft2(rowid, x) VALUES(16, 'bbb bbb bbb');
INSERT INTO ft2(rowid, x) VALUES(17, 'bbb bbb bbb');
INSERT INTO ft2(rowid, x) VALUES(18, 'bbb bbb bbb');
INSERT INTO ft2(rowid, x) VALUES(19, 'bbb bbb bbb');
INSERT INTO ft2(rowid, x) VALUES(20, 'bbb bbb bbb');
INSERT INTO ft2(rowid, x) VALUES(21, 'bbb bbb bbb');
INSERT INTO ft2(rowid, x) VALUES(22, 'bbb bbb bbb');
INSERT INTO ft2(rowid, x) VALUES(23, 'bbb bbb bbb');
INSERT INTO ft2(rowid, x) VALUES(24, 'aaa bbb BBB');
}
do_execsql_test 1.8 { SELECT rowid FROM ft2('aaa AND bbb'); } {10 24}
do_execsql_test 1.9 { SELECT rowid FROM ft2('bbb AND aaa'); } {10 24}
}
finish_test

80
ext/fts5/test/fts5origintext4.test Normal file
View File

@ -0,0 +1,80 @@
# 2023 November 22
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# Tests focused on phrase queries.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5origintext4
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
# The tests below verify that a doclist-index is used to limit the number
# of pages loaded into the cache. It does this by querying sqlite3_db_status()
# for the amount of memory used by the pager cache.
#
# memsubsys1 effectively limits the page-cache to 24 pages. Which masks
# the effect tested by the tests in this file. And "mmap" prevents the
# cache from being used, also preventing these tests from working.
#
if {[permutation]=="memsubsys1" || [permutation]=="mmap"} {
finish_test
return
}
sqlite3_fts5_register_origintext db
do_execsql_test 1.0 {
PRAGMA page_size = 4096;
CREATE VIRTUAL TABLE ft USING fts5(
x, tokenize="origintext unicode61", tokendata=1
);
}
do_execsql_test 1.1 {
BEGIN;
INSERT INTO ft SELECT 'the first thing';
WITH s(i) AS (
SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<90000
)
INSERT INTO ft SELECT 'The second thing' FROM s;
INSERT INTO ft SELECT 'the first thing';
COMMIT;
INSERT INTO ft(ft) VALUES('optimize');
}
foreach {tn sql expr} {
1 { SELECT rowid FROM ft('the') } {$mem > 250000}
2 { SELECT rowid FROM ft('first') } {$mem < 50000}
3 { SELECT rowid FROM ft('the first') } {$mem < 50000}
} {
db close
sqlite3 db test.db
sqlite3_fts5_register_origintext db
execsql $sql
do_test 1.2.$tn {
set mem [lindex [sqlite3_db_status db CACHE_USED 0] 1]
expr $expr
} 1
}
proc b {x} { string map [list "\0" "."] $x }
db func b b
# execsql_pp { SELECT segid, b(term), pgno from ft_idx }
finish_test

273
ext/fts5/test/fts5origintext5.test Normal file
View File

@ -0,0 +1,273 @@
# 2023 Dec 04
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# Tests for tables that use both tokendata=1 and contentless_delete=1.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5origintext
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
# Return a random integer between 0 and n-1.
#
proc random {n} { expr {abs(int(rand()*$n))} }
# Select an element of the list passed as the only argument at random and
# return it.
#
proc select_one {list} {
set n [llength $list]
lindex $list [random $n]
}
# Given a term that consists entirely of alphabet characters, return all
# permutations of the term using upper and lower case characters. e.g.
#
# "abc" -> {CBA cBA CbA cbA CBa cBa Cba cba}
#
proc casify {term {lRet {{}}}} {
if {$term==""} { return $lRet }
set t [string range $term 1 end]
set f1 [string toupper [string range $term 0 0]]
set f2 [string tolower [string range $term 0 0]]
set ret [list]
foreach x $lRet {
lappend ret "$x$f1"
lappend ret "$x$f2"
}
return [casify $t $ret]
}
proc vocab {} {
list abc def ghi jkl mno pqr stu vwx yza
}
# Return a random 3 letter term.
#
proc term {} {
if {[info exists ::expanded_vocab]==0} {
foreach v [vocab] { lappend ::expanded_vocab {*}[casify $v] }
}
select_one $::expanded_vocab
}
# Return a document - between 3 and 10 terms.
#
proc document {} {
set nTerm [expr [random 3] + 7]
set doc ""
for {set ii 0} {$ii < $nTerm} {incr ii} {
lappend doc [term]
}
set doc
}
db func document document
#-------------------------------------------------------------------------
expr srand(6)
set NDOC 200
set NLOOP 50
sqlite3_fts5_register_origintext db
proc tokens {cmd} {
set ret [list]
for {set iTok 0} {$iTok < [$cmd xInstCount]} {incr iTok} {
set txt [$cmd xInstToken $iTok 0]
set txt [string map [list "\0" "."] $txt]
lappend ret $txt
}
set ret
}
sqlite3_fts5_create_function db tokens tokens
proc rankfunc {cmd} {
$cmd xRowid
}
sqlite3_fts5_create_function db rankfunc rankfunc
proc ctrl_tokens {term args} {
set ret [list]
set term [string tolower $term]
foreach doc $args {
foreach a $doc {
if {[string tolower $a]==$term} {
if {$a==$term} {
lappend ret $a
} else {
lappend ret [string tolower $a].$a
}
}
}
}
set ret
}
db func ctrl_tokens ctrl_tokens
proc do_all_vocab_test {tn} {
foreach ::v [concat [vocab] nnn] {
set answer [execsql {
SELECT id, ctrl_tokens($::v, x) FROM ctrl WHERE x LIKE '%' || $::v || '%'
}]
do_execsql_test $tn.$::v.1 {
SELECT rowid, tokens(ft) FROM ft($::v)
} $answer
do_execsql_test $tn.$::v.2 {
SELECT rowid, tokens(ft) FROM ft($::v) ORDER BY rank
} $answer
}
}
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE ft USING fts5(
x, tokenize="origintext unicode61", content=, contentless_delete=1,
tokendata=1
);
CREATE TABLE ctrl(id INTEGER PRIMARY KEY, x TEXT);
INSERT INTO ft(ft, rank) VALUES('pgsz', 64);
INSERT INTO ft(ft, rank) VALUES('rank', 'rankfunc()');
}
do_test 1.1 {
for {set ii 0} {$ii < $NDOC} {incr ii} {
set doc [document]
execsql {
INSERT INTO ft(rowid, x) VALUES($ii, $doc);
INSERT INTO ctrl(id, x) VALUES($ii, $doc);
}
}
} {}
#execsql_pp { SELECT * FROM ctrl }
#execsql_pp { SELECT * FROM ft }
#fts5_aux_test_functions db
#execsql_pp { SELECT rowid, tokens(ft), fts5_test_poslist(ft) FROM ft('ghi'); }
do_all_vocab_test 1.2
for {set ii 0} {$ii < $NLOOP} {incr ii} {
set lRowid [execsql { SELECT id FROM ctrl WHERE random() % 2 }]
foreach r $lRowid {
execsql { DELETE FROM ft WHERE rowid = $r }
execsql { DELETE FROM ctrl WHERE rowid = $r }
set doc [document]
execsql { INSERT INTO ft(rowid, x) VALUES($r, $doc) }
execsql { INSERT INTO ctrl(id, x) VALUES($r, $doc) }
}
do_all_vocab_test 1.3.$ii
}
#-------------------------------------------------------------------------
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE ft2 USING fts5(
x, y, tokenize="origintext unicode61", content=, contentless_delete=1,
tokendata=1
);
CREATE TABLE ctrl2(id INTEGER PRIMARY KEY, x TEXT, y TEXT);
INSERT INTO ft2(ft2, rank) VALUES('pgsz', 64);
INSERT INTO ft2(ft2, rank) VALUES('rank', 'rankfunc()');
}
do_test 2.1 {
for {set ii 0} {$ii < $NDOC} {incr ii} {
set doc1 [document]
set doc2 [document]
execsql {
INSERT INTO ft2(rowid, x, y) VALUES($ii, $doc, $doc2);
INSERT INTO ctrl2(id, x, y) VALUES($ii, $doc, $doc2);
}
}
} {}
proc do_all_vocab_test2 {tn} {
foreach ::v [vocab] {
set answer [execsql {
SELECT id, ctrl_tokens($::v, x, y) FROM ctrl2
WHERE x LIKE '%' || $::v || '%' OR y LIKE '%' || $::v || '%';
}]
do_execsql_test $tn.$::v.1 {
SELECT rowid, tokens(ft2) FROM ft2($::v)
} $answer
do_execsql_test $tn.$::v.2 {
SELECT rowid, tokens(ft2) FROM ft2($::v) ORDER BY rank
} $answer
}
}
do_all_vocab_test2 2.2
for {set ii 0} {$ii < $NLOOP} {incr ii} {
set lRowid [execsql { SELECT id FROM ctrl2 WHERE random() % 2 }]
foreach r $lRowid {
execsql { DELETE FROM ft2 WHERE rowid = $r }
execsql { DELETE FROM ctrl2 WHERE rowid = $r }
set doc1 [document]
set doc2 [document]
execsql { INSERT INTO ft2(rowid, x, y) VALUES($r, $doc, $doc1) }
execsql { INSERT INTO ctrl2(id, x, y) VALUES($r, $doc, $doc2) }
}
do_all_vocab_test 2.3.$ii
}
#-------------------------------------------------------------------------
unset -nocomplain ::expanded_vocab
proc vocab {} {
list abcde fghij klmno
}
proc do_all_vocab_test3 {tn} {
foreach ::v [concat [vocab] nnn] {
set answer [execsql {
SELECT rowid, ctrl_tokens($::v, w) FROM ctrl3 WHERE w LIKE '%' || $::v || '%'
}]
do_execsql_test $tn.$::v.1 {
SELECT rowid, tokens(ft3) FROM ft3($::v)
} $answer
do_execsql_test $tn.$::v.2 {
SELECT rowid, tokens(ft3) FROM ft3($::v) ORDER BY rank
} $answer
}
}
do_execsql_test 3.0 {
CREATE VIRTUAL TABLE ft3 USING fts5(
w, tokenize="origintext unicode61", content=, contentless_delete=1,
tokendata=1
);
INSERT INTO ft3(ft3, rank) VALUES('rank', 'rankfunc()');
CREATE TABLE ctrl3(w);
}
do_execsql_test 3.1 {
WITH s(i) AS (
SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<2
)
INSERT INTO ctrl3 SELECT document() FROM s;
INSERT INTO ft3(rowid, w) SELECT rowid, w FROM ctrl3;
}
do_all_vocab_test3 3.2
finish_test

30
ext/fts5/test/fts5prefix2.test
View File

@ -52,6 +52,36 @@ do_execsql_test 2.1 {
SELECT * FROM t2('to*');
} {top to tommy}
#-------------------------------------------------------------------------
foreach {tn newrowid} {
1 122
2 123
3 124
} {
reset_db
do_execsql_test 3.$tn.0 {
CREATE VIRTUAL TABLE t12 USING fts5(x);
INSERT INTO t12(rowid, x) VALUES(123, 'wwww');
}
do_execsql_test 3.$tn.1 {
BEGIN;
DELETE FROM t12 WHERE rowid=123;
SELECT * FROM t12('wwww*');
INSERT INTO t12(rowid, x) VALUES($newrowid, 'wwww');
SELECT * FROM t12('wwww*');
END;
} {wwww}
do_execsql_test 3.$tn.2 {
INSERT INTO t12(t12) VALUES('integrity-check');
}
do_execsql_test 3.$tn.3 {
SELECT rowid FROM t12('wwww*');
} $newrowid
}
finish_test
finish_test

24
ext/fts5/test/fts5rank.test
View File

@ -180,4 +180,28 @@ do_execsql_test 6.1 {
{table table table} {the table names.} {rank on an fts5 table}
}
#-------------------------------------------------------------------------
# forum post: https://sqlite.org/forum/forumpost/a2dd636330
#
reset_db
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t USING fts5 (a, b);
INSERT INTO t (a, b) VALUES ('data1', 'sentence1'), ('data2', 'sentence2');
INSERT INTO t(t, rank) VALUES ('rank', 'bm25(10.0,1.0)');
}
sqlite3 db2 test.db
do_execsql_test -db db2 1.1 {
SELECT *, rank<0.0 FROM t('data*') ORDER BY RANK;
} {data1 sentence1 1 data2 sentence2 1}
do_execsql_test 1.2 {
INSERT INTO t(t, rank) VALUES ('rank', 'bm25(10.0,1.0)');
}
do_execsql_test -db db2 1.3 {
SELECT *, rank<0.0 FROM t('data*') ORDER BY RANK;
} {data1 sentence1 1 data2 sentence2 1}
db2 close
finish_test

2
ext/fts5/test/fts5savepoint.test
View File

@ -71,7 +71,7 @@ ifcapable fts3 {
do_catchsql_test 3.2 {
DROP TABLE vt1;
} {1 {SQL logic error}}
} {0 {}}
do_execsql_test 3.3 {
SAVEPOINT x;

70
ext/fts5/test/fts5secure.test
View File

@ -273,6 +273,76 @@ do_execsql_test 5.3 {
do_execsql_test 5.4 { SELECT rowid FROM t1('abc'); } 2
do_execsql_test 5.5 { SELECT rowid FROM t1('aa'); } 2
#-------------------------------------------------------------------------
# Tests for the bug fixed by https://sqlite.org/src/info/4b60a1c3
#
reset_db
do_execsql_test 6.0 {
CREATE VIRTUAL TABLE fts USING fts5(content);
INSERT INTO fts(fts, rank) VALUES ('secure-delete', 1);
INSERT INTO fts(rowid, content) VALUES
(3407, 'profile profile profile profile profile profile profile profile pull pulling pulling really');
DELETE FROM fts WHERE rowid IS 3407;
INSERT INTO fts(fts) VALUES ('integrity-check');
}
foreach {tn detail} {
1 full
2 column
3 none
} {
do_execsql_test 6.1.$detail "
DROP TABLE IF EXISTS t1;
CREATE VIRTUAL TABLE t1 USING fts5(x, detail=$detail);
"
do_execsql_test 6.2.$detail {
INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
}
for {set ii 1} {$ii < 100} {incr ii} {
do_execsql_test 6.3.$detail.$ii.1 {
BEGIN;
INSERT INTO t1(rowid, x) VALUES(10, 'word1');
WITH s(i) AS (
SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<CAST($ii AS integer)
)
INSERT INTO t1(x) SELECT 'word3' FROM s;
COMMIT;
INSERT INTO t1(t1) VALUES('optimize');
}
do_execsql_test 6.3.$detail.$ii.2 {
DELETE FROM t1 WHERE rowid=10;
INSERT INTO t1(t1) VALUES ('integrity-check');
}
do_execsql_test 6.3.$detail.$ii.3 {
DELETE FROM t1;
}
do_execsql_test 6.3.$detail.$ii.4 {
BEGIN;
INSERT INTO t1(rowid, x) VALUES(10, 'tokenA');
WITH s(i) AS (
SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<CAST($ii AS integer)
)
INSERT INTO t1(x) SELECT group_concat('tokenB ') FROM s;
COMMIT;
INSERT INTO t1(t1) VALUES('optimize');
}
do_execsql_test 6.3.$detail.$ii.5 {
DELETE FROM t1 WHERE rowid=10;
INSERT INTO t1(t1) VALUES ('integrity-check');
}
do_execsql_test 6.3.$detail.$ii.6 {
DELETE FROM t1;
}
}
}
finish_test

15
ext/fts5/test/fts5secure3.test
View File

@ -86,6 +86,10 @@ do_execsql_test 2.8 {
# Tests with large/small rowid values.
#
foreach {tn cfg} {
1 ""
2 "INSERT INTO fff(fff, rank) VALUES('secure-delete', 1)"
} {
reset_db
expr srand(0)
@ -108,7 +112,7 @@ proc newdoc {} {
}
db func newdoc newdoc
do_execsql_test 3.0 {
do_execsql_test 3.$tn.0 {
CREATE VIRTUAL TABLE fff USING fts5(y);
INSERT INTO fff(fff, rank) VALUES('pgsz', 64);
@ -120,10 +124,10 @@ do_execsql_test 3.0 {
WITH s(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM s WHERE x<1000 )
INSERT INTO fff(rowid, y) SELECT random() , newdoc() FROM s;
INSERT INTO fff(fff, rank) VALUES('secure-delete', 1);
}
execsql $cfg
proc lshuffle {in} {
set out [list]
while {[llength $in]>0} {
@ -140,18 +144,19 @@ set iTest 1
foreach ii [lshuffle [db eval {SELECT rowid FROM fff}]] {
#if {$iTest==1} { dump fff }
#if {$iTest==1} { breakpoint }
do_execsql_test 3.1.$iTest.$ii {
do_execsql_test 3.$tn.1.$iTest.$ii {
DELETE FROM fff WHERE rowid=$ii;
}
#if {$iTest==1} { dump fff }
if {($iTest % 20)==0} {
do_execsql_test 3.1.$iTest.$ii.ic {
do_execsql_test 3.$tn.1.$iTest.$ii.ic {
INSERT INTO fff(fff) VALUES('integrity-check');
}
}
#if {$iTest==1} { break }
incr iTest
}
}
#execsql_pp { SELECT rowid FROM fff('post') ORDER BY rowid ASC }
#breakpoint

69
ext/fts5/test/fts5secure6.test
View File

@ -18,7 +18,7 @@ db progress 1 progress_handler
set ::PHC 0
proc progress_handler {args} {
incr ::PHC
if {($::PHC % 100000)==0} breakpoint
# if {($::PHC % 100000)==0} breakpoint
return 0
}
@ -70,5 +70,72 @@ do_execsql_test 2.2 {
SELECT rowid FROM t1('def')
} {-100000 -99999 9223372036854775800}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 3.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x);
INSERT INTO t1(t1, rank) VALUES('secure-delete', $sd)
}
do_execsql_test 3.1 {
BEGIN;
INSERT INTO t1(rowid, x)
VALUES(51869, 'when whenever where weress what turn'),
(51871, 'to were');
COMMIT;
}
do_execsql_test 3.2 {
DELETE FROM t1 WHERE rowid=51871;
INSERT INTO t1(t1) VALUES('integrity-check');
}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x);
INSERT INTO t1(rowid, x) VALUES(10, 'one two');
}
do_execsql_test 4.1 {
UPDATE t1 SET x = 'one three' WHERE rowid=10;
INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
}
do_execsql_test 4.2 {
DELETE FROM t1 WHERE rowid=10;
}
do_execsql_test 4.3 {
INSERT INTO t1(t1) VALUES('integrity-check');
}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 5.0 {
CREATE VIRTUAL TABLE t1 USING fts5(content);
INSERT INTO t1(t1,rank) VALUES('secure-delete',1);
INSERT INTO t1 VALUES('active'),('boomer'),('atom'),('atomic'),
('alpha channel backup abandon test aback boomer atom alpha active');
DELETE FROM t1 WHERE t1 MATCH 'abandon';
}
do_execsql_test 5.1 {
INSERT INTO t1(t1) VALUES('rebuild');
}
do_execsql_test 5.2 {
DELETE FROM t1 WHERE rowid NOTNULL<5;
}
db close
sqlite3 db test.db
do_execsql_test 5.3 {
PRAGMA integrity_check;
} {ok}
finish_test

116
ext/fts5/test/fts5secure7.test Normal file
View File

@ -0,0 +1,116 @@
# 2023 Feb 17
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
#
# TESTRUNNER: slow
#
source [file join [file dirname [info script]] fts5_common.tcl]
ifcapable !fts5 { finish_test ; return }
set ::testprefix fts5secure7
set NVOCAB 500
set NDOC [expr 1000]
set NREP 100
set nDeletePerRep [expr 5]
set VOCAB [list]
proc select_one {list} {
set n [llength $list]
lindex $list [expr {abs(int(rand()*$n))}]
}
proc init_vocab {} {
set L [split "abcdefghijklmnopqrstuvwxyz" {}]
set nL [llength $L]
for {set i 0} {$i < $::NVOCAB} {incr i} {
set n [expr {6 + int(rand()*8)}]
set word ""
for {set j 0} {$j < $n} {incr j} {
append word [select_one $L]
}
lappend ::VOCAB $word
}
}
proc get_word {} {
select_one $::VOCAB
}
proc get_document {nWord} {
set ret [list]
for {set i 0} {$i < $nWord} {incr i} {
lappend ret [get_word]
}
return $ret
}
init_vocab
db func document [list get_document 12]
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(body);
INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
}
do_execsql_test 1.1 {
WITH s(i) AS (
SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<$NDOC
)
INSERT INTO t1 SELECT document() FROM s;
}
for {set iRep 0} {$iRep < $NREP} {incr iRep} {
set lRowid [db eval {SELECT rowid FROM t1}]
for {set iDel 0} {$iDel < $nDeletePerRep} {incr iDel} {
set idx [select_one $lRowid]
db eval {
DELETE FROM t1 WHERE rowid=$idx
}
}
db eval {
WITH s(i) AS (
SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<$nDeletePerRep
)
INSERT INTO t1 SELECT document() FROM s;
}
do_execsql_test 1.2.$iRep {
INSERT INTO t1(t1) VALUES('integrity-check');
}
}
reset_db
db func document [list get_document 12]
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE t1 USING fts5(body);
INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
INSERT INTO t1(t1, rank) VALUES('pgsz', 128);
}
do_execsql_test 2.1 {
WITH s(i) AS (
SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<$NDOC
)
INSERT INTO t1 SELECT document() FROM s;
}
for {set ii 0} {$ii < $NDOC} {incr ii} {
set lRowid [db eval {SELECT rowid FROM t1}]
set idx [select_one $lRowid]
db eval { DELETE FROM t1 WHERE rowid=$idx }
do_execsql_test 2.2.$ii {
INSERT INTO t1(t1) VALUES('integrity-check');
}
}
finish_test

51
ext/fts5/test/fts5secure8.test Normal file
View File

@ -0,0 +1,51 @@
# 2023 Nov 23
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
#
source [file join [file dirname [info script]] fts5_common.tcl]
ifcapable !fts5 { finish_test ; return }
set ::testprefix fts5secure8
proc sql_repeat {txt n} {
string repeat $txt $n
}
db func repeat sql_repeat
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE ft USING fts5(x);
INSERT INTO ft(ft, rank) VALUES('pgsz', 64);
INSERT INTO ft(rowid, x) VALUES(100, 'hello world');
INSERT INTO ft(rowid, x) VALUES(200, 'one day');
BEGIN;
INSERT INTO ft(rowid, x) VALUES(45, 'one two three');
UPDATE ft SET x = repeat('hello world ', 500) WHERE rowid=100;
COMMIT
}
do_execsql_test 1.1 {
INSERT INTO ft(ft, rank) VALUES('secure-delete', 1);
DELETE FROM ft WHERE rowid=100;
}
do_execsql_test 1.2 {
PRAGMA integrity_check;
} {ok}
finish_test

4
ext/fts5/test/fts5simple2.test
View File

@ -343,7 +343,9 @@ do_execsql_test 17.0 {
INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb');
COMMIT;
}
do_execsql_test 17.1 { SELECT * FROM t2('y:a*') WHERE rowid BETWEEN 10 AND 20 }
do_execsql_test 17.1 {
SELECT * FROM t2('y:a*') WHERE rowid BETWEEN 10 AND 20
}
do_execsql_test 17.2 {
BEGIN;
INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb');

2
ext/fts5/test/fts5synonym2.test
View File

@ -42,7 +42,7 @@ proc fts5_test_bothlist {cmd} {
}
sqlite3_fts5_create_function db fts5_test_bothlist fts5_test_bothlist
proc fts5_rowid {cmd} { expr [$cmd xColumnText -1] }
proc fts5_rowid {cmd} { expr [$cmd xRowid] }
sqlite3_fts5_create_function db fts5_rowid fts5_rowid
do_execsql_test 1.$tok.0.1 "

89
ext/fts5/test/fts5tokenizer2.test Normal file
View File

@ -0,0 +1,89 @@
# 2023 Nov 03
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# Tests focusing on the built-in fts5 tokenizers.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5tokenizer2
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
sqlite3_fts5_create_tokenizer db tst get_tst_tokenizer
proc get_tst_tokenizer {args} {
return "tst_tokenizer"
}
proc tst_tokenizer {flags txt} {
set token ""
set lTok [list]
foreach c [split $txt {}] {
if {$token==""} {
append token $c
} else {
set t1 [string is upper $token]
set t2 [string is upper $c]
if {$t1!=$t2} {
lappend lTok $token
set token ""
}
append token $c
}
}
if {$token!=""} { lappend lTok $token }
set iOff 0
foreach t $lTok {
set n [string length $t]
sqlite3_fts5_token $t $iOff [expr $iOff+$n]
incr iOff $n
}
}
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(t, tokenize=tst);
}
do_execsql_test 1.1 {
INSERT INTO t1 VALUES('AAdontBBmess');
}
do_execsql_test 1.2 {
SELECT snippet(t1, 0, '>', '<', '...', 4) FROM t1('BB');
} {AAdont>BB<mess}
do_execsql_test 1.3 {
SELECT highlight(t1, 0, '>', '<') FROM t1('BB');
} {AAdont>BB<mess}
do_execsql_test 1.4 {
SELECT highlight(t1, 0, '>', '<') FROM t1('AA');
} {>AA<dontBBmess}
do_execsql_test 1.5 {
SELECT highlight(t1, 0, '>', '<') FROM t1('dont');
} {AA>dont<BBmess}
do_execsql_test 1.6 {
SELECT highlight(t1, 0, '>', '<') FROM t1('mess');
} {AAdontBB>mess<}
do_execsql_test 1.7 {
SELECT highlight(t1, 0, '>', '<') FROM t1('BB mess');
} {AAdont>BBmess<}
finish_test

38
ext/fts5/test/fts5trigram.test
View File

@ -215,4 +215,42 @@ do_execsql_test 7.2 {
SELECT rowid FROM f WHERE filename GLOB '*ир*';
} {20}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 8.0 {
CREATE VIRTUAL TABLE t1 USING fts5(y, tokenize=trigram);
INSERT INTO t1 VALUES('abcdefghijklm');
}
foreach {tn match res} {
1 "abc ghi" "(abc)def(ghi)jklm"
2 "def ghi" "abc(defghi)jklm"
3 "efg ghi" "abcd(efghi)jklm"
4 "efghi" "abcd(efghi)jklm"
5 "abcd jklm" "(abcd)efghi(jklm)"
6 "ijkl jklm" "abcdefgh(ijklm)"
7 "ijk ijkl hijk" "abcdefg(hijkl)m"
} {
do_execsql_test 8.1.$tn {
SELECT highlight(t1, 0, '(', ')') FROM t1($match)
} $res
}
do_execsql_test 8.2 {
CREATE VIRTUAL TABLE ft2 USING fts5(a, tokenize="trigram");
INSERT INTO ft2 VALUES('abc x cde');
INSERT INTO ft2 VALUES('abc cde');
INSERT INTO ft2 VALUES('abcde');
}
do_execsql_test 8.3 {
SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'abc AND cde';
} {
{[abc] x [cde]}
{[abc] [cde]}
{[abcde]}
}
finish_test

109
ext/fts5/test/fts5trigram2.test Normal file
View File

@ -0,0 +1,109 @@
# 2023 October 24
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
#
# Tests for the fts5 "trigram" tokenizer.
#
source [file join [file dirname [info script]] fts5_common.tcl]
ifcapable !fts5 { finish_test ; return }
set ::testprefix fts5trigram2
do_execsql_test 1.0 "
CREATE VIRTUAL TABLE t1 USING fts5(y, tokenize='trigram remove_diacritics 1');
INSERT INTO t1 VALUES('abc\u0303defghijklm');
INSERT INTO t1 VALUES('a\u0303b\u0303c\u0303defghijklm');
"
do_execsql_test 1.1 {
SELECT highlight(t1, 0, '(', ')') FROM t1('abc');
} [list \
"(abc\u0303)defghijklm" \
"(a\u0303b\u0303c\u0303)defghijklm" \
]
do_execsql_test 1.2 {
SELECT highlight(t1, 0, '(', ')') FROM t1('bcde');
} [list \
"a(bc\u0303de)fghijklm" \
"a\u0303(b\u0303c\u0303de)fghijklm" \
]
do_execsql_test 1.3 {
SELECT highlight(t1, 0, '(', ')') FROM t1('cdef');
} [list \
"ab(c\u0303def)ghijklm" \
"a\u0303b\u0303(c\u0303def)ghijklm" \
]
do_execsql_test 1.4 {
SELECT highlight(t1, 0, '(', ')') FROM t1('def');
} [list \
"abc\u0303(def)ghijklm" \
"a\u0303b\u0303c\u0303(def)ghijklm" \
]
#-------------------------------------------------------------------------
do_catchsql_test 2.0 {
CREATE VIRTUAL TABLE t2 USING fts5(
z, tokenize='trigram case_sensitive 1 remove_diacritics 1'
);
} {1 {error in tokenizer constructor}}
do_execsql_test 2.1 {
CREATE VIRTUAL TABLE t2 USING fts5(
z, tokenize='trigram case_sensitive 0 remove_diacritics 1'
);
}
do_execsql_test 2.2 "
INSERT INTO t2 VALUES('\u00E3bcdef');
INSERT INTO t2 VALUES('b\u00E3cdef');
INSERT INTO t2 VALUES('bc\u00E3def');
INSERT INTO t2 VALUES('bcd\u00E3ef');
"
do_execsql_test 2.3 {
SELECT highlight(t2, 0, '(', ')') FROM t2('abc');
} "(\u00E3bc)def"
do_execsql_test 2.4 {
SELECT highlight(t2, 0, '(', ')') FROM t2('bac');
} "(b\u00E3c)def"
do_execsql_test 2.5 {
SELECT highlight(t2, 0, '(', ')') FROM t2('bca');
} "(bc\u00E3)def"
do_execsql_test 2.6 "
SELECT highlight(t2, 0, '(', ')') FROM t2('\u00E3bc');
" "(\u00E3bc)def"
#-------------------------------------------------------------------------
do_execsql_test 3.0 {
CREATE VIRTUAL TABLE t3 USING fts5(
z, tokenize='trigram remove_diacritics 1'
);
} {}
do_execsql_test 3.1 "
INSERT INTO t3 VALUES ('\u0303abc\u0303');
"
do_execsql_test 3.2 {
SELECT highlight(t3, 0, '(', ')') FROM t3('abc');
} "\u0303(abc\u0303)"
#-------------------------------------------------------------------------
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE t4 USING fts5(z, tokenize=trigram);
} {}
breakpoint
do_execsql_test 4.1 {
INSERT INTO t4 VALUES('ABCD');
} {}
finish_test

24
ext/fts5/test/fts5vocab2.test
View File

@ -280,6 +280,30 @@ do_catchsql_test 5.2 {
INSERT INTO t1 SELECT randomblob(3000) FROM v1
} {1 {query aborted}}
#-------------------------------------------------------------------------
reset_db
sqlite3_fts5_may_be_corrupt 1
do_execsql_test 6.0 {
BEGIN TRANSACTION;
CREATE VIRTUAL TABLE t1 USING fts5(a,b unindexed,c,tokenize="porter ascii",tokendata=1);
REPLACE INTO t1_data VALUES(1,X'03090009');
REPLACE INTO t1_data VALUES(10,X'000000000103030003010101020101030101');
REPLACE INTO t1_data VALUES(137438953473,X'0000002e023061010202010162010203010163010204010167010601020201016801060102030101690106010204040606060808');
REPLACE INTO t1_data VALUES(274877906945,X'0000001f013067020802010202010168020803010203010169020804010204040909');
REPLACE INTO t1_data VALUES(412316860417,X'0000002e023061030202010162030203010163030204010167030601020201016803060102030101690306010204040606060808');
COMMIT;
}
do_execsql_test 6.1 {
CREATE VIRTUAL TABLE t3 USING fts5vocab('t1', 'row');
}
do_catchsql_test 6.2 {
SELECT * FROM t3;
} {1 {database disk image is malformed}}
sqlite3_fts5_may_be_corrupt 0
finish_test

331
ext/intck/intck1.test Normal file
View File

@ -0,0 +1,331 @@
# 2008 Feb 19
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# The focus of this file is testing the incremental integrity check
# (intck) extension.
#
source [file join [file dirname [info script]] intck_common.tcl]
set testprefix intck1
foreach {tn sql} {
1 "CREATE TABLE t1(a PRIMARY KEY, b)"
2 "CREATE TABLE t2(a PRIMARY KEY, b) WITHOUT ROWID "
3 "CREATE TABLE t3(a PRIMARY KEY, b) WITHOUT rowID;"
4 "CREATE TABLE t4(a PRIMARY KEY, ROWID)"
5 {CREATE TABLE t5(a PRIMARY KEY, ROWID) WITHOUT ROWID
}
} {
do_test 1.1.$tn {
db eval $sql
set {} {}
} {}
}
set space " \n\v\t\r\f"
do_execsql_test 1.2 {
SELECT name, (rtrim(sql, $space) LIKE '%rowid')
FROM sqlite_schema WHERE type='table'
ORDER BY 1
} {
t1 0
t2 1
t3 1
t4 0
t5 1
}
do_execsql_test 1.3 {
CREATE TABLE x1(a COLLATE nocase, b INTEGER, c BLOB);
INSERT INTO x1 VALUES('lEtTeRs', 1234, 1234);
}
do_execsql_test 1.3.1 {
WITH wrapper(c1, c2, c3) AS (
SELECT a, b, c FROM x1
)
SELECT * FROM wrapper WHERE c1='letters';
} {lEtTeRs 1234 1234}
do_execsql_test 1.3.2 {
WITH wrapper(c1, c2, c3) AS (
SELECT a, b, c FROM x1
)
SELECT * FROM wrapper WHERE c2='1234';
} {lEtTeRs 1234 1234}
do_execsql_test 1.3.2 {
WITH wrapper(c1, c2, c3) AS (
SELECT a, b, c FROM x1
)
SELECT * FROM wrapper WHERE c3='1234';
} {}
do_execsql_test 1.4 {
CREATE TABLE z1(a, b);
CREATE INDEX z1ab ON z1(a+b COLLATE nocase);
}
do_execsql_test 1.4.1 {
SELECT * FROM z1 INDEXED BY z1ab
}
do_catchsql_test 1.5.1 {
CREATE INDEX z1b ON z1(b ASC NULLS LAST);
} {1 {unsupported use of NULLS LAST}}
do_catchsql_test 1.5.2 {
CREATE INDEX z1b ON z1(b DESC NULLS LAST);
} {1 {unsupported use of NULLS LAST}}
do_catchsql_test 1.5.3 {
CREATE INDEX z1b ON z1(b ASC NULLS FIRST);
} {1 {unsupported use of NULLS FIRST}}
do_catchsql_test 1.5.4 {
CREATE INDEX z1b ON z1(b DESC NULLS FIRST);
} {1 {unsupported use of NULLS FIRST}}
reset_db
do_execsql_test 1.6.1 {
CREATE TABLE t1(i INTEGER PRIMARY KEY, b, c);
CREATE INDEX i1 ON t1(b);
ANALYZE;
INSERT INTO sqlite_stat1 VALUES('t1', 'i1', '10000 10000');
ANALYZE sqlite_schema;
} {}
do_eqp_test 1.6.2 {
SELECT 1 FROM t1 INDEXED BY i1 WHERE (b, i) IS (?, ?);
} {SEARCH}
#-------------------------------------------------------------------------
reset_db
do_test 2.0 {
set ic [sqlite3_intck db main]
$ic close
} {}
do_execsql_test 2.1 {
CREATE TABLE t1(a, b);
INSERT INTO t1 VALUES(1, 2);
INSERT INTO t1 VALUES(3, 4);
CREATE INDEX i1 ON t1(a COLLATE nocase);
CREATE INDEX i2 ON t1(b, a);
CREATE INDEX i3 ON t1(b + a COLLATE nocase) WHERE a!=1;
}
do_intck_test 2.2 {
}
# Delete a row from each of the i1 and i2 indexes using the imposter
# table interface.
#
do_test 2.3 {
db eval {SELECT name, rootpage FROM sqlite_schema} {
set R($name) $rootpage
}
sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 $R(i1)
db eval { CREATE TABLE imp1(a PRIMARY KEY, rowid) WITHOUT ROWID; }
sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 $R(i2)
db eval { CREATE TABLE imp2(b, a, rowid, PRIMARY KEY(b, a)) WITHOUT ROWID; }
sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 0
db eval {
DELETE FROM imp1 WHERE rowid=1;
DELETE FROM imp2 WHERE rowid=2;
}
db close
sqlite3 db test.db
} {}
do_intck_test 2.4 {
{entry (1,1) missing from index i1}
{entry (4,3,2) missing from index i2}
}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 3.0 {
CREATE TABLE x1(a, b, c, PRIMARY KEY(c, b)) WITHOUT ROWID;
CREATE INDEX x1a ON x1(a COLLATE nocase);
INSERT INTO x1 VALUES(1, 2, 'three');
INSERT INTO x1 VALUES(4, 5, 'six');
INSERT INTO x1 VALUES(7, 8, 'nine');
}
do_intck_test 3.1 { }
do_test 3.2 {
db eval {SELECT name, rootpage FROM sqlite_schema} {
set R($name) $rootpage
}
sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 $R(x1a)
db eval { CREATE TABLE imp1(c, b, a, PRIMARY KEY(c, b)) WITHOUT ROWID }
sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 0
db eval {
DELETE FROM imp1 WHERE a=5;
}
execsql_pp {
}
db close
sqlite3 db test.db
} {}
do_intck_test 3.3 {
{entry (4,'six',5) missing from index x1a}
}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 4.0 {
CREATE TABLE www(x, y, z);
CREATE INDEX w1 ON www( (x+1), z );
INSERT INTO www VALUES(1, 1, 1), (2, 2, 2);
}
do_intck_test 4.1 { }
#-------------------------------------------------------------------------
reset_db
do_execsql_test 5.0 {
CREATE TABLE t1(a, b);
CREATE INDEX i1 ON t1(a COLLATE NOCASE);
INSERT INTO t1 VALUES(1, 1);
INSERT INTO t1 VALUES(2, 2);
}
do_test 5.1 {
set ic [sqlite3_intck db nosuchdb]
$ic step
} {SQLITE_ERROR}
do_test 5.2 {
$ic close
set ic [sqlite3_intck db {}]
while {[$ic step]=="SQLITE_OK"} {}
set res [$ic error]
$ic close
set res
} {SQLITE_OK {}}
do_test 5.3 { test_do_intck db "main" } {}
do_test 5.4 {
set ret {}
set ic [sqlite3_intck db main]
db eval [$ic test_sql t1] {
if {$error_message!=""} { lappend ret $error_message }
}
$ic close
set ret
} {}
do_test 5.5 {
set ret {}
set ic [sqlite3_intck db main]
db eval [$ic test_sql {}] {
if {$error_message!=""} { lappend ret $error_message }
}
$ic close
set ret
} {}
db cache flush
do_test 5.6 {
set ret {}
set ic [sqlite3_intck db main]
$ic step
db eval [$ic test_sql {}] {
if {$error_message!=""} { lappend ret $error_message }
}
$ic close
set ret
} {}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 6.0 {
CREATE TABLE t1(x, y, PRIMARY KEY(x)) WITHOUT ROWID;
CREATE INDEX i1 ON t1(y, x);
INSERT INTO t1 VALUES(X'0000', X'1111');
}
do_intck_test 6.1 {}
do_execsql_test 6.2.1 {
PRAGMA writable_schema = 1;
UPDATE sqlite_schema SET sql = 'CREATE INDEX i1' WHERE name='i1';
} {}
do_intck_test 6.2.2 {}
do_execsql_test 6.3.1 {
UPDATE sqlite_schema SET sql = 'CREATE INDEX i1(y' WHERE name='i1';
} {}
do_intck_test 6.3.2 {}
do_execsql_test 6.4.1 {
UPDATE sqlite_schema
SET sql = 'CREATE INDEX i1(y) hello world'
WHERE name='i1';
} {}
do_intck_test 6.4.2 {}
do_execsql_test 6.5.1 {
UPDATE sqlite_schema
SET sql = 'CREATE INDEX i1(y, x) WHERE 1 '
WHERE name='i1';
} {}
do_intck_test 6.5.2 {}
do_execsql_test 6.6.1 {
UPDATE sqlite_schema
SET sql = 'CREATE INDEX i1( , ) WHERE 1 '
WHERE name='i1';
} {}
do_test 6.7.2 {
set ic [sqlite3_intck db main]
$ic step
} {SQLITE_ERROR}
do_test 6.5.3 {
$ic error
} {SQLITE_ERROR {near "AS": syntax error}}
$ic close
do_execsql_test 6.6.1 {
UPDATE sqlite_schema
SET sql = 'CREATE INDEX i1([y'
WHERE name='i1';
} {}
do_intck_test 6.6.2 {}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 7.0 {
CREATE TABLE x1("1", "22", "3333", four);
CREATE INDEX i1 ON x1( "1" , "22", NULL);
INSERT INTO x1 VALUES(1, 22, 3333, NULL);
INSERT INTO x1 VALUES(1, 22, 3333, NULL);
}
do_execsql_test 7.1 " CREATE INDEX i2 ON x1( \"1\"\r\n\t ) "
do_execsql_test 7.2 { CREATE INDEX i3 ON x1( "22" || 'abc''def' || `1` ) }
do_execsql_test 7.3 { CREATE INDEX i4 ON x1( [22] + [1] ) }
do_execsql_test 7.4 { CREATE INDEX i5 ON x1( four||'hello' ) }
do_intck_test 7.5 {}
finish_test

176
ext/intck/intck2.test Normal file
View File

@ -0,0 +1,176 @@
# 2024 Feb 19
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# The focus of this file is testing the incremental integrity check
# (intck) extension.
#
source [file join [file dirname [info script]] intck_common.tcl]
set testprefix intck2
do_execsql_test 1.0 {
CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT);
INSERT INTO t1 VALUES(1, 'one');
INSERT INTO t1 VALUES(2, 'two');
INSERT INTO t1 VALUES(3, 'three');
CREATE INDEX i1 ON t1(b);
}
proc imposter_edit {obj create sql} {
sqlite3 xdb test.db
set pgno [xdb one {SELECT rootpage FROM sqlite_schema WHERE name=$obj}]
sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER xdb main 1 $pgno
xdb eval $create
sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER xdb main 0 0
xdb eval $sql
xdb close
}
imposter_edit i1 {
CREATE TABLE imp(b, a, PRIMARY KEY(b)) WITHOUT ROWID;
} {
DELETE FROM imp WHERE b='two';
INSERT INTO imp(b, a) VALUES('four', 4);
}
do_intck_test 1.1 {
{surplus entry ('four',4) in index i1}
{entry ('two',2) missing from index i1}
}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 2.0 {
CREATE TABLE x1(a, b, "c d");
CREATE INDEX x1a ON x1(a COLLATE nocase DESC , b ASC);
CREATE INDEX x1b ON x1( a || b || ' "''" ' COLLATE binary ASC );
CREATE INDEX x1c ON x1( format('%s', a)ASC, format('%d', "c d" ) );
INSERT INTO x1 VALUES('one', 2, 3);
INSERT INTO x1 VALUES('One', 4, 5);
INSERT INTO x1 VALUES('ONE', 6, 7);
INSERT INTO x1 VALUES(NULL, NULL, NULL);
}
do_intck_test 2.1 {}
imposter_edit x1 {
CREATE TABLE imp(a, b, c);
} {
DELETE FROM imp WHERE c=7;
}
do_intck_test 2.2 {
{surplus entry ('ONE',6,3) in index x1a}
{surplus entry ('ONE6 "''" ',3) in index x1b}
{surplus entry ('ONE','7',3) in index x1c}
}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 3.0 {
CREATE TABLE x1(a, b, c);
CREATE INDEX x1all ON x1(a DESC, b ASC, c DESC);
INSERT INTO x1 VALUES(2, 1, 2);
INSERT INTO x1 VALUES(2, 1, 1);
INSERT INTO x1 VALUES(2, 2, 2);
INSERT INTO x1 VALUES(2, 2, 1);
INSERT INTO x1 VALUES(1, 1, 2);
INSERT INTO x1 VALUES(1, 1, 1);
INSERT INTO x1 VALUES(1, 2, 2);
INSERT INTO x1 VALUES(1, 2, 1);
}
do_intck_test 3.1 {
}
imposter_edit x1 {
CREATE TABLE imp(a, b, c);
} {
DELETE FROM imp WHERE 1;
}
db close
sqlite3 db test.db
do_intck_test 3.2 {
{surplus entry (2,1,2,1) in index x1all}
{surplus entry (2,1,1,2) in index x1all}
{surplus entry (2,2,2,3) in index x1all}
{surplus entry (2,2,1,4) in index x1all}
{surplus entry (1,1,2,5) in index x1all}
{surplus entry (1,1,1,6) in index x1all}
{surplus entry (1,2,2,7) in index x1all}
{surplus entry (1,2,1,8) in index x1all}
}
do_execsql_test 3.3 {
DELETE FROM x1;
INSERT INTO x1 VALUES(NULL, NULL, NULL);
INSERT INTO x1 VALUES(NULL, NULL, NULL);
INSERT INTO x1 VALUES(NULL, NULL, NULL);
INSERT INTO x1 VALUES(NULL, NULL, NULL);
}
do_intck_test 3.4 {
}
imposter_edit x1 {
CREATE TABLE imp(a, b, c);
} {
DELETE FROM imp WHERE 1;
INSERT INTO imp(rowid) VALUES(-123);
INSERT INTO imp(rowid) VALUES(456);
}
db close
sqlite3 db test.db
do_intck_test 3.5 {
{entry (NULL,NULL,NULL,-123) missing from index x1all}
{entry (NULL,NULL,NULL,456) missing from index x1all}
{surplus entry (NULL,NULL,NULL,1) in index x1all}
{surplus entry (NULL,NULL,NULL,2) in index x1all}
{surplus entry (NULL,NULL,NULL,3) in index x1all}
{surplus entry (NULL,NULL,NULL,4) in index x1all}
}
reset_db
do_execsql_test 3.6 {
CREATE TABLE w1(a PRIMARY KEY, b, c);
INSERT INTO w1 VALUES(1.0, NULL, NULL);
INSERT INTO w1 VALUES(33.0, NULL, NULL);
INSERT INTO w1 VALUES(100.0, NULL, NULL);
CREATE INDEX w1bc ON w1(b, c);
}
do_intck_test 3.7 {
}
imposter_edit w1 {
CREATE TABLE imp(a, b, c);
} {
DELETE FROM imp WHERE a=33;
INSERT INTO imp(a) VALUES(1234.5);
INSERT INTO imp(a) VALUES(-1234.5);
}
do_intck_test 3.8 {
{surplus entry (33.0,2) in index sqlite_autoindex_w1_1}
{entry (1234.5,4) missing from index sqlite_autoindex_w1_1}
{entry (NULL,NULL,4) missing from index w1bc}
{entry (-1234.5,5) missing from index sqlite_autoindex_w1_1}
{entry (NULL,NULL,5) missing from index w1bc}
{surplus entry (NULL,NULL,2) in index w1bc}
}
finish_test

56
ext/intck/intck_common.tcl Normal file
View File

@ -0,0 +1,56 @@
# 2024 Feb 18
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
if {![info exists testdir]} {
set testdir [file join [file dirname [info script]] .. .. test]
}
source $testdir/tester.tcl
proc do_intck {db {bSuspend 0}} {
set ic [sqlite3_intck $db main]
set ret [list]
while {"SQLITE_OK"==[$ic step]} {
set msg [$ic message]
if {$msg!=""} {
lappend ret $msg
}
if {$bSuspend} {
$ic unlock
#puts "SQL: [$ic test_sql {}]"
#execsql_pp "EXPLAIN query plan [$ic test_sql {}]"
#explain_i [$ic test_sql {}]
}
}
set err [$ic error]
if {[lindex $err 0]!="SQLITE_OK"} {
error $err
}
$ic close
return $ret
}
proc intck_sql {db tbl} {
set ic [sqlite3_intck $db main]
set sql [$ic test_sql $tbl]
$ic close
return $sql
}
proc do_intck_test {tn expect} {
uplevel [list do_test $tn.a [list do_intck db] [list {*}$expect]]
uplevel [list do_test $tn.b [list do_intck db 1] [list {*}$expect]]
}

48
ext/intck/intckbusy.test Normal file
View File

@ -0,0 +1,48 @@
# 2024 February 24
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
source [file join [file dirname [info script]] intck_common.tcl]
set testprefix intckbusy
do_execsql_test 1.0 {
CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
INSERT INTO t1 VALUES(1, 2, 3);
INSERT INTO t1 VALUES(2, 'two', 'three');
INSERT INTO t1 VALUES(3, NULL, NULL);
CREATE INDEX i1 ON t1(b, c);
}
sqlite3 db2 test.db
do_execsql_test -db db2 1.1 {
BEGIN EXCLUSIVE;
INSERT INTO t1 VALUES(4, 5, 6);
}
do_test 1.2 {
set ic [sqlite3_intck db main]
$ic step
} {SQLITE_BUSY}
do_test 1.3 {
$ic unlock
} {SQLITE_BUSY}
do_test 1.4 {
$ic error
} {SQLITE_BUSY {database is locked}}
do_test 1.4 {
$ic close
} {}
finish_test

235
ext/intck/intckcorrupt.test Normal file
View File

@ -0,0 +1,235 @@
# 2024 Feb 21
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# The focus of this file is testing the intck extensions response
# to corruption at the b-tree level.
#
source [file join [file dirname [info script]] intck_common.tcl]
set testprefix intckcorrupt
#-------------------------------------------------------------------------
reset_db
do_test 1.0 {
sqlite3 db {}
db deserialize [decode_hexdb {
| size 356352 pagesize 4096 filename crash-acaae0347204ae.db
| page 1 offset 0
| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3.
| 16: 10 00 01 01 00 40 20 20 00 00 00 00 d0 00 00 00 .....@ ........
| 32: 40 00 ea 00 00 00 00 00 00 40 00 00 00 40 00 00 @........@...@..
| 96: 00 00 00 00 0d 00 00 00 04 0e 9c 00 0f ad 0f 4f ...............O
| 112: 0e fc 0e 9c 00 00 00 00 00 00 00 00 00 00 00 00 ................
| 3728: 00 00 00 00 00 00 00 00 00 00 00 00 5e 04 07 17 ............^...
| 3744: 1f 1f 01 81 0b 74 61 62 6c 65 74 31 5f 70 61 72 .....tablet1_par
| 3760: 65 6e 74 74 31 5f 70 61 72 65 6e 74 04 43 52 45 entt1_parent.CRE
| 3776: 41 54 45 20 54 41 42 4c 45 20 22 74 31 5f 70 61 ATE TABLE .t1_pa
| 3792: 72 65 6e 74 22 28 6e 6f 64 65 6e 6f 20 49 4e 54 rent.(nodeno INT
| 3808: 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 EGER PRIMARY KEY
| 3824: 2c 70 61 72 65 6e 74 6e 6f 64 65 29 51 03 06 17 ,parentnode)Q...
| 3840: 1b 1b 01 7b 74 61 62 6c 65 74 31 5f 6e 6f 64 65 ....tablet1_node
| 3856: 74 31 5f 6e 6f 64 65 03 43 52 45 41 54 45 20 54 t1_node.CREATE T
| 3872: 41 42 4c 45 20 22 74 31 5f 6e 6f 64 65 22 28 6e ABLE .t1_node.(n
| 3888: 6f 64 65 6e 6f 20 49 4e 54 45 47 45 52 20 50 52 odeno INTEGER PR
| 3904: 49 4d 41 52 59 20 4b 45 59 2c 64 61 74 61 29 5c IMARY KEY,data).
| 3920: 02 07 17 1d 1d 01 81 0b 74 61 62 6c 65 74 31 5f ........tablet1_
| 3936: 72 6f 77 69 64 74 31 5f 72 6f 77 69 64 02 43 52 rowidt1_rowid.CR
| 3952: 45 41 54 45 20 54 41 42 4c 45 20 22 74 31 5f 72 EATE TABLE .t1_r
| 3968: 6f 77 69 64 22 28 72 6f 77 69 64 20 49 4e 54 45 owid.(rowid INTE
| 3984: 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c GER PRIMARY KEY,
| 4000: 6e 6f 64 65 6e 6f 2c 61 30 2c 61 31 29 51 01 07 nodeno,a0,a1)Q..
| 4016: 17 11 11 08 81 0f 74 61 62 6c 65 74 31 74 31 43 ......tablet1t1C
| 4032: 52 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 REATE VIRTUAL TA
| 4048: 42 4c 45 20 74 31 20 55 53 49 4e 47 20 72 74 72 BLE t1 USING rtr
| 4064: 65 65 28 69 64 2c 78 30 20 50 52 49 4d 41 52 59 ee(id,x0 PRIMARY
| 4080: 20 4b 45 59 2c 70 61 72 65 6e 74 6e 6f 64 65 29 KEY,parentnode)
| page 2 offset 4096
| 0: 51 03 06 17 1b 1b 01 7b 74 61 62 6c 65 74 31 5f Q.......tablet1_
| 16: 6e 6f 64 65 74 31 5f 6e 6f 64 65 03 43 52 45 41 nodet1_node.CREA
| 32: 54 45 20 54 41 42 4c 45 20 22 74 31 5f 6e 6f 64 TE TABLE .t1_nod
| 48: 65 22 28 6e 6f 64 65 6e 6f 20 49 4e 54 45 47 45 e.(nodeno INTEGE
| 64: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 64 61 R PRIMARY KEY,da
| 80: 74 61 29 5c 02 07 17 1d 1d 01 81 0b 74 61 62 6c ta).........tabl
| 96: 65 74 31 5f 72 6f 77 69 64 74 31 5f 72 6f 77 69 et1_rowidt1_rowi
| 112: 64 02 43 52 45 41 54 45 20 54 41 42 4c 45 00 00 d.CREATE TABLE..
| 128: 01 0a 02 00 00 00 01 0e 0d 00 00 00 00 24 0e 0d .............$..
| 144: 0c 1a 06 85 50 46 60 27 70 08 00 00 00 00 00 00 ....PF`'p.......
| 3824: 00 00 00 00 00 00 00 0d 0e 05 00 09 1d 00 74 6f ..............to
| 3840: 79 20 68 61 6c 66 10 0d 05 00 09 23 00 62 6f 74 y half.....#.bot
| 3856: 74 6f 6d 20 68 61 6c 66 0f 0c 05 00 09 21 00 72 tom half.....!.r
| 3872: 69 67 68 74 20 68 61 6c 66 0e 0b 05 00 09 1f 00 ight half.......
| 3888: 6c 65 66 74 20 43 15 f6 e6 f6 46 50 34 35 24 54 left C....FP45$T
| 3904: 15 44 52 05 44 14 24 c4 52 02 27 43 15 f6 e6 f6 .DR.D.$.R.'C....
| 3920: 46 52 22 8e 6f 64 65 6e 6f 20 49 4e 54 45 47 45 FR..odeno INTEGE
| 3936: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 64 61 R PRIMARY KEY,da
| 3952: 74 61 29 5c 02 07 17 1d 1d 01 81 0b 74 61 62 6c ta).........tabl
| 3968: 65 74 31 5f 72 6f 74 74 6f 6d 20 65 64 67 65 0f et1_rottom edge.
| 3984: 07 05 00 09 21 00 72 69 67 68 74 20 65 64 67 65 ....!.right edge
| 4000: 0e 06 05 00 09 1f 00 6c 65 66 74 20 65 64 67 65 .......left edge
| 4016: 0b 05 05 00 09 19 00 63 65 6e 74 65 72 17 04 05 .......center...
| 4032: 00 09 31 00 75 70 70 65 72 2d 72 69 67 68 74 20 ..1.upper-right
| 4048: 63 6f 72 6e 65 72 17 03 05 00 09 31 00 6c 6f 77 corner.....1.low
| 4064: 65 72 2d 72 69 67 68 74 20 63 6f 72 6e 65 72 16 er-right corner.
| 4080: 02 05 00 09 2f 00 75 70 70 65 72 2d 6c 65 66 74 ..../.upper-left
| page 3 offset 8192
| 0: 20 63 6f 72 6e 65 72 16 01 05 00 09 2f 01 8c 6f corner...../..o
| 16: 77 65 72 2d 6c 53 51 4c 69 74 65 20 66 6f 72 6d wer-lSQLite form
| 32: 61 74 20 33 00 10 00 01 01 00 40 20 20 00 00 00 at 3......@ ...
| 48: 00 00 00 00 2f 00 00 0d eb 13 00 00 00 03 00 00 ..../...........
| 64: 00 04 00 00 00 00 00 00 00 06 00 00 00 01 00 00 ................
| 80: 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 ................
| page 6 offset 20480
| 128: 00 00 00 00 00 00 00 00 97 3d 04 ae 7c 01 00 00 .........=..|...
| 624: 00 00 00 00 00 00 21 97 3d 04 ae 7c 01 00 00 00 ......!.=..|....
| 1120: 00 00 00 00 00 20 97 3d 04 ae 7c 01 00 00 00 00 ..... .=..|.....
| 1616: 00 00 00 00 1f 97 3d 04 ae 7c 01 00 00 00 00 00 ......=..|......
| 2112: 00 00 00 1e 97 3d 04 ae 7c 01 00 00 00 00 00 00 .....=..|.......
| 2608: 00 00 1d 97 d3 d0 4a e7 c0 00 00 00 00 00 00 00 ......J.........
| 3088: 00 00 00 00 00 00 00 00 00 00 00 00 01 f3 00 00 ................
| 3600: 23 97 3d 04 ae 7c 01 00 00 00 00 00 00 00 00 00 #.=..|..........
| 4080: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 26 ...............&
| page 8 offset 28672
| 0: 0d 00 00 00 01 04 30 00 04 30 00 00 00 00 00 00 ......0..0......
| 1072: 97 4d 1e 14 00 ae 7c 00 00 00 00 00 00 00 00 00 .M....|.........
| 1088: 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 ................
| 4080: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 ................
| page 10 offset 36864
| 0: 0d 00 00 00 01 04 30 00 04 30 00 00 00 00 00 00 ......0..0......
| 1072: 9a ee c1 80 fd 78 1f ce 1b ae eb b4 00 00 00 00 .....x..........
| 1088: 13 20 ff 20 00 70 00 00 00 60 50 00 00 00 11 e0 . . .p...`P.....
| 1104: 00 00 00 70 00 00 00 60 50 05 35 14 c6 97 46 52 ...p...`P.5...FR
| 1120: 06 66 f7 26 d6 17 42 03 30 01 00 00 10 10 04 02 .f.&..B.0.......
| 1136: 02 00 00 00 00 00 00 00 00 40 00 00 00 00 00 00 .........@......
| 1152: 00 00 00 00 00 40 00 00 00 40 00 00 00 00 00 00 .....@...@......
| 4080: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 05 ................
| page 12 offset 45056
| 0: 0d 00 00 00 01 04 30 00 04 30 e1 b4 30 97 4d 46 ......0..0..0.MF
| 16: 14 00 ae 7c 00 00 00 00 00 00 00 03 00 00 43 00 ...|..........C.
| page 47 offset 188416
| 2512: 00 00 00 00 00 00 00 00 be 00 00 00 00 00 00 00 ................
| page 87 offset 352256
| 2512: 00 00 00 00 00 00 00 00 aa 00 00 00 00 00 00 00 ................
| end crash-acaae0347204ae.db
}]} {}
do_intck_test 1.1 {
{corruption found while reading database schema}
}
#-------------------------------------------------------------------------
reset_db
do_test 2.0 {
sqlite3 db {}
db deserialize [decode_hexdb {
| size 28672 pagesize 4096 filename crash-3afa1ca9e9c1bd.db
| page 1 offset 0
| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3.
| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 07 .....@ ........
| 32: 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 04 ................
| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................
| 96: 00 00 00 00 0d 00 00 00 06 0e 88 00 0f b8 0f 6d ...............m
| 112: 0f 3a 0f 0b 0e d5 0e 88 01 00 00 00 00 00 00 00 .:..............
| 3712: 00 00 00 00 00 00 00 00 4b 06 06 17 25 25 01 5b ........K...%%.[
| 3728: 74 61 62 6c 65 73 71 6c 69 74 65 5f 73 74 61 74 tablesqlite_stat
| 3744: 31 73 71 6c 69 74 65 5f 73 74 61 74 31 07 43 52 1sqlite_stat1.CR
| 3760: 45 41 54 45 20 54 41 42 4c 45 20 73 71 6c 69 74 EATE TABLE sqlit
| 3776: 65 5f 73 74 61 74 31 28 74 62 6c 2c 69 64 78 2c e_stat1(tbl,idx,
| 3792: 73 74 61 74 29 34 05 06 17 13 11 01 53 69 6e 64 stat)4......Sind
| 3808: 65 78 63 31 63 63 31 06 43 52 45 41 54 45 20 55 exc1cc1.CREATE U
| 3824: 4e 49 51 55 45 20 49 4e 44 45 58 20 63 31 63 20 NIQUE INDEX c1c
| 3840: 4f 4e 20 63 31 28 63 2c 20 62 29 2d 04 06 17 13 ON c1(c, b)-....
| 3856: 11 01 45 69 6e 64 65 78 63 31 64 63 31 05 43 52 ..Eindexc1dc1.CR
| 3872: 45 41 54 45 20 49 4e 44 45 58 20 63 31 64 20 4f EATE INDEX c1d O
| 3888: 4e 20 63 31 28 64 2c 20 62 29 31 03 06 17 13 11 N c1(d, b)1.....
| 3904: 01 4d 69 6e 64 65 78 62 31 63 62 31 05 43 52 45 .Mindexb1cb1.CRE
| 3920: 41 54 45 20 55 4e 49 51 55 45 20 49 4e 44 45 58 ATE UNIQUE INDEX
| 3936: 20 62 31 63 20 4f 4e 20 62 31 28 63 29 49 02 06 b1c ON b1(c)I..
| 3952: 17 11 11 0f 7f 74 61 62 6c 65 63 31 63 31 03 43 .....tablec1c1.C
| 3968: 52 45 41 54 45 20 54 41 42 4c 45 20 63 31 28 61 REATE TABLE c1(a
| 3984: 20 49 4e 54 20 50 52 49 4d 41 52 59 20 4b 45 59 INT PRIMARY KEY
| 4000: 2c 20 62 2c 20 63 2c 20 64 29 20 57 49 54 48 4f , b, c, d) WITHO
| 4016: 55 54 20 52 4f 57 49 44 46 01 06 17 11 11 01 79 UT ROWIDF......y
| 4032: 74 61 62 6c 65 62 31 62 31 02 43 52 45 41 54 45 tableb1b1.CREATE
| 4048: 20 54 41 42 4c 45 20 62 31 28 61 20 49 4e 54 20 TABLE b1(a INT
| 4064: 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 62 2c 20 PRIMARY KEY, b,
| 4080: 63 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 c) WITHOUT ROWID
| page 2 offset 4096
| 0: 0a 00 00 00 07 0f ca 00 0f fa 0f f2 0f ea 0f e2 ................
| 16: 0f da 00 00 00 01 00 00 00 00 00 00 00 00 00 00 ................
| 4032: 00 00 00 00 00 00 00 00 00 00 07 04 01 0f 01 06 ................
| 4048: 67 07 07 04 01 0f 01 06 66 06 07 04 01 0f 01 05 g.......f.......
| 4064: 65 05 07 04 01 0f 01 04 64 04 07 04 01 0f 01 03 e.......d.......
| 4080: 63 03 07 04 01 0f 01 02 62 0f 05 04 09 0f 09 61 c.......b......a
| page 3 offset 8192
| 0: 0a 00 00 00 07 0f bd 00 0f f9 0f ef 0f e5 0f db ................
| 16: 0f d1 0f c7 0f bd 00 00 00 00 01 00 00 00 00 00 ................
| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 09 05 01 ................
| 4032: 0f 01 01 07 61 07 07 09 05 01 0f 01 01 06 61 06 ....a.........a.
| 4048: 06 09 05 01 0f 01 01 05 61 05 05 09 05 01 0f 01 ........a.......
| 4064: 01 04 61 04 04 09 05 01 0f 01 01 03 61 03 03 09 ..a.........a...
| 4080: 05 01 0f 01 01 02 61 0f 02 06 05 09 0f 09 09 61 ......a........a
| page 4 offset 12288
| 0: 0a 00 00 00 07 0f d8 00 0f fc 0f f0 0f ea 0f e4 ................
| 16: 0f de 0f d8 0f f6 00 00 00 00 00 00 00 00 00 00 ................
| 4048: 00 00 00 00 00 00 00 00 05 03 01 01 07 07 05 03 ................
| 4064: 01 01 06 06 05 03 01 01 05 05 05 03 01 01 04 04 ................
| 4080: 05 03 01 01 03 03 05 03 01 01 0f 02 03 03 09 09 ................
| page 5 offset 16384
| 0: 0a 00 00 00 07 0f ca 00 0f fa 0f f2 0f ea 0f 00 ................
| 4032: 00 00 00 00 00 00 00 00 00 00 07 04 01 0f 01 07 ................
| 4048: 61 07 07 04 01 0f 01 06 61 06 07 04 01 0f 01 05 a.......a.......
| 4064: 61 05 07 04 01 1f 01 04 61 04 07 04 01 0f 01 03 a.......a.......
| 4080: 61 03 07 04 01 0f 01 02 61 02 05 04 09 0f 09 61 a.......a......a
| page 6 offset 20480
| 0: 0a 00 00 00 07 0f ca 00 0f fa 0f ea 0f e2 00 00 ................
| 4032: 00 00 00 00 00 00 00 00 00 00 07 04 01 0f 01 07 ................
| 4048: 61 07 07 04 01 0f 01 06 61 06 07 04 01 0f 01 05 a.......a.......
| 4064: 61 05 07 04 01 0f 01 04 61 04 07 04 01 0f 01 03 a.......a.......
| 4080: 61 03 07 04 01 0f 01 0f 61 02 05 04 09 0f 09 61 a.......a......a
| page 7 offset 24576
| 0: 0d 00 00 00 05 0f 1c 00 0f f0 0f e0 0f d3 0f c5 ................
| 16: 0f b8 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
| 4016: 00 00 00 00 00 00 00 00 0b 05 04 11 11 13 62 31 ..............b1
| 4032: 62 31 37 20 31 0c 04 04 11 13 13 62 31 62 31 63 b17 1......b1b1c
| 4048: 37 20 31 0b 03 04 11 11 13 63 31 63 31 37 20 31 7 1......c1c17 1
| 4064: 0e 02 04 11 13 07 63 31 63 31 64 37 20 31 20 31 ......c1c1d7 1 1
| 4080: 0e 01 04 11 13 17 63 31 63 31 63 37 20 31 00 00 ......c1c1c7 1..
| end crash-3afa1ca9e9c1bd.db
}]} {}
do_intck_test 2.1 {
{corruption found while reading database schema}
}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 3.0 {
PRAGMA page_size = 1024;
CREATE TABLE t1(a, b);
CREATE INDEX i1 ON t1(a);
INSERT INTO t1 VALUES(1, 1), (2, 2), (3, 3);
}
do_test 3.1 {
set pgno [db one {SELECT rootpage FROM sqlite_schema WHERE name='t1'}]
db close
hexio_write test.db [expr ($pgno-1)*1024] 0000
} {2}
sqlite3 db test.db
do_intck_test 3.2 {
{corruption found while scanning database object i1}
{corruption found while scanning database object t1}
}
finish_test

45
ext/intck/intckfault.test Normal file
View File

@ -0,0 +1,45 @@
# 2024 February 24
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
source [file join [file dirname [info script]] intck_common.tcl]
set testprefix intckfault
do_execsql_test 1.0 {
CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
INSERT INTO t1 VALUES(1, 2, 3);
INSERT INTO t1 VALUES(2, 'two', 'three');
INSERT INTO t1 VALUES(3, NULL, NULL);
CREATE INDEX i1 ON t1(b, c);
}
do_faultsim_test 1 -faults oom-t* -prep {
} -body {
set ::ic [sqlite3_intck db main]
set nStep 0
while {"SQLITE_OK"==[$::ic step]} {
incr nStep
if {$nStep==3} { $::ic unlock }
}
set res [$::ic error]
$::ic close
set res
} -test {
catch { $::ic close }
puts $testresult
puts $testnfail
faultsim_test_result {0 {SQLITE_OK {}}} {0 {SQLITE_NOMEM {}}} {0 {SQLITE_NOMEM {out of memory}}}
}
finish_test

941
ext/intck/sqlite3intck.c Normal file
View File

@ -0,0 +1,941 @@
/*
** 2024-02-08
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
*/
#include "sqlite3intck.h"
#include <string.h>
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
/*
** nKeyVal:
** The number of values that make up the 'key' for the current pCheck
** statement.
**
** rc:
** Error code returned by most recent sqlite3_intck_step() or
** sqlite3_intck_unlock() call. This is set to SQLITE_DONE when
** the integrity-check operation is finished.
**
** zErr:
** If the object has entered the error state, this is the error message.
** Is freed using sqlite3_free() when the object is deleted.
**
** zTestSql:
** The value returned by the most recent call to sqlite3_intck_testsql().
** Each call to testsql() frees the previous zTestSql value (using
** sqlite3_free()) and replaces it with the new value it will return.
*/
struct sqlite3_intck {
sqlite3 *db;
const char *zDb; /* Copy of zDb parameter to _open() */
char *zObj; /* Current object. Or NULL. */
sqlite3_stmt *pCheck; /* Current check statement */
char *zKey;
int nKeyVal;
char *zMessage;
int bCorruptSchema;
int rc; /* Error code */
char *zErr; /* Error message */
char *zTestSql; /* Returned by sqlite3_intck_test_sql() */
};
/*
** Some error has occurred while using database p->db. Save the error message
** and error code currently held by the database handle in p->rc and p->zErr.
*/
static void intckSaveErrmsg(sqlite3_intck *p){
p->rc = sqlite3_errcode(p->db);
sqlite3_free(p->zErr);
p->zErr = sqlite3_mprintf("%s", sqlite3_errmsg(p->db));
}
/*
** If the handle passed as the first argument is already in the error state,
** then this function is a no-op (returns NULL immediately). Otherwise, if an
** error occurs within this function, it leaves an error in said handle.
**
** Otherwise, this function attempts to prepare SQL statement zSql and
** return the resulting statement handle to the user.
*/
static sqlite3_stmt *intckPrepare(sqlite3_intck *p, const char *zSql){
sqlite3_stmt *pRet = 0;
if( p->rc==SQLITE_OK ){
p->rc = sqlite3_prepare_v2(p->db, zSql, -1, &pRet, 0);
if( p->rc!=SQLITE_OK ){
intckSaveErrmsg(p);
assert( pRet==0 );
}
}
return pRet;
}
/*
** If the handle passed as the first argument is already in the error state,
** then this function is a no-op (returns NULL immediately). Otherwise, if an
** error occurs within this function, it leaves an error in said handle.
**
** Otherwise, this function treats argument zFmt as a printf() style format
** string. It formats it according to the trailing arguments and then
** attempts to prepare the results and return the resulting prepared
** statement.
*/
static sqlite3_stmt *intckPrepareFmt(sqlite3_intck *p, const char *zFmt, ...){
sqlite3_stmt *pRet = 0;
va_list ap;
char *zSql = 0;
va_start(ap, zFmt);
zSql = sqlite3_vmprintf(zFmt, ap);
if( p->rc==SQLITE_OK && zSql==0 ){
p->rc = SQLITE_NOMEM;
}
pRet = intckPrepare(p, zSql);
sqlite3_free(zSql);
va_end(ap);
return pRet;
}
/*
** Finalize SQL statement pStmt. If an error occurs and the handle passed
** as the first argument does not already contain an error, store the
** error in the handle.
*/
static void intckFinalize(sqlite3_intck *p, sqlite3_stmt *pStmt){
int rc = sqlite3_finalize(pStmt);
if( p->rc==SQLITE_OK && rc!=SQLITE_OK ){
intckSaveErrmsg(p);
}
}
/*
** If there is already an error in handle p, return it. Otherwise, call
** sqlite3_step() on the statement handle and return that value.
*/
static int intckStep(sqlite3_intck *p, sqlite3_stmt *pStmt){
if( p->rc ) return p->rc;
return sqlite3_step(pStmt);
}
/*
** Execute SQL statement zSql. There is no way to obtain any results
** returned by the statement. This function uses the sqlite3_intck error
** code convention.
*/
static void intckExec(sqlite3_intck *p, const char *zSql){
sqlite3_stmt *pStmt = 0;
pStmt = intckPrepare(p, zSql);
intckStep(p, pStmt);
intckFinalize(p, pStmt);
}
/*
** A wrapper around sqlite3_mprintf() that uses the sqlite3_intck error
** code convention.
*/
static char *intckMprintf(sqlite3_intck *p, const char *zFmt, ...){
va_list ap;
char *zRet = 0;
va_start(ap, zFmt);
zRet = sqlite3_vmprintf(zFmt, ap);
if( p->rc==SQLITE_OK ){
if( zRet==0 ){
p->rc = SQLITE_NOMEM;
}
}else{
sqlite3_free(zRet);
zRet = 0;
}
return zRet;
}
/*
** This is used by sqlite3_intck_unlock() to save the vector key value
** required to restart the current pCheck query as a nul-terminated string
** in p->zKey.
*/
static void intckSaveKey(sqlite3_intck *p){
int ii;
char *zSql = 0;
sqlite3_stmt *pStmt = 0;
sqlite3_stmt *pXinfo = 0;
const char *zDir = 0;
assert( p->pCheck );
assert( p->zKey==0 );
pXinfo = intckPrepareFmt(p,
"SELECT group_concat(desc, '') FROM %Q.sqlite_schema s, "
"pragma_index_xinfo(%Q, %Q) "
"WHERE s.type='index' AND s.name=%Q",
p->zDb, p->zObj, p->zDb, p->zObj
);
if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXinfo) ){
zDir = (const char*)sqlite3_column_text(pXinfo, 0);
}
if( zDir==0 ){
/* Object is a table, not an index. This is the easy case,as there are
** no DESC columns or NULL values in a primary key. */
const char *zSep = "SELECT '(' || ";
for(ii=0; ii<p->nKeyVal; ii++){
zSql = intckMprintf(p, "%z%squote(?)", zSql, zSep);
zSep = " || ', ' || ";
}
zSql = intckMprintf(p, "%z || ')'", zSql);
}else{
/* Object is an index. */
assert( p->nKeyVal>1 );
for(ii=p->nKeyVal; ii>0; ii--){
int bLastIsDesc = zDir[ii-1]=='1';
int bLastIsNull = sqlite3_column_type(p->pCheck, ii)==SQLITE_NULL;
const char *zLast = sqlite3_column_name(p->pCheck, ii);
char *zLhs = 0;
char *zRhs = 0;
char *zWhere = 0;
if( bLastIsNull ){
if( bLastIsDesc ) continue;
zWhere = intckMprintf(p, "'%s IS NOT NULL'", zLast);
}else{
const char *zOp = bLastIsDesc ? "<" : ">";
zWhere = intckMprintf(p, "'%s %s ' || quote(?%d)", zLast, zOp, ii);
}
if( ii>1 ){
const char *zLhsSep = "";
const char *zRhsSep = "";
int jj;
for(jj=0; jj<ii-1; jj++){
const char *zAlias = (const char*)sqlite3_column_name(p->pCheck,jj+1);
zLhs = intckMprintf(p, "%z%s%s", zLhs, zLhsSep, zAlias);
zRhs = intckMprintf(p, "%z%squote(?%d)", zRhs, zRhsSep, jj+1);
zLhsSep = ",";
zRhsSep = " || ',' || ";
}
zWhere = intckMprintf(p,
"'(%z) IS (' || %z || ') AND ' || %z",
zLhs, zRhs, zWhere);
}
zWhere = intckMprintf(p, "'WHERE ' || %z", zWhere);
zSql = intckMprintf(p, "%z%s(quote( %z ) )",
zSql,
(zSql==0 ? "VALUES" : ",\n "),
zWhere
);
}
zSql = intckMprintf(p,
"WITH wc(q) AS (\n%z\n)"
"SELECT 'VALUES' || group_concat('(' || q || ')', ',\n ') FROM wc"
, zSql
);
}
pStmt = intckPrepare(p, zSql);
if( p->rc==SQLITE_OK ){
for(ii=0; ii<p->nKeyVal; ii++){
sqlite3_bind_value(pStmt, ii+1, sqlite3_column_value(p->pCheck, ii+1));
}
if( SQLITE_ROW==sqlite3_step(pStmt) ){
p->zKey = intckMprintf(p,"%s",(const char*)sqlite3_column_text(pStmt, 0));
}
intckFinalize(p, pStmt);
}
sqlite3_free(zSql);
intckFinalize(p, pXinfo);
}
/*
** Find the next database object (table or index) to check. If successful,
** set sqlite3_intck.zObj to point to a nul-terminated buffer containing
** the object's name before returning.
*/
static void intckFindObject(sqlite3_intck *p){
sqlite3_stmt *pStmt = 0;
char *zPrev = p->zObj;
p->zObj = 0;
assert( p->rc==SQLITE_OK );
assert( p->pCheck==0 );
pStmt = intckPrepareFmt(p,
"WITH tables(table_name) AS ("
" SELECT name"
" FROM %Q.sqlite_schema WHERE (type='table' OR type='index') AND rootpage"
" UNION ALL "
" SELECT 'sqlite_schema'"
")"
"SELECT table_name FROM tables "
"WHERE ?1 IS NULL OR table_name%s?1 "
"ORDER BY 1"
, p->zDb, (p->zKey ? ">=" : ">")
);
if( p->rc==SQLITE_OK ){
sqlite3_bind_text(pStmt, 1, zPrev, -1, SQLITE_TRANSIENT);
if( sqlite3_step(pStmt)==SQLITE_ROW ){
p->zObj = intckMprintf(p,"%s",(const char*)sqlite3_column_text(pStmt, 0));
}
}
intckFinalize(p, pStmt);
/* If this is a new object, ensure the previous key value is cleared. */
if( sqlite3_stricmp(p->zObj, zPrev) ){
sqlite3_free(p->zKey);
p->zKey = 0;
}
sqlite3_free(zPrev);
}
/*
** Return the size in bytes of the first token in nul-terminated buffer z.
** For the purposes of this call, a token is either:
**
** * a quoted SQL string,
* * a contiguous series of ascii alphabet characters, or
* * any other single byte.
*/
static int intckGetToken(const char *z){
char c = z[0];
int iRet = 1;
if( c=='\'' || c=='"' || c=='`' ){
while( 1 ){
if( z[iRet]==c ){
iRet++;
if( z[iRet]!=c ) break;
}
iRet++;
}
}
else if( c=='[' ){
while( z[iRet++]!=']' && z[iRet] );
}
else if( (c>='A' && c<='Z') || (c>='a' && c<='z') ){
while( (z[iRet]>='A' && z[iRet]<='Z') || (z[iRet]>='a' && z[iRet]<='z') ){
iRet++;
}
}
return iRet;
}
/*
** Return true if argument c is an ascii whitespace character.
*/
static int intckIsSpace(char c){
return (c==' ' || c=='\t' || c=='\n' || c=='\r');
}
/*
** Argument z points to the text of a CREATE INDEX statement. This function
** identifies the part of the text that contains either the index WHERE
** clause (if iCol<0) or the iCol'th column of the index.
**
** If (iCol<0), the identified fragment does not include the "WHERE" keyword,
** only the expression that follows it. If (iCol>=0) then the identified
** fragment does not include any trailing sort-order keywords - "ASC" or
** "DESC".
**
** If the CREATE INDEX statement does not contain the requested field or
** clause, NULL is returned and (*pnByte) is set to 0. Otherwise, a pointer to
** the identified fragment is returned and output parameter (*pnByte) set
** to its size in bytes.
*/
static const char *intckParseCreateIndex(const char *z, int iCol, int *pnByte){
int iOff = 0;
int iThisCol = 0;
int iStart = 0;
int nOpen = 0;
const char *zRet = 0;
int nRet = 0;
int iEndOfCol = 0;
/* Skip forward until the first "(" token */
while( z[iOff]!='(' ){
iOff += intckGetToken(&z[iOff]);
if( z[iOff]=='\0' ) return 0;
}
assert( z[iOff]=='(' );
nOpen = 1;
iOff++;
iStart = iOff;
while( z[iOff] ){
const char *zToken = &z[iOff];
int nToken = 0;
/* Check if this is the end of the current column - either a "," or ")"
** when nOpen==1. */
if( nOpen==1 ){
if( z[iOff]==',' || z[iOff]==')' ){
if( iCol==iThisCol ){
int iEnd = iEndOfCol ? iEndOfCol : iOff;
nRet = (iEnd - iStart);
zRet = &z[iStart];
break;
}
iStart = iOff+1;
while( intckIsSpace(z[iStart]) ) iStart++;
iThisCol++;
}
if( z[iOff]==')' ) break;
}
if( z[iOff]=='(' ) nOpen++;
if( z[iOff]==')' ) nOpen--;
nToken = intckGetToken(zToken);
if( (nToken==3 && 0==sqlite3_strnicmp(zToken, "ASC", nToken))
|| (nToken==4 && 0==sqlite3_strnicmp(zToken, "DESC", nToken))
){
iEndOfCol = iOff;
}else if( 0==intckIsSpace(zToken[0]) ){
iEndOfCol = 0;
}
iOff += nToken;
}
/* iStart is now the byte offset of 1 byte passed the final ')' in the
** CREATE INDEX statement. Try to find a WHERE clause to return. */
while( zRet==0 && z[iOff] ){
int n = intckGetToken(&z[iOff]);
if( n==5 && 0==sqlite3_strnicmp(&z[iOff], "where", 5) ){
zRet = &z[iOff+5];
nRet = strlen(zRet);
}
iOff += n;
}
/* Trim any whitespace from the start and end of the returned string. */
if( zRet ){
while( intckIsSpace(zRet[0]) ){
nRet--;
zRet++;
}
while( nRet>0 && intckIsSpace(zRet[nRet-1]) ) nRet--;
}
*pnByte = nRet;
return zRet;
}
/*
** User-defined SQL function wrapper for intckParseCreateIndex():
**
** SELECT parse_create_index(<sql>, <icol>);
*/
static void intckParseCreateIndexFunc(
sqlite3_context *pCtx,
int nVal,
sqlite3_value **apVal
){
const char *zSql = (const char*)sqlite3_value_text(apVal[0]);
int idx = sqlite3_value_int(apVal[1]);
const char *zRes = 0;
int nRes = 0;
assert( nVal==2 );
if( zSql ){
zRes = intckParseCreateIndex(zSql, idx, &nRes);
}
sqlite3_result_text(pCtx, zRes, nRes, SQLITE_TRANSIENT);
}
/*
** Return true if sqlite3_intck.db has automatic indexes enabled, false
** otherwise.
*/
static int intckGetAutoIndex(sqlite3_intck *p){
int bRet = 0;
sqlite3_stmt *pStmt = 0;
pStmt = intckPrepare(p, "PRAGMA automatic_index");
if( SQLITE_ROW==intckStep(p, pStmt) ){
bRet = sqlite3_column_int(pStmt, 0);
}
intckFinalize(p, pStmt);
return bRet;
}
/*
** Return true if zObj is an index, or false otherwise.
*/
static int intckIsIndex(sqlite3_intck *p, const char *zObj){
int bRet = 0;
sqlite3_stmt *pStmt = 0;
pStmt = intckPrepareFmt(p,
"SELECT 1 FROM %Q.sqlite_schema WHERE name=%Q AND type='index'",
p->zDb, zObj
);
if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
bRet = 1;
}
intckFinalize(p, pStmt);
return bRet;
}
/*
** Return a pointer to a nul-terminated buffer containing the SQL statement
** used to check database object zObj (a table or index) for corruption.
** If parameter zPrev is not NULL, then it must be a string containing the
** vector key required to restart the check where it left off last time.
** If pnKeyVal is not NULL, then (*pnKeyVal) is set to the number of
** columns in the vector key value for the specified object.
**
** This function uses the sqlite3_intck error code convention.
*/
static char *intckCheckObjectSql(
sqlite3_intck *p, /* Integrity check object */
const char *zObj, /* Object (table or index) to scan */
const char *zPrev, /* Restart key vector, if any */
int *pnKeyVal /* OUT: Number of key-values for this scan */
){
char *zRet = 0;
sqlite3_stmt *pStmt = 0;
int bAutoIndex = 0;
int bIsIndex = 0;
const char *zCommon =
/* Relation without_rowid also contains just one row. Column "b" is
** set to true if the table being examined is a WITHOUT ROWID table,
** or false otherwise. */
", without_rowid(b) AS ("
" SELECT EXISTS ("
" SELECT 1 FROM tabname, pragma_index_list(tab, db) AS l"
" WHERE origin='pk' "
" AND NOT EXISTS (SELECT 1 FROM sqlite_schema WHERE name=l.name)"
" )"
")"
""
/* Table idx_cols contains 1 row for each column in each index on the
** table being checked. Columns are:
**
** idx_name: Name of the index.
** idx_ispk: True if this index is the PK of a WITHOUT ROWID table.
** col_name: Name of indexed column, or NULL for index on expression.
** col_expr: Indexed expression, including COLLATE clause.
** col_alias: Alias used for column in 'intck_wrapper' table.
*/
", idx_cols(idx_name, idx_ispk, col_name, col_expr, col_alias) AS ("
" SELECT l.name, (l.origin=='pk' AND w.b), i.name, COALESCE(("
" SELECT parse_create_index(sql, i.seqno) FROM "
" sqlite_schema WHERE name = l.name"
" ), format('\"%w\"', i.name) || ' COLLATE ' || quote(i.coll)),"
" 'c' || row_number() OVER ()"
" FROM "
" tabname t,"
" without_rowid w,"
" pragma_index_list(t.tab, t.db) l,"
" pragma_index_xinfo(l.name) i"
" WHERE i.key"
" UNION ALL"
" SELECT '', 1, '_rowid_', '_rowid_', 'r1' FROM without_rowid WHERE b=0"
")"
""
""
/*
** For a PK declared as "PRIMARY KEY(a, b) ... WITHOUT ROWID", where
** the intck_wrapper aliases of "a" and "b" are "c1" and "c2":
**
** o_pk: "o.c1, o.c2"
** i_pk: "i.'a', i.'b'"
** ...
** n_pk: 2
*/
", tabpk(db, tab, idx, o_pk, i_pk, q_pk, eq_pk, ps_pk, pk_pk, n_pk) AS ("
" WITH pkfields(f, a) AS ("
" SELECT i.col_name, i.col_alias FROM idx_cols i WHERE i.idx_ispk"
" )"
" SELECT t.db, t.tab, t.idx, "
" group_concat(a, ', '), "
" group_concat('i.'||quote(f), ', '), "
" group_concat('quote(o.'||a||')', ' || '','' || '), "
" format('(%s)==(%s)',"
" group_concat('o.'||a, ', '), "
" group_concat(format('\"%w\"', f), ', ')"
" ),"
" group_concat('%s', ','),"
" group_concat('quote('||a||')', ', '), "
" count(*)"
" FROM tabname t, pkfields"
")"
""
", idx(name, match_expr, partial, partial_alias, idx_ps, idx_idx) AS ("
" SELECT idx_name,"
" format('(%s,%s) IS (%s,%s)', "
" group_concat(i.col_expr, ', '), i_pk,"
" group_concat('o.'||i.col_alias, ', '), o_pk"
" ), "
" parse_create_index("
" (SELECT sql FROM sqlite_schema WHERE name=idx_name), -1"
" ),"
" 'cond' || row_number() OVER ()"
" , group_concat('%s', ',')"
" , group_concat('quote('||i.col_alias||')', ', ')"
" FROM tabpk t, "
" without_rowid w,"
" idx_cols i"
" WHERE i.idx_ispk==0 "
" GROUP BY idx_name"
")"
""
", wrapper_with(s) AS ("
" SELECT 'intck_wrapper AS (\n SELECT\n ' || ("
" WITH f(a, b) AS ("
" SELECT col_expr, col_alias FROM idx_cols"
" UNION ALL "
" SELECT partial, partial_alias FROM idx WHERE partial IS NOT NULL"
" )"
" SELECT group_concat(format('%s AS %s', a, b), ',\n ') FROM f"
" )"
" || format('\n FROM %Q.%Q ', t.db, t.tab)"
/* If the object being checked is a table, append "NOT INDEXED".
** Otherwise, append "INDEXED BY <index>", and then, if the index
** is a partial index " WHERE <condition>". */
" || CASE WHEN t.idx IS NULL THEN "
" 'NOT INDEXED'"
" ELSE"
" format('INDEXED BY %Q%s', t.idx, ' WHERE '||i.partial)"
" END"
" || '\n)'"
" FROM tabname t LEFT JOIN idx i ON (i.name=t.idx)"
")"
""
;
bAutoIndex = intckGetAutoIndex(p);
if( bAutoIndex ) intckExec(p, "PRAGMA automatic_index = 0");
bIsIndex = intckIsIndex(p, zObj);
if( bIsIndex ){
pStmt = intckPrepareFmt(p,
/* Table idxname contains a single row. The first column, "db", contains
** the name of the db containing the table (e.g. "main") and the second,
** "tab", the name of the table itself. */
"WITH tabname(db, tab, idx) AS ("
" SELECT %Q, (SELECT tbl_name FROM %Q.sqlite_schema WHERE name=%Q), %Q "
")"
""
", whereclause(w_c) AS (%s)"
""
"%s" /* zCommon */
""
", case_statement(c) AS ("
" SELECT "
" 'CASE WHEN (' || group_concat(col_alias, ', ') || ', 1) IS (\n' "
" || ' SELECT ' || group_concat(col_expr, ', ') || ', 1 FROM '"
" || format('%%Q.%%Q NOT INDEXED WHERE %%s\n', t.db, t.tab, p.eq_pk)"
" || ' )\n THEN NULL\n '"
" || 'ELSE format(''surplus entry ('"
" || group_concat('%%s', ',') || ',' || p.ps_pk"
" || ') in index ' || t.idx || ''', ' "
" || group_concat('quote('||i.col_alias||')', ', ') || ', ' || p.pk_pk"
" || ')'"
" || '\n END AS error_message'"
" FROM tabname t, tabpk p, idx_cols i WHERE i.idx_name=t.idx"
")"
""
", thiskey(k, n) AS ("
" SELECT group_concat(i.col_alias, ', ') || ', ' || p.o_pk, "
" count(*) + p.n_pk "
" FROM tabpk p, idx_cols i WHERE i.idx_name=p.idx"
")"
""
", main_select(m, n) AS ("
" SELECT format("
" 'WITH %%s\n' ||"
" ', idx_checker AS (\n' ||"
" ' SELECT %%s,\n' ||"
" ' %%s\n' || "
" ' FROM intck_wrapper AS o\n' ||"
" ')\n',"
" ww.s, c, t.k"
" ), t.n"
" FROM case_statement, wrapper_with ww, thiskey t"
")"
"SELECT m || "
" group_concat('SELECT * FROM idx_checker ' || w_c, ' UNION ALL '), n"
" FROM "
"main_select, whereclause "
, p->zDb, p->zDb, zObj, zObj
, zPrev ? zPrev : "VALUES('')", zCommon
);
}else{
pStmt = intckPrepareFmt(p,
/* Table tabname contains a single row. The first column, "db", contains
** the name of the db containing the table (e.g. "main") and the second,
** "tab", the name of the table itself. */
"WITH tabname(db, tab, idx, prev) AS (SELECT %Q, %Q, NULL, %Q)"
""
"%s" /* zCommon */
/* expr(e) contains one row for each index on table zObj. Value e
** is set to an expression that evaluates to NULL if the required
** entry is present in the index, or an error message otherwise. */
", expr(e, p) AS ("
" SELECT format('CASE WHEN EXISTS \n"
" (SELECT 1 FROM %%Q.%%Q AS i INDEXED BY %%Q WHERE %%s%%s)\n"
" THEN NULL\n"
" ELSE format(''entry (%%s,%%s) missing from index %%s'', %%s, %%s)\n"
" END\n'"
" , t.db, t.tab, i.name, i.match_expr, ' AND (' || partial || ')',"
" i.idx_ps, t.ps_pk, i.name, i.idx_idx, t.pk_pk),"
" CASE WHEN partial IS NULL THEN NULL ELSE i.partial_alias END"
" FROM tabpk t, idx i"
")"
", numbered(ii, cond, e) AS ("
" SELECT 0, 'n.ii=0', 'NULL'"
" UNION ALL "
" SELECT row_number() OVER (),"
" '(n.ii='||row_number() OVER ()||COALESCE(' AND '||p||')', ')'), e"
" FROM expr"
")"
", counter_with(w) AS ("
" SELECT 'WITH intck_counter(ii) AS (\n ' || "
" group_concat('SELECT '||ii, ' UNION ALL\n ') "
" || '\n)' FROM numbered"
")"
""
", case_statement(c) AS ("
" SELECT 'CASE ' || "
" group_concat(format('\n WHEN %%s THEN (%%s)', cond, e), '') ||"
" '\nEND AS error_message'"
" FROM numbered"
")"
""
/* This table contains a single row consisting of a single value -
** the text of an SQL expression that may be used by the main SQL
** statement to output an SQL literal that can be used to resume
** the scan if it is suspended. e.g. for a rowid table, an expression
** like:
**
** format('(%d,%d)', _rowid_, n.ii)
*/
", thiskey(k, n) AS ("
" SELECT o_pk || ', ii', n_pk+1 FROM tabpk"
")"
""
", whereclause(w_c) AS ("
" SELECT CASE WHEN prev!='' THEN "
" '\nWHERE (' || o_pk ||', n.ii) > ' || prev"
" ELSE ''"
" END"
" FROM tabpk, tabname"
")"
""
", main_select(m, n) AS ("
" SELECT format("
" '%%s, %%s\nSELECT %%s,\n%%s\nFROM intck_wrapper AS o"
", intck_counter AS n%%s\nORDER BY %%s', "
" w, ww.s, c, thiskey.k, whereclause.w_c, t.o_pk"
" ), thiskey.n"
" FROM case_statement, tabpk t, counter_with, "
" wrapper_with ww, thiskey, whereclause"
")"
"SELECT m, n FROM main_select",
p->zDb, zObj, zPrev, zCommon
);
}
while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
zRet = intckMprintf(p, "%s", (const char*)sqlite3_column_text(pStmt, 0));
if( pnKeyVal ){
*pnKeyVal = sqlite3_column_int(pStmt, 1);
}
}
intckFinalize(p, pStmt);
if( bAutoIndex ) intckExec(p, "PRAGMA automatic_index = 1");
return zRet;
}
/*
** Open a new integrity-check object.
*/
int sqlite3_intck_open(
sqlite3 *db, /* Database handle to operate on */
const char *zDbArg, /* "main", "temp" etc. */
sqlite3_intck **ppOut /* OUT: New integrity-check handle */
){
sqlite3_intck *pNew = 0;
int rc = SQLITE_OK;
const char *zDb = zDbArg ? zDbArg : "main";
int nDb = strlen(zDb);
pNew = (sqlite3_intck*)sqlite3_malloc(sizeof(*pNew) + nDb + 1);
if( pNew==0 ){
rc = SQLITE_NOMEM;
}else{
memset(pNew, 0, sizeof(*pNew));
pNew->db = db;
pNew->zDb = (const char*)&pNew[1];
memcpy(&pNew[1], zDb, nDb+1);
rc = sqlite3_create_function(db, "parse_create_index",
2, SQLITE_UTF8, 0, intckParseCreateIndexFunc, 0, 0
);
if( rc!=SQLITE_OK ){
sqlite3_intck_close(pNew);
pNew = 0;
}
}
*ppOut = pNew;
return rc;
}
/*
** Free the integrity-check object.
*/
void sqlite3_intck_close(sqlite3_intck *p){
if( p ){
sqlite3_finalize(p->pCheck);
sqlite3_create_function(
p->db, "parse_create_index", 1, SQLITE_UTF8, 0, 0, 0, 0
);
sqlite3_free(p->zObj);
sqlite3_free(p->zKey);
sqlite3_free(p->zTestSql);
sqlite3_free(p->zErr);
sqlite3_free(p->zMessage);
sqlite3_free(p);
}
}
/*
** Step the integrity-check object.
*/
int sqlite3_intck_step(sqlite3_intck *p){
if( p->rc==SQLITE_OK ){
if( p->zMessage ){
sqlite3_free(p->zMessage);
p->zMessage = 0;
}
if( p->bCorruptSchema ){
p->rc = SQLITE_DONE;
}else
if( p->pCheck==0 ){
intckFindObject(p);
if( p->rc==SQLITE_OK ){
if( p->zObj ){
char *zSql = 0;
zSql = intckCheckObjectSql(p, p->zObj, p->zKey, &p->nKeyVal);
p->pCheck = intckPrepare(p, zSql);
sqlite3_free(zSql);
sqlite3_free(p->zKey);
p->zKey = 0;
}else{
p->rc = SQLITE_DONE;
}
}else if( p->rc==SQLITE_CORRUPT ){
p->rc = SQLITE_OK;
p->zMessage = intckMprintf(p, "%s",
"corruption found while reading database schema"
);
p->bCorruptSchema = 1;
}
}
if( p->pCheck ){
assert( p->rc==SQLITE_OK );
if( sqlite3_step(p->pCheck)==SQLITE_ROW ){
/* Normal case, do nothing. */
}else{
intckFinalize(p, p->pCheck);
p->pCheck = 0;
p->nKeyVal = 0;
if( p->rc==SQLITE_CORRUPT ){
p->rc = SQLITE_OK;
p->zMessage = intckMprintf(p,
"corruption found while scanning database object %s", p->zObj
);
}
}
}
}
return p->rc;
}
/*
** Return a message describing the corruption encountered by the most recent
** call to sqlite3_intck_step(), or NULL if no corruption was encountered.
*/
const char *sqlite3_intck_message(sqlite3_intck *p){
assert( p->pCheck==0 || p->zMessage==0 );
if( p->zMessage ){
return p->zMessage;
}
if( p->pCheck ){
return (const char*)sqlite3_column_text(p->pCheck, 0);
}
return 0;
}
/*
** Return the error code and message.
*/
int sqlite3_intck_error(sqlite3_intck *p, const char **pzErr){
if( pzErr ) *pzErr = p->zErr;
return (p->rc==SQLITE_DONE ? SQLITE_OK : p->rc);
}
/*
** Close any read transaction the integrity-check object is holding open
** on the database.
*/
int sqlite3_intck_unlock(sqlite3_intck *p){
if( p->rc==SQLITE_OK && p->pCheck ){
assert( p->zKey==0 && p->nKeyVal>0 );
intckSaveKey(p);
intckFinalize(p, p->pCheck);
p->pCheck = 0;
}
return p->rc;
}
/*
** Return the SQL statement used to check object zObj. Or, if zObj is
** NULL, the current SQL statement.
*/
const char *sqlite3_intck_test_sql(sqlite3_intck *p, const char *zObj){
sqlite3_free(p->zTestSql);
if( zObj ){
p->zTestSql = intckCheckObjectSql(p, zObj, 0, 0);
}else{
if( p->zObj ){
p->zTestSql = intckCheckObjectSql(p, p->zObj, p->zKey, 0);
}else{
sqlite3_free(p->zTestSql);
p->zTestSql = 0;
}
}
return p->zTestSql;
}

171
ext/intck/sqlite3intck.h Normal file
View File

@ -0,0 +1,171 @@
/*
** 2024-02-08
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
*/
/*
** Incremental Integrity-Check Extension
** -------------------------------------
**
** This module contains code to check whether or not an SQLite database
** is well-formed or corrupt. This is the same task as performed by SQLite's
** built-in "PRAGMA integrity_check" command. This module differs from
** "PRAGMA integrity_check" in that:
**
** + It is less thorough - this module does not detect certain types
** of corruption that are detected by the PRAGMA command. However,
** it does detect all kinds of corruption that are likely to cause
** errors in SQLite applications.
**
** + It is slower. Sometimes up to three times slower.
**
** + It allows integrity-check operations to be split into multiple
** transactions, so that the database does not need to be read-locked
** for the duration of the integrity-check.
**
** One way to use the API to run integrity-check on the "main" database
** of handle db is:
**
** int rc = SQLITE_OK;
** sqlite3_intck *p = 0;
**
** sqlite3_intck_open(db, "main", &p);
** while( SQLITE_OK==sqlite3_intck_step(p) ){
** const char *zMsg = sqlite3_intck_message(p);
** if( zMsg ) printf("corruption: %s\n", zMsg);
** }
** rc = sqlite3_intck_error(p, &zErr);
** if( rc!=SQLITE_OK ){
** printf("error occured (rc=%d), (errmsg=%s)\n", rc, zErr);
** }
** sqlite3_intck_close(p);
**
** Usually, the sqlite3_intck object opens a read transaction within the
** first call to sqlite3_intck_step() and holds it open until the
** integrity-check is complete. However, if sqlite3_intck_unlock() is
** called, the read transaction is ended and a new read transaction opened
** by the subsequent call to sqlite3_intck_step().
*/
#ifndef _SQLITE_INTCK_H
#define _SQLITE_INTCK_H
#include "sqlite3.h"
#ifdef __cplusplus
extern "C" {
#endif
/*
** An ongoing incremental integrity-check operation is represented by an
** opaque pointer of the following type.
*/
typedef struct sqlite3_intck sqlite3_intck;
/*
** Open a new incremental integrity-check object. If successful, populate
** output variable (*ppOut) with the new object handle and return SQLITE_OK.
** Or, if an error occurs, set (*ppOut) to NULL and return an SQLite error
** code (e.g. SQLITE_NOMEM).
**
** The integrity-check will be conducted on database zDb (which must be "main",
** "temp", or the name of an attached database) of database handle db. Once
** this function has been called successfully, the caller should not use
** database handle db until the integrity-check object has been destroyed
** using sqlite3_intck_close().
*/
int sqlite3_intck_open(
sqlite3 *db, /* Database handle */
const char *zDb, /* Database name ("main", "temp" etc.) */
sqlite3_intck **ppOut /* OUT: New sqlite3_intck handle */
);
/*
** Close and release all resources associated with a handle opened by an
** earlier call to sqlite3_intck_open(). The results of using an
** integrity-check handle after it has been passed to this function are
** undefined.
*/
void sqlite3_intck_close(sqlite3_intck *pCk);
/*
** Do the next step of the integrity-check operation specified by the handle
** passed as the only argument. This function returns SQLITE_DONE if the
** integrity-check operation is finished, or an SQLite error code if
** an error occurs, or SQLITE_OK if no error occurs but the integrity-check
** is not finished. It is not considered an error if database corruption
** is encountered.
**
** Following a successful call to sqlite3_intck_step() (one that returns
** SQLITE_OK), sqlite3_intck_message() returns a non-NULL value if
** corruption was detected in the db.
**
** If an error occurs and a value other than SQLITE_OK or SQLITE_DONE is
** returned, then the integrity-check handle is placed in an error state.
** In this state all subsequent calls to sqlite3_intck_step() or
** sqlite3_intck_unlock() will immediately return the same error. The
** sqlite3_intck_error() method may be used to obtain an English language
** error message in this case.
*/
int sqlite3_intck_step(sqlite3_intck *pCk);
/*
** If the previous call to sqlite3_intck_step() encountered corruption
** within the database, then this function returns a pointer to a buffer
** containing a nul-terminated string describing the corruption in
** English. If the previous call to sqlite3_intck_step() did not encounter
** corruption, or if there was no previous call, this function returns
** NULL.
*/
const char *sqlite3_intck_message(sqlite3_intck *pCk);
/*
** Close any read-transaction opened by an earlier call to
** sqlite3_intck_step(). Any subsequent call to sqlite3_intck_step() will
** open a new transaction. Return SQLITE_OK if successful, or an SQLite error
** code otherwise.
**
** If an error occurs, then the integrity-check handle is placed in an error
** state. In this state all subsequent calls to sqlite3_intck_step() or
** sqlite3_intck_unlock() will immediately return the same error. The
** sqlite3_intck_error() method may be used to obtain an English language
** error message in this case.
*/
int sqlite3_intck_unlock(sqlite3_intck *pCk);
/*
** If an error has occurred in an earlier call to sqlite3_intck_step()
** or sqlite3_intck_unlock(), then this method returns the associated
** SQLite error code. Additionally, if pzErr is not NULL, then (*pzErr)
** may be set to point to a nul-terminated string containing an English
** language error message. Or, if no error message is available, to
** NULL.
**
** If no error has occurred within sqlite3_intck_step() or
** sqlite_intck_unlock() calls on the handle passed as the first argument,
** then SQLITE_OK is returned and (*pzErr) set to NULL.
*/
int sqlite3_intck_error(sqlite3_intck *pCk, const char **pzErr);
/*
** This API is used for testing only. It returns the full-text of an SQL
** statement used to test object zObj, which may be a table or index.
** The returned buffer is valid until the next call to either this function
** or sqlite3_intck_close() on the same sqlite3_intck handle.
*/
const char *sqlite3_intck_test_sql(sqlite3_intck *pCk, const char *zObj);
#ifdef __cplusplus
} /* end of the 'extern "C"' block */
#endif
#endif /* ifndef _SQLITE_INTCK_H */

238
ext/intck/test_intck.c Normal file
View File

@ -0,0 +1,238 @@
/*
** 2010 August 28
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** Code for testing all sorts of SQLite interfaces. This code
** is not included in the SQLite library.
*/
#include "sqlite3.h"
#include "sqlite3intck.h"
#if defined(INCLUDE_SQLITE_TCL_H)
# include "sqlite_tcl.h"
#else
# include "tcl.h"
#endif
#include <string.h>
#include <assert.h>
/* In test1.c */
int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb);
const char *sqlite3ErrName(int);
typedef struct TestIntck TestIntck;
struct TestIntck {
sqlite3_intck *intck;
};
static int testIntckCmd(
void * clientData,
Tcl_Interp *interp,
int objc,
Tcl_Obj *CONST objv[]
){
struct Subcmd {
const char *zName;
int nArg;
const char *zExpect;
} aCmd[] = {
{"close", 0, ""}, /* 0 */
{"step", 0, ""}, /* 1 */
{"message", 0, ""}, /* 2 */
{"error", 0, ""}, /* 3 */
{"unlock", 0, ""}, /* 4 */
{"test_sql", 1, ""}, /* 5 */
{0 , 0}
};
int rc = TCL_OK;
int iIdx = -1;
TestIntck *p = (TestIntck*)clientData;
if( objc<2 ){
Tcl_WrongNumArgs(interp, 1, objv, "SUB-COMMAND ...");
return TCL_ERROR;
}
rc = Tcl_GetIndexFromObjStruct(
interp, objv[1], aCmd, sizeof(aCmd[0]), "SUB-COMMAND", 0, &iIdx
);
if( rc ) return rc;
if( objc!=2+aCmd[iIdx].nArg ){
Tcl_WrongNumArgs(interp, 2, objv, aCmd[iIdx].zExpect);
return TCL_ERROR;
}
switch( iIdx ){
case 0: assert( 0==strcmp("close", aCmd[iIdx].zName) ); {
Tcl_DeleteCommand(interp, Tcl_GetStringFromObj(objv[0], 0));
break;
}
case 1: assert( 0==strcmp("step", aCmd[iIdx].zName) ); {
int rc = sqlite3_intck_step(p->intck);
Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
break;
}
case 2: assert( 0==strcmp("message", aCmd[iIdx].zName) ); {
const char *z = sqlite3_intck_message(p->intck);
Tcl_SetObjResult(interp, Tcl_NewStringObj(z ? z : "", -1));
break;
}
case 3: assert( 0==strcmp("error", aCmd[iIdx].zName) ); {
const char *zErr = 0;
int rc = sqlite3_intck_error(p->intck, 0);
Tcl_Obj *pRes = Tcl_NewObj();
Tcl_ListObjAppendElement(
interp, pRes, Tcl_NewStringObj(sqlite3ErrName(rc), -1)
);
sqlite3_intck_error(p->intck, &zErr);
Tcl_ListObjAppendElement(
interp, pRes, Tcl_NewStringObj(zErr ? zErr : 0, -1)
);
Tcl_SetObjResult(interp, pRes);
break;
}
case 4: assert( 0==strcmp("unlock", aCmd[iIdx].zName) ); {
int rc = sqlite3_intck_unlock(p->intck);
Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
break;
}
case 5: assert( 0==strcmp("test_sql", aCmd[iIdx].zName) ); {
const char *zObj = Tcl_GetString(objv[2]);
const char *zSql = sqlite3_intck_test_sql(p->intck, zObj[0] ? zObj : 0);
Tcl_SetObjResult(interp, Tcl_NewStringObj(zSql, -1));
break;
}
}
return TCL_OK;
}
/*
** Destructor for commands created by test_sqlite3_intck().
*/
static void testIntckFree(void *clientData){
TestIntck *p = (TestIntck*)clientData;
sqlite3_intck_close(p->intck);
ckfree(p);
}
/*
** tclcmd: sqlite3_intck DB DBNAME
*/
static int test_sqlite3_intck(
void * clientData,
Tcl_Interp *interp,
int objc,
Tcl_Obj *CONST objv[]
){
char zName[64];
int iName = 0;
Tcl_CmdInfo info;
TestIntck *p = 0;
sqlite3 *db = 0;
const char *zDb = 0;
int rc = SQLITE_OK;
if( objc!=3 ){
Tcl_WrongNumArgs(interp, 1, objv, "DB DBNAME");
return TCL_ERROR;
}
p = (TestIntck*)ckalloc(sizeof(TestIntck));
memset(p, 0, sizeof(TestIntck));
if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){
return TCL_ERROR;
}
zDb = Tcl_GetString(objv[2]);
if( zDb[0]=='\0' ) zDb = 0;
rc = sqlite3_intck_open(db, zDb, &p->intck);
if( rc!=SQLITE_OK ){
ckfree(p);
Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3_errstr(rc), -1));
return TCL_ERROR;
}
do {
sprintf(zName, "intck%d", iName++);
}while( Tcl_GetCommandInfo(interp, zName, &info)!=0 );
Tcl_CreateObjCommand(interp, zName, testIntckCmd, (void*)p, testIntckFree);
Tcl_SetObjResult(interp, Tcl_NewStringObj(zName, -1));
return TCL_OK;
}
/*
** tclcmd: test_do_intck DB DBNAME
*/
static int test_do_intck(
void * clientData,
Tcl_Interp *interp,
int objc,
Tcl_Obj *CONST objv[]
){
sqlite3 *db = 0;
const char *zDb = 0;
int rc = SQLITE_OK;
sqlite3_intck *pCk = 0;
Tcl_Obj *pRet = 0;
const char *zErr = 0;
if( objc!=3 ){
Tcl_WrongNumArgs(interp, 1, objv, "DB DBNAME");
return TCL_ERROR;
}
if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){
return TCL_ERROR;
}
zDb = Tcl_GetString(objv[2]);
pRet = Tcl_NewObj();
Tcl_IncrRefCount(pRet);
rc = sqlite3_intck_open(db, zDb, &pCk);
if( rc==SQLITE_OK ){
while( sqlite3_intck_step(pCk)==SQLITE_OK ){
const char *zMsg = sqlite3_intck_message(pCk);
if( zMsg ){
Tcl_ListObjAppendElement(interp, pRet, Tcl_NewStringObj(zMsg, -1));
}
}
rc = sqlite3_intck_error(pCk, &zErr);
}
if( rc!=SQLITE_OK ){
if( zErr ){
Tcl_SetObjResult(interp, Tcl_NewStringObj(zErr, -1));
}else{
Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
}
}else{
Tcl_SetObjResult(interp, pRet);
}
Tcl_DecrRefCount(pRet);
sqlite3_intck_close(pCk);
sqlite3_intck_close(0);
return rc ? TCL_ERROR : TCL_OK;
}
int Sqlitetestintck_Init(Tcl_Interp *interp){
Tcl_CreateObjCommand(interp, "sqlite3_intck", test_sqlite3_intck, 0, 0);
Tcl_CreateObjCommand(interp, "test_do_intck", test_do_intck, 0, 0);
return TCL_OK;
}

290
ext/jni/GNUmakefile
View File

@ -26,28 +26,50 @@ dir.src.c := $(dir.src)/c
dir.bld := $(dir.jni)/bld
dir.bld.c := $(dir.bld)
dir.src.jni := $(dir.src)/org/sqlite/jni
dir.src.jni.tester := $(dir.src.jni)/tester
mkdir := mkdir -p
dir.src.capi := $(dir.src.jni)/capi
dir.src.fts5 := $(dir.src.jni)/fts5
dir.tests := $(dir.src)/tests
mkdir ?= mkdir -p
$(dir.bld.c):
$(mkdir) $@
javac.flags ?= -Xlint:unchecked -Xlint:deprecation
java.flags ?=
javac.flags += -encoding utf8
# -------------^^^^^^^^^^^^^^ required for Windows builds
jnicheck ?= 1
ifeq (1,$(jnicheck))
java.flags += -Xcheck:jni
endif
classpath := $(dir.src)
CLEAN_FILES := $(package.jar)
DISTCLEAN_FILES := $(dir.jni)/*~ $(dir.src.c)/*~ $(dir.src.jni)/*~
sqlite3-jni.h := $(dir.src.c)/sqlite3-jni.h
.NOTPARALLEL: $(sqlite3-jni.h)
SQLite3Jni.java := src/org/sqlite/jni/SQLite3Jni.java
SQLTester.java := src/org/sqlite/jni/tester/SQLTester.java
SQLite3Jni.class := $(SQLite3Jni.java:.java=.class)
CApi.java := $(dir.src.capi)/CApi.java
SQLTester.java := $(dir.src.capi)/SQLTester.java
CApi.class := $(CApi.java:.java=.class)
SQLTester.class := $(SQLTester.java:.java=.class)
########################################################################
# The future of FTS5 customization in this API is as yet unclear.
# It would be a real doozy to bind to JNI.
# The pieces are all in place, and are all thin proxies so not much
# complexity, but some semantic changes were required in porting
# which are largely untested.
#
# Reminder: this flag influences the contents of $(sqlite3-jni.h),
# which is checked in. Please do not check in changes to that file in
# which the fts5 APIs have been stripped unless that feature is
# intended to be stripped for good.
enable.fts5 ?= 1
# If enable.tester is 0, the org/sqlite/jni/tester/* bits are elided.
enable.tester ?= 1
ifeq (,$(wildcard $(dir.tests)/*))
enable.tester := 0
else
enable.tester := 1
endif
# bin.version-info = binary to output various sqlite3 version info
# building the distribution zip file.
@ -58,11 +80,11 @@ $(bin.version-info): $(dir.tool)/version-info.c $(sqlite3.h) $(dir.top)/Makefile
# Be explicit about which Java files to compile so that we can work on
# in-progress files without requiring them to be in a compilable statae.
JAVA_FILES.main := $(patsubst %,$(dir.src.jni)/%,\
annotation/Canonical.java \
annotation/NotNull.java \
annotation/Nullable.java \
annotation/package-info.java \
JAVA_FILES.main := $(patsubst %,$(dir.src.jni)/annotation/%,\
Experimental.java \
NotNull.java \
Nullable.java \
) $(patsubst %,$(dir.src.capi)/%,\
AbstractCollationCallback.java \
AggregateFunction.java \
AuthorizerCallback.java \
@ -71,69 +93,86 @@ JAVA_FILES.main := $(patsubst %,$(dir.src.jni)/%,\
CollationCallback.java \
CollationNeededCallback.java \
CommitHookCallback.java \
ConfigSqllogCallback.java \
ConfigLogCallback.java \
ConfigSqlLogCallback.java \
NativePointerHolder.java \
OutputPointer.java \
PrepareMultiCallback.java \
PreupdateHookCallback.java \
ProgressHandlerCallback.java \
ResultCode.java \
RollbackHookCallback.java \
ScalarFunction.java \
SQLFunction.java \
SQLite3Jni.java \
Tester1.java \
CallbackProxy.java \
CApi.java \
TableColumnMetadata.java \
TraceV2Callback.java \
UpdateHookCallback.java \
ValueHolder.java \
WindowFunction.java \
XDestroyCallback.java \
package-info.java \
sqlite3.java \
sqlite3_blob.java \
sqlite3_context.java \
sqlite3_stmt.java \
sqlite3_value.java \
) $(patsubst %,$(dir.src.jni)/wrapper1/%,\
AggregateFunction.java \
ScalarFunction.java \
SqlFunction.java \
Sqlite.java \
SqliteException.java \
ValueHolder.java \
WindowFunction.java \
)
JAVA_FILES.unittest := $(patsubst %,$(dir.src.jni)/%,\
capi/Tester1.java \
wrapper1/Tester2.java \
)
ifeq (1,$(enable.fts5))
JAVA_FILES.main += $(patsubst %,$(dir.src.jni)/%,\
JAVA_FILES.unittest += $(patsubst %,$(dir.src.fts5)/%,\
TesterFts5.java \
)
JAVA_FILES.main += $(patsubst %,$(dir.src.fts5)/%,\
fts5_api.java \
fts5_extension_function.java \
fts5_tokenizer.java \
Fts5.java \
Fts5Context.java \
Fts5ExtensionApi.java \
Fts5Function.java \
Fts5PhraseIter.java \
Fts5Tokenizer.java \
TesterFts5.java \
XTokenizeCallback.java \
)
endif
JAVA_FILES.tester := $(dir.src.jni.tester)/SQLTester.java
JAVA_FILES.tester := $(SQLTester.java)
JAVA_FILES.package.info := \
$(dir.src.jni)/package-info.java \
$(dir.src.jni)/annotation/package-info.java
CLASS_FILES.main := $(JAVA_FILES.main:.java=.class)
CLASS_FILES.unittest := $(JAVA_FILES.unittest:.java=.class)
CLASS_FILES.tester := $(JAVA_FILES.tester:.java=.class)
JAVA_FILES += $(JAVA_FILES.main)
JAVA_FILES += $(JAVA_FILES.main) $(JAVA_FILES.unittest)
ifeq (1,$(enable.tester))
JAVA_FILES += $(JAVA_FILES.tester)
endif
CLASS_FILES :=
define DOTCLASS_DEPS
$(1).class: $(1).java $(MAKEFILE)
define CLASSFILE_DEPS
all: $(1).class
$(1).class: $(1).java
CLASS_FILES += $(1).class
endef
$(foreach B,$(basename $(JAVA_FILES)),$(eval $(call DOTCLASS_DEPS,$(B))))
$(CLASS_FILES.tester): $(CLASS_FILES.main)
javac.flags ?= -Xlint:unchecked -Xlint:deprecation
java.flags ?=
jnicheck ?= 1
ifeq (1,$(jnicheck))
java.flags += -Xcheck:jni
endif
$(SQLite3Jni.class): $(JAVA_FILES)
$(foreach B,$(basename \
$(JAVA_FILES.main) $(JAVA_FILES.unittest) $(JAVA_FILES.tester)),\
$(eval $(call CLASSFILE_DEPS,$(B))))
$(CLASS_FILES): $(MAKEFILE)
$(bin.javac) $(javac.flags) -h $(dir.bld.c) -cp $(classpath) $(JAVA_FILES)
all: $(SQLite3Jni.class)
#.PHONY: classfiles
########################################################################
@ -168,9 +207,23 @@ $(sqlite3.h):
$(sqlite3.c): $(sqlite3.h)
opt.threadsafe ?= 1
opt.oom ?= 0
opt.fatal-oom ?= 1
opt.debug ?= 1
opt.metrics ?= 1
SQLITE_OPT = \
-DSQLITE_ENABLE_RTREE \
-DSQLITE_THREADSAFE=$(opt.threadsafe) \
-DSQLITE_TEMP_STORE=2 \
-DSQLITE_USE_URI=1 \
-DSQLITE_OMIT_LOAD_EXTENSION \
-DSQLITE_OMIT_DEPRECATED \
-DSQLITE_OMIT_SHARED_CACHE \
-DSQLITE_C=$(sqlite3.c) \
-DSQLITE_JNI_FATAL_OOM=$(opt.fatal-oom) \
-DSQLITE_JNI_ENABLE_METRICS=$(opt.metrics)
opt.extras ?= 1
ifeq (1,$(opt.extras))
SQLITE_OPT += -DSQLITE_ENABLE_RTREE \
-DSQLITE_ENABLE_EXPLAIN_COMMENTS \
-DSQLITE_ENABLE_STMTVTAB \
-DSQLITE_ENABLE_DBPAGE_VTAB \
@ -178,18 +231,16 @@ SQLITE_OPT = \
-DSQLITE_ENABLE_BYTECODE_VTAB \
-DSQLITE_ENABLE_OFFSET_SQL_FUNC \
-DSQLITE_ENABLE_PREUPDATE_HOOK \
-DSQLITE_ENABLE_NORMALIZE \
-DSQLITE_ENABLE_SQLLOG \
-DSQLITE_OMIT_LOAD_EXTENSION \
-DSQLITE_OMIT_DEPRECATED \
-DSQLITE_OMIT_SHARED_CACHE \
-DSQLITE_THREADSAFE=$(opt.threadsafe) \
-DSQLITE_TEMP_STORE=2 \
-DSQLITE_USE_URI=1 \
-DSQLITE_C=$(sqlite3.c) \
-DSQLITE_JNI_FATAL_OOM=$(opt.oom) \
-DSQLITE_DEBUG
-DSQLITE_ENABLE_COLUMN_METADATA
endif
SQLITE_OPT += -g -DDEBUG -UNDEBUG
ifeq (1,$(opt.debug))
SQLITE_OPT += -DSQLITE_DEBUG -g -DDEBUG -UNDEBUG
else
SQLITE_OPT += -Os
endif
ifeq (1,$(enable.fts5))
SQLITE_OPT += -DSQLITE_ENABLE_FTS5
@ -198,25 +249,30 @@ endif
sqlite3-jni.c := $(dir.src.c)/sqlite3-jni.c
sqlite3-jni.o := $(dir.bld.c)/sqlite3-jni.o
sqlite3-jni.h := $(dir.src.c)/sqlite3-jni.h
sqlite3-jni.dll := $(dir.bld.c)/libsqlite3-jni.so
package.dll := $(dir.bld.c)/libsqlite3-jni.so
# All javac-generated .h files must be listed in $(sqlite3-jni.h.in):
sqlite3-jni.h.in :=
# $(java.with.jni) lists all Java files which contain JNI decls:
java.with.jni :=
define ADD_JNI_H
sqlite3-jni.h.in += $$(dir.bld.c)/org_sqlite_jni_$(1).h
$$(dir.bld.c)/org_sqlite_jni_$(1).h: $$(dir.src.jni)/$(1).java
sqlite3-jni.h.in += $$(dir.bld.c)/org_sqlite_jni$(3)_$(2).h
java.with.jni += $(1)/$(2).java
$$(dir.bld.c)/org_sqlite_jni$(3)_$(2).h: $(1)/$(2).java
endef
$(eval $(call ADD_JNI_H,SQLite3Jni))
# Invoke ADD_JNI_H once for each Java file which includes JNI
# declarations:
$(eval $(call ADD_JNI_H,$(dir.src.capi),CApi,_capi))
$(eval $(call ADD_JNI_H,$(dir.src.capi),SQLTester,_capi))
ifeq (1,$(enable.fts5))
$(eval $(call ADD_JNI_H,Fts5ExtensionApi))
$(eval $(call ADD_JNI_H,fts5_api))
$(eval $(call ADD_JNI_H,fts5_tokenizer))
$(eval $(call ADD_JNI_H,$(dir.src.fts5),Fts5ExtensionApi,_fts5))
$(eval $(call ADD_JNI_H,$(dir.src.fts5),fts5_api,_fts5))
$(eval $(call ADD_JNI_H,$(dir.src.fts5),fts5_tokenizer,_fts5))
endif
ifeq (1,$(enable.tester))
sqlite3-jni.h.in += $(dir.bld.c)/org_sqlite_jni_tester_SQLTester.h
$(dir.bld.c)/org_sqlite_jni_tester_SQLTester.h: $(dir.src.jni.tester)/SQLTester.java
endif
#sqlite3-jni.dll.cfiles := $(dir.src.c)
sqlite3-jni.dll.cflags = \
$(sqlite3-jni.h.in): $(dir.bld.c)
#package.dll.cfiles :=
package.dll.cflags = \
-std=c99 \
-fPIC \
-I. \
-I$(dir $(sqlite3.h)) \
@ -224,39 +280,56 @@ sqlite3-jni.dll.cflags = \
-I$(JDK_HOME)/include \
$(patsubst %,-I%,$(patsubst %.h,,$(wildcard $(JDK_HOME)/include/*))) \
-Wall
# Using (-Wall -Wextra) triggers an untennable number of
# gcc warnings from sqlite3.c for mundane things like
# unused parameters.
#
# The gross $(patsubst...) above is to include the platform-specific
# subdir which lives under $(JDK_HOME)/include and is a required
# include path for client-level code.
#
# Using (-Wall -Wextra) triggers an untennable number of
# gcc warnings from sqlite3.c for mundane things like
# unused parameters.
########################################################################
ifeq (1,$(enable.tester))
sqlite3-jni.dll.cflags += -DSQLITE_JNI_ENABLE_SQLTester
package.dll.cflags += -DSQLITE_JNI_ENABLE_SQLTester
endif
$(sqlite3-jni.h): $(sqlite3-jni.h.in) $(MAKEFILE)
cat $(sqlite3-jni.h.in) > $@
$(sqlite3-jni.dll): $(sqlite3-jni.h) $(sqlite3.c) $(sqlite3.h)
$(sqlite3-jni.dll): $(dir.bld.c) $(sqlite3-jni.c) $(SQLite3Jni.java) $(MAKEFILE)
$(CC) $(sqlite3-jni.dll.cflags) $(SQLITE_OPT) \
@cat $(sqlite3-jni.h.in) > $@.tmp
@if cmp $@ $@.tmp >/dev/null; then \
rm -f $@.tmp; \
echo "$@ not modified"; \
else \
mv $@.tmp $@; \
echo "Updated $@"; \
fi
@if [ x1 != x$(enable.fts5) ]; then \
echo "*** REMINDER:"; \
echo "*** enable.fts5=0, so please do not check in changes to $@."; \
fi
$(package.dll): $(sqlite3-jni.h) $(sqlite3.c) $(sqlite3.h)
$(package.dll): $(sqlite3-jni.c) $(MAKEFILE)
$(CC) $(package.dll.cflags) $(SQLITE_OPT) \
$(sqlite3-jni.c) -shared -o $@
all: $(sqlite3-jni.dll)
all: $(package.dll)
.PHONY: test test-one
test.flags ?=
test.main.flags = -ea -Djava.library.path=$(dir.bld.c) \
$(java.flags) -cp $(classpath) \
org.sqlite.jni.Tester1
test.deps := $(SQLite3Jni.class) $(sqlite3-jni.dll)
Tester1.flags ?=
Tester2.flags ?=
test.flags.jvm = -ea -Djava.library.path=$(dir.bld.c) \
$(java.flags) -cp $(classpath)
test.deps := $(CLASS_FILES) $(package.dll)
test-one: $(test.deps)
$(bin.java) $(test.main.flags) $(test.flags)
$(bin.java) $(test.flags.jvm) org.sqlite.jni.capi.Tester1 $(Tester1.flags)
$(bin.java) $(test.flags.jvm) org.sqlite.jni.wrapper1.Tester2 $(Tester2.flags)
test-sqllog: $(test.deps)
@echo "Testing with -sqllog..."
$(bin.java) $(test.main.flags) -sqllog
$(bin.java) $(test.flags.jvm) org.sqlite.jni.capi.Tester1 $(Tester1.flags) -sqllog
test-mt: $(test.deps)
@echo "Testing in multi-threaded mode:";
$(bin.java) $(test.main.flags) -t 11 -r 50 -shuffle $(test.flags)
$(bin.java) $(test.flags.jvm) org.sqlite.jni.capi.Tester1 \
-t 7 -r 50 -shuffle $(Tester1.flags)
$(bin.java) $(test.flags.jvm) org.sqlite.jni.wrapper1.Tester2 \
-t 7 -r 50 -shuffle $(Tester2.flags)
test: test-one test-mt
tests: test test-sqllog
@ -265,24 +338,24 @@ tester.scripts := $(sort $(wildcard $(dir.src)/tests/*.test))
tester.flags ?= # --verbose
.PHONY: tester tester-local tester-ext
ifeq (1,$(enable.tester))
tester-local: $(CLASS_FILES.tester) $(sqlite3-jni.dll)
tester-local: $(CLASS_FILES.tester) $(package.dll)
$(bin.java) -ea -Djava.library.path=$(dir.bld.c) \
$(java.flags) -cp $(classpath) \
org.sqlite.jni.tester.SQLTester $(tester.flags) $(tester.scripts)
org.sqlite.jni.capi.SQLTester $(tester.flags) $(tester.scripts)
tester: tester-local
else
tester:
@echo "SQLTester support is disabled. Build with enable.tester=1 to enable it."
@echo "SQLTester support is disabled."
endif
tester.extdir.default := src/tests/ext
tester.extdir.default := $(dir.tests)/ext
tester.extdir ?= $(tester.extdir.default)
tester.extern-scripts := $(wildcard $(tester.extdir)/*.test)
ifneq (,$(tester.extern-scripts))
tester-ext:
$(bin.java) -ea -Djava.library.path=$(dir.bld.c) \
$(java.flags) -cp $(classpath) \
org.sqlite.jni.tester.SQLTester $(tester.flags) $(tester.extern-scripts)
org.sqlite.jni.capi.SQLTester $(tester.flags) $(tester.extern-scripts)
else
tester-ext:
@echo "******************************************************"; \
@ -296,41 +369,60 @@ endif
tester-ext: tester-local
tester: tester-ext
tests: tester
########################################################################
# Build each SQLITE_THREADMODE variant and run all tests against them.
multitest: clean
$(MAKE) opt.threadsafe=0 opt.oom=1 tests clean
$(MAKE) opt.threadsafe=0 opt.oom=0 tests clean
$(MAKE) opt.threadsafe=1 opt.oom=1 tests clean
$(MAKE) opt.threadsafe=1 opt.oom=0 tests clean
$(MAKE) opt.threadsafe=2 opt.oom=1 tests clean
$(MAKE) opt.threadsafe=2 opt.oom=0 tests clean
define MULTIOPT
multitest: multitest-$(1)
multitest-$(1):
$$(MAKE) opt.debug=$$(opt.debug) $(patsubst %,opt.%,$(2)) \
tests clean enable.fts5=1
endef
$(eval $(call MULTIOPT,01,threadsafe=0 oom=1))
$(eval $(call MULTIOPT,00,threadsafe=0 oom=0))
$(eval $(call MULTIOPT,11,threadsafe=1 oom=1))
$(eval $(call MULTIOPT,10,threadsafe=1 oom=0))
$(eval $(call MULTIOPT,21,threadsafe=2 oom=1))
$(eval $(call MULTIOPT,20,threadsafe=2 oom=0))
########################################################################
# jar bundle...
package.jar.in := $(abspath $(dir.src)/jar.in)
CLEAN_FILES += $(package.jar.in)
$(package.jar.in): $(MAKEFILE) $(CLASS_FILES.main)
cd $(dir.src); ls -1 org/sqlite/jni/*.java org/sqlite/jni/*.class > $@
@echo "To use this jar you will need the -Djava.library.path=DIR/CONTAINING/libsqlite3-jni.so flag."
@echo "e.g. java -jar $@ -Djava.library.path=bld"
JAVA_FILES.jar := $(JAVA_FILES.main) $(JAVA_FILES.unittest) $(JAVA_FILES.package.info)
CLASS_FILES.jar := $(filter-out %/package-info.class,$(JAVA_FILES.jar:.java=.class))
$(package.jar.in): $(package.dll) $(MAKEFILE)
ls -1 \
$(dir.src.jni)/*/*.java $(dir.src.jni)/*/*.class \
| sed -e 's,^$(dir.src)/,,' | sort > $@
$(package.jar): $(CLASS_FILES) $(MAKEFILE) $(package.jar.in)
rm -f $(dir.src)/c/*~ $(dir.src.jni)/*~
cd $(dir.src); $(bin.jar) -cfe ../$@ org.sqlite.jni.Tester1 @$(package.jar.in)
$(package.jar): $(CLASS_FILES.jar) $(MAKEFILE) $(package.jar.in)
@rm -f $(dir.src)/c/*~ $(dir.src.jni)/*~
cd $(dir.src); $(bin.jar) -cfe ../$@ org.sqlite.jni.capi.Tester1 @$(package.jar.in)
@ls -la $@
@echo "To use this jar you will need the -Djava.library.path=DIR/CONTAINING/libsqlite3-jni.so flag."
@echo "e.g. java -Djava.library.path=bld -jar $@"
jar: $(package.jar)
run-jar: $(package.jar) $(package.dll)
$(bin.java) -Djava.library.path=$(dir.bld) -jar $(package.jar) $(run-jar.flags)
########################################################################
# javadoc...
dir.doc := $(dir.jni)/javadoc
doc.index := $(dir.doc)/index.html
javadoc.exclude := -exclude org.sqlite.jni.fts5
# ^^^^ 2023-09-13: elide the fts5 parts from the public docs for
# the time being, as it's not clear where the Java bindings for
# those bits are going.
# javadoc.exclude += -exclude org.sqlite.jni.capi
# ^^^^ exclude the capi API only for certain builds (TBD)
$(doc.index): $(JAVA_FILES.main) $(MAKEFILE)
@if [ -d $(dir.doc) ]; then rm -fr $(dir.doc)/*; fi
$(bin.javadoc) -cp $(classpath) -d $(dir.doc) -quiet \
-subpackages org.sqlite.jni -exclude org.sqlite.jni.tester
-subpackages org.sqlite.jni $(javadoc.exclude)
@echo "javadoc output is in $@"
.PHONY: doc javadoc docserve
@ -347,8 +439,8 @@ docserve: $(doc.index)
# Clean up...
CLEAN_FILES += $(dir.bld.c)/* \
$(dir.src.jni)/*.class \
$(dir.src.jni.tester)/*.class \
$(sqlite3-jni.dll) \
$(dir.src.jni)/*/*.class \
$(package.dll) \
hs_err_pid*.log
.PHONY: clean distclean

43
ext/jni/README.md
View File

@ -16,9 +16,8 @@ Technical support is available in the forum:
> **FOREWARNING:** this subproject is very much in development and
subject to any number of changes. Please do not rely on any
information about its API until this disclaimer is removed. The JNI
bindings released with version 3.43 are a "tech preview" and 3.44
will be "final," at which point strong backward compatibility
guarantees will apply.
bindings released with version 3.43 are a "tech preview." Once
finalized, strong backward compatibility guarantees will apply.
Project goals/requirements:
@ -43,11 +42,13 @@ Non-goals:
- Creation of high-level OO wrapper APIs. Clients are free to create
them off of the C-style API.
- Virtual tables are unlikely to be supported due to the amount of
glue code needed to fit them into Java.
- Support for mixed-mode operation, where client code accesses SQLite
both via the Java-side API and the C API via their own native
code. In such cases, proxy functionalities (primarily callback
handler wrappers of all sorts) may fail because the C-side use of
the SQLite APIs will bypass those proxies.
code. Such cases would be a minefield of potential mis-interactions
between this project's JNI bindings and mixed-mode client code.
Hello World
@ -55,7 +56,7 @@ Hello World
```java
import org.sqlite.jni.*;
import static SQLite3Jni.*;
import static org.sqlite.jni.CApi.*;
...
@ -123,15 +124,13 @@ sensible default argument values. In all such cases they are thin
proxies around the corresponding C APIs and do not introduce new
semantics.
In some very few cases, Java-specific capabilities have been added in
In a few cases, Java-specific capabilities have been added in
new APIs, all of which have "_java" somewhere in their names.
Examples include:
- `sqlite3_result_java_object()`
- `sqlite3_column_java_object()`
- `sqlite3_column_java_casted()`
- `sqlite3_value_java_object()`
- `sqlite3_value_java_casted()`
which, as one might surmise, collectively enable the passing of
arbitrary Java objects from user-defined SQL functions through to the
@ -150,6 +149,9 @@ pending statements have been closed. Be aware that Java garbage
collection _cannot_ close a database or finalize a prepared statement.
Those things require explicit API calls.
Classes for which it is sensible support Java's `AutoCloseable`
interface so can be used with try-with-resources constructs.
Golden Rule #2: _Never_ Throw from Callbacks (Unless...)
------------------------------------------------------------------------
@ -159,14 +161,15 @@ retain C-like semantics. For example, they are not permitted to throw
or propagate exceptions and must return error information (if any) via
result codes or `null`. The only cases where the C-style APIs may
throw is through client-side misuse, e.g. passing in a null where it
shouldn't be used. The APIs clearly mark function parameters which
should not be null, but does not actively defend itself against such
misuse. Some C-style APIs explicitly accept `null` as a no-op for
usability's sake, and some of the JNI APIs deliberately return an
error code, instead of segfaulting, when passed a `null`.
may cause a `NullPointerException`. The APIs clearly mark function
parameters which should not be null, but does not generally actively
defend itself against such misuse. Some C-style APIs explicitly accept
`null` as a no-op for usability's sake, and some of the JNI APIs
deliberately return an error code, instead of segfaulting, when passed
a `null`.
Client-defined callbacks _must never throw exceptions_ unless _very
explicitly documented_ as being throw-safe. Exceptions are generally
explitly documented_ as being throw-safe. Exceptions are generally
reserved for higher-level bindings which are constructed to
specifically deal with them and ensure that they do not leak C-level
resources. In some cases, callback handlers are permitted to throw, in
@ -292,14 +295,14 @@ int sqlite3_create_function(sqlite3 db, String funcName, int nArgs,
`SQLFunction` is not used directly, but is instead instantiated via
one of its three subclasses:
- `SQLFunction.Scalar` implements simple scalar functions using but a
- `ScalarFunction` implements simple scalar functions using but a
single callback.
- `SQLFunction.Aggregate` implements aggregate functions using two
- `AggregateFunction` implements aggregate functions using two
callbacks.
- `SQLFunction.Window` implements window functions using four
- `WindowFunction` implements window functions using four
callbacks.
Search [`Tester1.java`](/file/ext/jni/src/org/sqlite/jni/Tester1.java) for
Search [`Tester1.java`](/file/ext/jni/src/org/sqlite/jni/capi/Tester1.java) for
`SQLFunction` for how it's used.
Reminder: see the disclaimer at the top of this document regarding the

3112
ext/jni/src/c/sqlite3-jni.c
View File

File diff suppressed because it is too large Load Diff

3972
ext/jni/src/c/sqlite3-jni.h
View File

File diff suppressed because it is too large Load Diff

72
ext/jni/src/org/sqlite/jni/AggregateFunction.java
View File

@ -1,72 +0,0 @@
/*
** 2023-08-25
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** This file is part of the JNI bindings for the sqlite3 C API.
*/
package org.sqlite.jni;
/**
A SQLFunction implementation for aggregate functions. Its T is the
data type of its "accumulator" state, an instance of which is
intended to be be managed using the getAggregateState() and
takeAggregateState() methods.
*/
public abstract class AggregateFunction<T> implements SQLFunction {
/**
As for the xStep() argument of the C API's
sqlite3_create_function(). If this function throws, the
exception is not propagated and a warning might be emitted to a
debugging channel.
*/
public abstract void xStep(sqlite3_context cx, sqlite3_value[] args);
/**
As for the xFinal() argument of the C API's sqlite3_create_function().
If this function throws, it is translated into an sqlite3_result_error().
*/
public abstract void xFinal(sqlite3_context cx);
/**
Optionally override to be notified when the UDF is finalized by
SQLite.
*/
public void xDestroy() {}
/** Per-invocation state for the UDF. */
private final SQLFunction.PerContextState<T> map =
new SQLFunction.PerContextState<>();
/**
To be called from the implementation's xStep() method, as well
as the xValue() and xInverse() methods of the {@link WindowFunction}
subclass, to fetch the current per-call UDF state. On the
first call to this method for any given sqlite3_context
argument, the context is set to the given initial value. On all other
calls, the 2nd argument is ignored.
@see SQLFunction.PerContextState#getAggregateState
*/
protected final ValueHolder<T> getAggregateState(sqlite3_context cx, T initialValue){
return map.getAggregateState(cx, initialValue);
}
/**
To be called from the implementation's xFinal() method to fetch
the final state of the UDF and remove its mapping.
see SQLFunction.PerContextState#takeAggregateState
*/
protected final T takeAggregateState(sqlite3_context cx){
return map.takeAggregateState(cx);
}
}

27
ext/jni/src/org/sqlite/jni/Fts5Function.java
View File

@ -1,27 +0,0 @@
/*
** 2023-08-04
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** This file is part of the JNI bindings for the sqlite3 C API.
*/
package org.sqlite.jni;
/**
Fts5Function is used in conjunction with the
sqlite3_create_fts_function() JNI-bound API to give that native code
access to the callback functions needed in order to implement
FTS5 auxiliary functions in Java.
*/
public abstract class Fts5Function {
public abstract void xFunction(Fts5ExtensionApi pApi, Fts5Context pFts,
sqlite3_context pCtx, sqlite3_value argv[]);
public void xDestroy() {}
}

155
ext/jni/src/org/sqlite/jni/ResultCode.java
View File

@ -1,155 +0,0 @@
/*
** 2023-07-21
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** This file is part of the JNI bindings for the sqlite3 C API.
*/
package org.sqlite.jni;
/**
This enum contains all of the core and "extended" result codes used
by the sqlite3 library. It is provided not for use with the C-style
API (with which it won't work) but for higher-level code which may
find it useful to map SQLite result codes to human-readable names.
*/
public enum ResultCode {
SQLITE_OK(SQLite3Jni.SQLITE_OK),
SQLITE_ERROR(SQLite3Jni.SQLITE_ERROR),
SQLITE_INTERNAL(SQLite3Jni.SQLITE_INTERNAL),
SQLITE_PERM(SQLite3Jni.SQLITE_PERM),
SQLITE_ABORT(SQLite3Jni.SQLITE_ABORT),
SQLITE_BUSY(SQLite3Jni.SQLITE_BUSY),
SQLITE_LOCKED(SQLite3Jni.SQLITE_LOCKED),
SQLITE_NOMEM(SQLite3Jni.SQLITE_NOMEM),
SQLITE_READONLY(SQLite3Jni.SQLITE_READONLY),
SQLITE_INTERRUPT(SQLite3Jni.SQLITE_INTERRUPT),
SQLITE_IOERR(SQLite3Jni.SQLITE_IOERR),
SQLITE_CORRUPT(SQLite3Jni.SQLITE_CORRUPT),
SQLITE_NOTFOUND(SQLite3Jni.SQLITE_NOTFOUND),
SQLITE_FULL(SQLite3Jni.SQLITE_FULL),
SQLITE_CANTOPEN(SQLite3Jni.SQLITE_CANTOPEN),
SQLITE_PROTOCOL(SQLite3Jni.SQLITE_PROTOCOL),
SQLITE_EMPTY(SQLite3Jni.SQLITE_EMPTY),
SQLITE_SCHEMA(SQLite3Jni.SQLITE_SCHEMA),
SQLITE_TOOBIG(SQLite3Jni.SQLITE_TOOBIG),
SQLITE_CONSTRAINT(SQLite3Jni.SQLITE_CONSTRAINT),
SQLITE_MISMATCH(SQLite3Jni.SQLITE_MISMATCH),
SQLITE_MISUSE(SQLite3Jni.SQLITE_MISUSE),
SQLITE_NOLFS(SQLite3Jni.SQLITE_NOLFS),
SQLITE_AUTH(SQLite3Jni.SQLITE_AUTH),
SQLITE_FORMAT(SQLite3Jni.SQLITE_FORMAT),
SQLITE_RANGE(SQLite3Jni.SQLITE_RANGE),
SQLITE_NOTADB(SQLite3Jni.SQLITE_NOTADB),
SQLITE_NOTICE(SQLite3Jni.SQLITE_NOTICE),
SQLITE_WARNING(SQLite3Jni.SQLITE_WARNING),
SQLITE_ROW(SQLite3Jni.SQLITE_ROW),
SQLITE_DONE(SQLite3Jni.SQLITE_DONE),
SQLITE_ERROR_MISSING_COLLSEQ(SQLite3Jni.SQLITE_ERROR_MISSING_COLLSEQ),
SQLITE_ERROR_RETRY(SQLite3Jni.SQLITE_ERROR_RETRY),
SQLITE_ERROR_SNAPSHOT(SQLite3Jni.SQLITE_ERROR_SNAPSHOT),
SQLITE_IOERR_READ(SQLite3Jni.SQLITE_IOERR_READ),
SQLITE_IOERR_SHORT_READ(SQLite3Jni.SQLITE_IOERR_SHORT_READ),
SQLITE_IOERR_WRITE(SQLite3Jni.SQLITE_IOERR_WRITE),
SQLITE_IOERR_FSYNC(SQLite3Jni.SQLITE_IOERR_FSYNC),
SQLITE_IOERR_DIR_FSYNC(SQLite3Jni.SQLITE_IOERR_DIR_FSYNC),
SQLITE_IOERR_TRUNCATE(SQLite3Jni.SQLITE_IOERR_TRUNCATE),
SQLITE_IOERR_FSTAT(SQLite3Jni.SQLITE_IOERR_FSTAT),
SQLITE_IOERR_UNLOCK(SQLite3Jni.SQLITE_IOERR_UNLOCK),
SQLITE_IOERR_RDLOCK(SQLite3Jni.SQLITE_IOERR_RDLOCK),
SQLITE_IOERR_DELETE(SQLite3Jni.SQLITE_IOERR_DELETE),
SQLITE_IOERR_BLOCKED(SQLite3Jni.SQLITE_IOERR_BLOCKED),
SQLITE_IOERR_NOMEM(SQLite3Jni.SQLITE_IOERR_NOMEM),
SQLITE_IOERR_ACCESS(SQLite3Jni.SQLITE_IOERR_ACCESS),
SQLITE_IOERR_CHECKRESERVEDLOCK(SQLite3Jni.SQLITE_IOERR_CHECKRESERVEDLOCK),
SQLITE_IOERR_LOCK(SQLite3Jni.SQLITE_IOERR_LOCK),
SQLITE_IOERR_CLOSE(SQLite3Jni.SQLITE_IOERR_CLOSE),
SQLITE_IOERR_DIR_CLOSE(SQLite3Jni.SQLITE_IOERR_DIR_CLOSE),
SQLITE_IOERR_SHMOPEN(SQLite3Jni.SQLITE_IOERR_SHMOPEN),
SQLITE_IOERR_SHMSIZE(SQLite3Jni.SQLITE_IOERR_SHMSIZE),
SQLITE_IOERR_SHMLOCK(SQLite3Jni.SQLITE_IOERR_SHMLOCK),
SQLITE_IOERR_SHMMAP(SQLite3Jni.SQLITE_IOERR_SHMMAP),
SQLITE_IOERR_SEEK(SQLite3Jni.SQLITE_IOERR_SEEK),
SQLITE_IOERR_DELETE_NOENT(SQLite3Jni.SQLITE_IOERR_DELETE_NOENT),
SQLITE_IOERR_MMAP(SQLite3Jni.SQLITE_IOERR_MMAP),
SQLITE_IOERR_GETTEMPPATH(SQLite3Jni.SQLITE_IOERR_GETTEMPPATH),
SQLITE_IOERR_CONVPATH(SQLite3Jni.SQLITE_IOERR_CONVPATH),
SQLITE_IOERR_VNODE(SQLite3Jni.SQLITE_IOERR_VNODE),
SQLITE_IOERR_AUTH(SQLite3Jni.SQLITE_IOERR_AUTH),
SQLITE_IOERR_BEGIN_ATOMIC(SQLite3Jni.SQLITE_IOERR_BEGIN_ATOMIC),
SQLITE_IOERR_COMMIT_ATOMIC(SQLite3Jni.SQLITE_IOERR_COMMIT_ATOMIC),
SQLITE_IOERR_ROLLBACK_ATOMIC(SQLite3Jni.SQLITE_IOERR_ROLLBACK_ATOMIC),
SQLITE_IOERR_DATA(SQLite3Jni.SQLITE_IOERR_DATA),
SQLITE_IOERR_CORRUPTFS(SQLite3Jni.SQLITE_IOERR_CORRUPTFS),
SQLITE_LOCKED_SHAREDCACHE(SQLite3Jni.SQLITE_LOCKED_SHAREDCACHE),
SQLITE_LOCKED_VTAB(SQLite3Jni.SQLITE_LOCKED_VTAB),
SQLITE_BUSY_RECOVERY(SQLite3Jni.SQLITE_BUSY_RECOVERY),
SQLITE_BUSY_SNAPSHOT(SQLite3Jni.SQLITE_BUSY_SNAPSHOT),
SQLITE_BUSY_TIMEOUT(SQLite3Jni.SQLITE_BUSY_TIMEOUT),
SQLITE_CANTOPEN_NOTEMPDIR(SQLite3Jni.SQLITE_CANTOPEN_NOTEMPDIR),
SQLITE_CANTOPEN_ISDIR(SQLite3Jni.SQLITE_CANTOPEN_ISDIR),
SQLITE_CANTOPEN_FULLPATH(SQLite3Jni.SQLITE_CANTOPEN_FULLPATH),
SQLITE_CANTOPEN_CONVPATH(SQLite3Jni.SQLITE_CANTOPEN_CONVPATH),
SQLITE_CANTOPEN_SYMLINK(SQLite3Jni.SQLITE_CANTOPEN_SYMLINK),
SQLITE_CORRUPT_VTAB(SQLite3Jni.SQLITE_CORRUPT_VTAB),
SQLITE_CORRUPT_SEQUENCE(SQLite3Jni.SQLITE_CORRUPT_SEQUENCE),
SQLITE_CORRUPT_INDEX(SQLite3Jni.SQLITE_CORRUPT_INDEX),
SQLITE_READONLY_RECOVERY(SQLite3Jni.SQLITE_READONLY_RECOVERY),
SQLITE_READONLY_CANTLOCK(SQLite3Jni.SQLITE_READONLY_CANTLOCK),
SQLITE_READONLY_ROLLBACK(SQLite3Jni.SQLITE_READONLY_ROLLBACK),
SQLITE_READONLY_DBMOVED(SQLite3Jni.SQLITE_READONLY_DBMOVED),
SQLITE_READONLY_CANTINIT(SQLite3Jni.SQLITE_READONLY_CANTINIT),
SQLITE_READONLY_DIRECTORY(SQLite3Jni.SQLITE_READONLY_DIRECTORY),
SQLITE_ABORT_ROLLBACK(SQLite3Jni.SQLITE_ABORT_ROLLBACK),
SQLITE_CONSTRAINT_CHECK(SQLite3Jni.SQLITE_CONSTRAINT_CHECK),
SQLITE_CONSTRAINT_COMMITHOOK(SQLite3Jni.SQLITE_CONSTRAINT_COMMITHOOK),
SQLITE_CONSTRAINT_FOREIGNKEY(SQLite3Jni.SQLITE_CONSTRAINT_FOREIGNKEY),
SQLITE_CONSTRAINT_FUNCTION(SQLite3Jni.SQLITE_CONSTRAINT_FUNCTION),
SQLITE_CONSTRAINT_NOTNULL(SQLite3Jni.SQLITE_CONSTRAINT_NOTNULL),
SQLITE_CONSTRAINT_PRIMARYKEY(SQLite3Jni.SQLITE_CONSTRAINT_PRIMARYKEY),
SQLITE_CONSTRAINT_TRIGGER(SQLite3Jni.SQLITE_CONSTRAINT_TRIGGER),
SQLITE_CONSTRAINT_UNIQUE(SQLite3Jni.SQLITE_CONSTRAINT_UNIQUE),
SQLITE_CONSTRAINT_VTAB(SQLite3Jni.SQLITE_CONSTRAINT_VTAB),
SQLITE_CONSTRAINT_ROWID(SQLite3Jni.SQLITE_CONSTRAINT_ROWID),
SQLITE_CONSTRAINT_PINNED(SQLite3Jni.SQLITE_CONSTRAINT_PINNED),
SQLITE_CONSTRAINT_DATATYPE(SQLite3Jni.SQLITE_CONSTRAINT_DATATYPE),
SQLITE_NOTICE_RECOVER_WAL(SQLite3Jni.SQLITE_NOTICE_RECOVER_WAL),
SQLITE_NOTICE_RECOVER_ROLLBACK(SQLite3Jni.SQLITE_NOTICE_RECOVER_ROLLBACK),
SQLITE_WARNING_AUTOINDEX(SQLite3Jni.SQLITE_WARNING_AUTOINDEX),
SQLITE_AUTH_USER(SQLite3Jni.SQLITE_AUTH_USER),
SQLITE_OK_LOAD_PERMANENTLY(SQLite3Jni.SQLITE_OK_LOAD_PERMANENTLY);
public final int value;
ResultCode(int rc){
value = rc;
ResultCodeMap.set(rc, this);
}
/**
Returns the entry from this enum for the given result code, or
null if no match is found.
*/
public static ResultCode getEntryForInt(int rc){
return ResultCodeMap.get(rc);
}
/**
Internal level of indirection required because we cannot initialize
static enum members in an enum before the enum constructor is
invoked.
*/
private static final class ResultCodeMap {
private static final java.util.Map<Integer,ResultCode> i2e
= new java.util.HashMap<>();
private static void set(int rc, ResultCode e){ i2e.put(rc, e); }
private static ResultCode get(int rc){ return i2e.get(rc); }
}
}

89
ext/jni/src/org/sqlite/jni/TesterFts5.java
View File

@ -1,89 +0,0 @@
/*
** 2023-08-04
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** This file contains a set of tests for the sqlite3 JNI bindings.
*/
package org.sqlite.jni;
import static org.sqlite.jni.SQLite3Jni.*;
import static org.sqlite.jni.Tester1.*;
public class TesterFts5 {
private static void test1(){
Fts5ExtensionApi fea = Fts5ExtensionApi.getInstance();
affirm( null != fea );
affirm( fea.getNativePointer() != 0 );
affirm( fea == Fts5ExtensionApi.getInstance() )/*singleton*/;
sqlite3 db = createNewDb();
fts5_api fApi = fts5_api.getInstanceForDb(db);
affirm( fApi != null );
affirm( fApi == fts5_api.getInstanceForDb(db) /* singleton per db */ );
execSql(db, new String[] {
"CREATE VIRTUAL TABLE ft USING fts5(a, b);",
"INSERT INTO ft(rowid, a, b) VALUES(1, 'X Y', 'Y Z');",
"INSERT INTO ft(rowid, a, b) VALUES(2, 'A Z', 'Y Y');"
});
final String pUserData = "This is pUserData";
ValueHolder<Boolean> xDestroyCalled = new ValueHolder<>(false);
ValueHolder<Integer> xFuncCount = new ValueHolder<>(0);
final fts5_extension_function func = new fts5_extension_function(){
public void xFunction(Fts5ExtensionApi ext, Fts5Context fCx,
sqlite3_context pCx, sqlite3_value argv[]){
int nCols = ext.xColumnCount(fCx);
affirm( 2 == nCols );
affirm( nCols == argv.length );
affirm( ext.xUserData(fCx) == pUserData );
if(true){
OutputPointer.String op = new OutputPointer.String();
for(int i = 0; i < nCols; ++i ){
int rc = ext.xColumnText(fCx, i, op);
affirm( 0 == rc );
final String val = op.value;
affirm( val.equals(sqlite3_value_text(argv[i])) );
//outln("xFunction col "+i+": "+val);
}
}
++xFuncCount.value;
}
public void xDestroy(){
xDestroyCalled.value = true;
}
};
int rc = fApi.xCreateFunction("myaux", pUserData, func);
affirm( 0==rc );
affirm( 0==xFuncCount.value );
execSql(db, "select myaux(ft,a,b) from ft;");
affirm( 2==xFuncCount.value );
affirm( !xDestroyCalled.value );
sqlite3_close_v2(db);
affirm( xDestroyCalled.value );
}
public TesterFts5(boolean verbose){
if(verbose){
final long timeStart = System.currentTimeMillis();
final int oldAffirmCount = Tester1.affirmCount;
test1();
final int affirmCount = Tester1.affirmCount - oldAffirmCount;
final long timeEnd = System.currentTimeMillis();
outln("FTS5 Tests done. Assertions checked = ",affirmCount,
", Total time = ",(timeEnd - timeStart),"ms");
}else{
test1();
}
}
public TesterFts5(){ this(false); }
}

44
ext/jni/src/org/sqlite/jni/annotation/Canonical.java
View File

@ -1,44 +0,0 @@
package org.sqlite.jni.annotation;
/**
This annotation is for marking functions as "canonical", meaning
that they map directly to a function in the core sqlite3 C API. The
intent is to distinguish them from functions added specifically to
the Java API.
<p>Canonical functions, unless specifically documented, have the
same semantics as their counterparts in
<a href="https://sqlite.org/c3ref/intro.html">the C API documentation</a>,
despite their signatures perhaps differing. The Java API adds a
number of overloads to simplify use, as well as a few Java-specific
functions, and those are never flagged as @Canonical.
<p>In some cases, the canonical version of a function is private
and exposed to Java via public overloads.
<p>In rare cases, the Java interface for a canonical function has a
different name than its C counterpart. For such cases,
(cname=the-C-side-name) is passed to this annotation and a
Java-side implementation with a slightly different signature is
added to with the canonical name. As of this writing, that applies
only to {@link org.sqlite.jni.SQLite3Jni#sqlite3_value_text_utf8}
and {@link org.sqlite.jni.SQLite3Jni#sqlite3_column_text_utf8}.
<p>The comment property can be used to add a comment.
*/
@java.lang.annotation.Documented
@java.lang.annotation.Retention(java.lang.annotation.RetentionPolicy.SOURCE)
@java.lang.annotation.Target(java.lang.annotation.ElementType.METHOD)
public @interface Canonical{
/**
Java functions which directly map to a canonical function but
change its name for some reason should not the original name
in this property.
*/
String cname() default ""/*doesn't allow null*/;
/**
Brief comments about the binding, e.g. noting any major
semantic differences.
*/
String comment() default "";
}

30
ext/jni/src/org/sqlite/jni/annotation/Experimental.java Normal file
View File

@ -0,0 +1,30 @@
/*
** 2023-09-27
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** This file houses the Experimental annotation for the sqlite3 C API.
*/
package org.sqlite.jni.annotation;
import java.lang.annotation.*;
/**
This annotation is for flagging methods, constructors, and types
which are expressly experimental and subject to any amount of
change or outright removal. Client code should not rely on such
features.
*/
@Documented
@Retention(RetentionPolicy.SOURCE)
@Target({
ElementType.METHOD,
ElementType.CONSTRUCTOR,
ElementType.TYPE
})
public @interface Experimental{}

69
ext/jni/src/org/sqlite/jni/annotation/NotNull.java
View File

@ -1,26 +1,71 @@
/*
** 2023-09-27
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** This file houses the NotNull annotation for the sqlite3 C API.
*/
package org.sqlite.jni.annotation;
import java.lang.annotation.*;
/**
This annotation is for flagging parameters which may not legally be
null. When used in the context of callback methods which are
called into from the C APIs, this annotation communicates that the
C API will never pass a null value to the callback.
null or point to closed/finalized C-side resources.
<p>In the case of Java types which map directly to C struct types
(e.g. {@link org.sqlite.jni.capi.sqlite3}, {@link
org.sqlite.jni.capi.sqlite3_stmt}, and {@link
org.sqlite.jni.capi.sqlite3_context}), a closed/finalized resource
is also considered to be null for purposes this annotation because
the C-side effect of passing such a handle is the same as if null
is passed.</p>
<p>When used in the context of Java interfaces which are called
from the C APIs, this annotation communicates that the C API will
never pass a null value to the callback for that parameter.</p>
<p>Passing a null, for this annotation's definition of null, for
any parameter marked with this annoation specifically invokes
undefined behavior (see below).</p>
<p>Passing 0 (i.e. C NULL) or a negative value for any long-type
parameter marked with this annoation specifically invokes undefined
behavior (see below). Such values are treated as C pointers in the
JNI layer.</p>
<p><b>Undefined behaviour:</b> the JNI build uses the {@code
SQLITE_ENABLE_API_ARMOR} build flag, meaning that the C code
invoked with invalid NULL pointers and the like will not invoke
undefined behavior in the conventional C sense, but may, for
example, return result codes which are not documented for the
affected APIs or may otherwise behave unpredictably. In no known
cases will such arguments result in C-level code dereferencing a
NULL pointer or accessing out-of-bounds (or otherwise invalid)
memory. In other words, they may cause unexpected behavior but
should never cause an outright crash or security issue.</p>
<p>Note that the C-style API does not throw any exceptions on its
own because it has a no-throw policy in order to retain its C-style
semantics, but it may trigger NullPointerExceptions (or similar) if
passed a null for a parameter flagged with this annotation.
passed a null for a parameter flagged with this annotation.</p>
<p>This annotation is informational only. No policy is in place to
programmatically ensure that NotNull is conformed to in client
code.
code.</p>
<p>This annotation is solely for the use by the classes in this
package but is made public so that javadoc will link to it from the
annotated functions. It is not part of the public API and
client-level code must not rely on it.
<p>This annotation is solely for the use by the classes in the
org.sqlite.jni package and subpackages, but is made public so that
javadoc will link to it from the annotated functions. It is not
part of the public API and client-level code must not rely on
it.</p>
*/
@java.lang.annotation.Documented
@java.lang.annotation.Retention(java.lang.annotation.RetentionPolicy.SOURCE)
@java.lang.annotation.Target(java.lang.annotation.ElementType.PARAMETER)
@Documented
@Retention(RetentionPolicy.SOURCE)
@Target(ElementType.PARAMETER)
public @interface NotNull{}

20
ext/jni/src/org/sqlite/jni/annotation/Nullable.java
View File

@ -1,4 +1,18 @@
/*
** 2023-09-27
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** This file houses the Nullable annotation for the sqlite3 C API.
*/
package org.sqlite.jni.annotation;
import java.lang.annotation.*;
/**
This annotation is for flagging parameters which may legally be
@ -13,7 +27,7 @@ package org.sqlite.jni.annotation;
annotated functions. It is not part of the public API and
client-level code must not rely on it.
*/
@java.lang.annotation.Documented
@java.lang.annotation.Retention(java.lang.annotation.RetentionPolicy.SOURCE)
@java.lang.annotation.Target(java.lang.annotation.ElementType.PARAMETER)
@Documented
@Retention(RetentionPolicy.SOURCE)
@Target(ElementType.PARAMETER)
public @interface Nullable{}

15
ext/jni/src/org/sqlite/jni/annotation/package-info.java
View File

@ -1,4 +1,17 @@
/*
** 2023-09-27
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
*/
/**
This package houses annotations specific a JNI binding to the SQLite3 C API.
This package houses annotations specific to the JNI bindings of the
SQLite3 C API.
*/
package org.sqlite.jni.annotation;

2
ext/jni/src/org/sqlite/jni/AbstractCollationCallback.java → ext/jni/src/org/sqlite/jni/capi/AbstractCollationCallback.java
View File

@ -11,7 +11,7 @@
*************************************************************************
** This file is part of the JNI bindings for the sqlite3 C API.
*/
package org.sqlite.jni;
package org.sqlite.jni.capi;
import org.sqlite.jni.annotation.NotNull;
/**

73
ext/jni/src/org/sqlite/jni/SQLFunction.java → ext/jni/src/org/sqlite/jni/capi/AggregateFunction.java
View File

@ -1,5 +1,5 @@
/*
** 2023-07-22
** 2023-08-25
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
@ -11,27 +11,36 @@
*************************************************************************
** This file is part of the JNI bindings for the sqlite3 C API.
*/
package org.sqlite.jni;
package org.sqlite.jni.capi;
/**
SQLFunction is used in conjunction with the
sqlite3_create_function() JNI-bound API to give that native code
access to the callback functions needed in order to implement SQL
functions in Java.
<p>
This class is not used by itself, but is a marker base class. The
three UDF types are modelled by the inner classes Scalar,
Aggregate<T>, and Window<T>. Most simply, clients may subclass
those, or create anonymous classes from them, to implement
UDFs. Clients are free to create their own classes for use with
UDFs, so long as they conform to the public interfaces defined by
those three classes. The JNI layer only actively relies on the
SQLFunction base class and the method names and signatures used by
the UDF callback interfaces.
A SQLFunction implementation for aggregate functions. Its T is the
data type of its "accumulator" state, an instance of which is
intended to be be managed using the getAggregateState() and
takeAggregateState() methods.
*/
public interface SQLFunction {
public abstract class AggregateFunction<T> implements SQLFunction {
/**
As for the xStep() argument of the C API's
sqlite3_create_function(). If this function throws, the
exception is not propagated and a warning might be emitted to a
debugging channel.
*/
public abstract void xStep(sqlite3_context cx, sqlite3_value[] args);
/**
As for the xFinal() argument of the C API's sqlite3_create_function().
If this function throws, it is translated into an sqlite3_result_error().
*/
public abstract void xFinal(sqlite3_context cx);
/**
Optionally override to be notified when the UDF is finalized by
SQLite.
*/
public void xDestroy() {}
/**
PerContextState assists aggregate and window functions in
@ -100,4 +109,30 @@ public interface SQLFunction {
}
}
/** Per-invocation state for the UDF. */
private final PerContextState<T> map = new PerContextState<>();
/**
To be called from the implementation's xStep() method, as well
as the xValue() and xInverse() methods of the {@link WindowFunction}
subclass, to fetch the current per-call UDF state. On the
first call to this method for any given sqlite3_context
argument, the context is set to the given initial value. On all other
calls, the 2nd argument is ignored.
@see SQLFunction.PerContextState#getAggregateState
*/
protected final ValueHolder<T> getAggregateState(sqlite3_context cx, T initialValue){
return map.getAggregateState(cx, initialValue);
}
/**
To be called from the implementation's xFinal() method to fetch
the final state of the UDF and remove its mapping.
see SQLFunction.PerContextState#takeAggregateState
*/
protected final T takeAggregateState(sqlite3_context cx){
return map.takeAggregateState(cx);
}
}

Some files were not shown because too many files have changed in this diff Show More