database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2025-08-08 14:02:16 +03:00
Code Activity

Merge latest trunk into this branch.

Browse Source 
FossilOrigin-Name: cb721d0b36268a7b0ef493fa4d7f6bcbaa9ead8b1990e3c3fae015fa1d545226
This commit is contained in:
dan
2017-03-31 11:20:20 +00:00
parent b5c39ac0c9 53b2459a36
commit fbf3bdcdbc
476 changed files with 55806 additions and 9625 deletions
Download Patch File Download Diff File Expand all files Collapse all files

101
Makefile.in
View File

@@ -30,7 +30,8 @@ BCC = @BUILD_CC@ @BUILD_CFLAGS@
#
CC = @CC@
CFLAGS = @CPPFLAGS@ @CFLAGS@
TCC = $(CC) $(CFLAGS) -I. -I${TOP}/src -I${TOP}/ext/rtree -I${TOP}/ext/fts3
TCC = ${CC} ${CFLAGS} -I. -I${TOP}/src -I${TOP}/ext/rtree -I${TOP}/ext/icu
TCC += -I${TOP}/ext/fts3 -I${TOP}/ext/async -I${TOP}/ext/session
# Define this for the autoconf-based build, so that the code knows it can
# include the generated config.h
@@ -179,7 +180,8 @@ LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \
mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \
notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \
pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
random.lo resolve.lo rowset.lo rtree.lo select.lo sqlite3rbu.lo status.lo \
random.lo resolve.lo rowset.lo rtree.lo \
sqlite3session.lo select.lo sqlite3rbu.lo status.lo \
table.lo threads.lo tokenize.lo treeview.lo trigger.lo \
update.lo util.lo vacuum.lo \
vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
@@ -341,14 +343,15 @@ SRC += \
SRC += \
$(TOP)/ext/rtree/rtree.h \
$(TOP)/ext/rtree/rtree.c
SRC += \
$(TOP)/ext/session/sqlite3session.c \
$(TOP)/ext/session/sqlite3session.h
SRC += \
$(TOP)/ext/rbu/sqlite3rbu.h \
$(TOP)/ext/rbu/sqlite3rbu.c
SRC += \
$(TOP)/ext/misc/json1.c
# Generated source code files
#
SRC += \
@@ -379,6 +382,7 @@ TESTSRC = \
$(TOP)/src/test_blob.c \
$(TOP)/src/test_btree.c \
$(TOP)/src/test_config.c \
$(TOP)/src/test_delete.c \
$(TOP)/src/test_demovfs.c \
$(TOP)/src/test_devsym.c \
$(TOP)/src/test_fs.c \
@@ -405,14 +409,17 @@ TESTSRC = \
$(TOP)/src/test_windirent.c \
$(TOP)/src/test_wsd.c \
$(TOP)/ext/fts3/fts3_term.c \
$(TOP)/ext/fts3/fts3_test.c \
$(TOP)/ext/fts3/fts3_test.c \
$(TOP)/ext/session/test_session.c \
$(TOP)/ext/rbu/test_rbu.c
# Statically linked extensions
#
TESTSRC += \
$(TOP)/ext/misc/amatch.c \
$(TOP)/ext/misc/carray.c \
$(TOP)/ext/misc/closure.c \
$(TOP)/ext/misc/csv.c \
$(TOP)/ext/misc/eval.c \
$(TOP)/ext/misc/fileio.c \
$(TOP)/ext/misc/fuzzer.c \
@@ -423,6 +430,7 @@ TESTSRC += \
$(TOP)/ext/misc/nextchar.c \
$(TOP)/ext/misc/percentile.c \
$(TOP)/ext/misc/regexp.c \
$(TOP)/ext/misc/remember.c \
$(TOP)/ext/misc/series.c \
$(TOP)/ext/misc/spellfix.c \
$(TOP)/ext/misc/totype.c \
@@ -474,7 +482,8 @@ TESTSRC2 = \
$(TOP)/ext/fts3/fts3_term.c \
$(TOP)/ext/fts3/fts3_tokenizer.c \
$(TOP)/ext/fts3/fts3_write.c \
$(TOP)/ext/async/sqlite3async.c
$(TOP)/ext/async/sqlite3async.c \
$(TOP)/ext/session/sqlite3session.c
# Header files used by all library source files.
#
@@ -533,7 +542,8 @@ TESTPROGS = \
testfixture$(TEXE) \
sqlite3$(TEXE) \
sqlite3_analyzer$(TEXE) \
sqldiff$(TEXE)
sqldiff$(TEXE) \
dbhash$(TEXE)
# Databases containing fuzzer test cases
#
@@ -541,7 +551,8 @@ FUZZDATA = \
$(TOP)/test/fuzzdata1.db \
$(TOP)/test/fuzzdata2.db \
$(TOP)/test/fuzzdata3.db \
$(TOP)/test/fuzzdata4.db
$(TOP)/test/fuzzdata4.db \
$(TOP)/test/fuzzdata5.db
# Standard options to testfixture
#
@@ -550,9 +561,14 @@ TESTOPTS = --verbose=file --output=test-out.txt
# Extra compiler options for various shell tools
#
SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4
# SHELL_OPT += -DSQLITE_ENABLE_FTS5
SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS
SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5
FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ
FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c
DBFUZZ_OPT =
# This is the default Makefile target. The objects listed here
# are what get build when you type just "make" with no arguments.
@@ -581,8 +597,15 @@ sqlite3$(TEXE): $(TOP)/src/shell.c sqlite3.c
$(TOP)/src/shell.c sqlite3.c \
$(LIBREADLINE) $(TLIBS) -rpath "$(libdir)"
sqldiff$(TEXE): $(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h
$(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS)
sqldiff$(TEXE): $(TOP)/tool/sqldiff.c sqlite3.lo sqlite3.h
$(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.lo $(TLIBS)
dbhash$(TEXE): $(TOP)/tool/dbhash.c sqlite3.lo sqlite3.h
$(LTLINK) -o $@ $(TOP)/tool/dbhash.c sqlite3.lo $(TLIBS)
scrub$(TEXE): $(TOP)/ext/misc/scrub.c sqlite3.lo
$(LTLINK) -o $@ -I. -DSCRUB_STANDALONE \
$(TOP)/ext/misc/scrub.c sqlite3.lo $(TLIBS)
srcck1$(BEXE): $(TOP)/tool/srcck1.c
$(BCC) -o srcck1$(BEXE) $(TOP)/tool/srcck1.c
@@ -594,11 +617,18 @@ fuzzershell$(TEXE): $(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h
$(LTLINK) -o $@ $(FUZZERSHELL_OPT) \
$(TOP)/tool/fuzzershell.c sqlite3.c $(TLIBS)
fuzzcheck$(TEXE): $(TOP)/test/fuzzcheck.c sqlite3.c sqlite3.h
$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/fuzzcheck.c sqlite3.c $(TLIBS)
fuzzcheck$(TEXE): $(FUZZCHECK_SRC) sqlite3.c sqlite3.h
$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) sqlite3.c $(TLIBS)
mptester$(TEXE): sqlite3.c $(TOP)/mptest/mptest.c
$(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.c \
ossshell$(TEXE): $(TOP)/test/ossfuzz.c $(TOP)/test/ossshell.c sqlite3.c sqlite3.h
$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/ossshell.c \
$(TOP)/test/ossfuzz.c sqlite3.c $(TLIBS)
dbfuzz$(TEXE): $(TOP)/test/dbfuzz.c sqlite3.c sqlite3.h
$(LTLINK) -o $@ $(DBFUZZ_OPT) $(TOP)/test/dbfuzz.c sqlite3.c $(TLIBS)
mptester$(TEXE): sqlite3.lo $(TOP)/mptest/mptest.c
$(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.lo \
$(TLIBS) -rpath "$(libdir)"
MPTEST1=./mptester$(TEXE) mptest.db $(TOP)/mptest/crash01.test --repeat 20
@@ -634,6 +664,7 @@ mptest: mptester$(TEXE)
sqlite3.c: .target_source $(TOP)/tool/mksqlite3c.tcl
$(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl
cp tsrc/shell.c tsrc/sqlite3ext.h .
cp $(TOP)/ext/session/sqlite3session.h .
sqlite3ext.h: .target_source
cp tsrc/sqlite3ext.h .
@@ -997,6 +1028,9 @@ fts3_write.lo: $(TOP)/ext/fts3/fts3_write.c $(HDR) $(EXTHDR)
rtree.lo: $(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR)
$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c
sqlite3session.lo: $(TOP)/ext/session/sqlite3session.c $(HDR) $(EXTHDR)
$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/session/sqlite3session.c
json1.lo: $(TOP)/ext/misc/json1.c
$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/misc/json1.c
@@ -1022,7 +1056,7 @@ FTS5_SRC = \
fts5parse.c: $(TOP)/ext/fts5/fts5parse.y lemon
cp $(TOP)/ext/fts5/fts5parse.y .
rm -f fts5parse.h
./lemon $(OPTS) fts5parse.y
./lemon$(BEXE) $(OPTS) fts5parse.y
fts5parse.h: fts5parse.c
@@ -1081,6 +1115,11 @@ fastfuzztest: fuzzcheck$(TEXE) $(FUZZDATA)
valgrindfuzz: fuzzcheck$(TEXT) $(FUZZDATA)
valgrind ./fuzzcheck$(TEXE) --cell-size-check --limit-mem 10M --timeout 600 $(FUZZDATA)
# The veryquick.test TCL tests.
#
tcltest: ./testfixture$(TEXE)
./testfixture$(TEXE) $(TOP)/test/veryquick.test $(TESTOPTS)
# Minimal testing that runs in less than 3 minutes
#
quicktest: ./testfixture$(TEXE)
@@ -1089,8 +1128,7 @@ quicktest: ./testfixture$(TEXE)
# This is the common case. Run many tests that do not take too long,
# including fuzzcheck, sqlite3_analyzer, and sqldiff tests.
#
test: $(TESTPROGS) sourcetest fastfuzztest
./testfixture$(TEXE) $(TOP)/test/veryquick.test $(TESTOPTS)
test: fastfuzztest sourcetest $(TESTPROGS) tcltest
# Run a test using valgrind. This can take a really long time
# because valgrind is so much slower than a native machine.
@@ -1117,6 +1155,10 @@ sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl
sqlite3_analyzer$(TEXE): sqlite3_analyzer.c
$(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS)
dbdump$(TEXE): $(TOP)/ext/misc/dbdump.c sqlite3.lo
$(LTLINK) -DDBDUMP_STANDALONE -o $@ \
$(TOP)/ext/misc/dbdump.c sqlite3.lo $(TLIBS)
showdb$(TEXE): $(TOP)/tool/showdb.c sqlite3.lo
$(LTLINK) -o $@ $(TOP)/tool/showdb.c sqlite3.lo $(TLIBS)
@@ -1129,17 +1171,25 @@ showjournal$(TEXE): $(TOP)/tool/showjournal.c sqlite3.lo
showwal$(TEXE): $(TOP)/tool/showwal.c sqlite3.lo
$(LTLINK) -o $@ $(TOP)/tool/showwal.c sqlite3.lo $(TLIBS)
changeset$(TEXE): $(TOP)/ext/session/changeset.c sqlite3.lo
$(LTLINK) -o $@ $(TOP)/ext/session/changeset.c sqlite3.lo $(TLIBS)
rollback-test$(TEXE): $(TOP)/tool/rollback-test.c sqlite3.lo
$(LTLINK) -o $@ $(TOP)/tool/rollback-test.c sqlite3.lo $(TLIBS)
LogEst$(TEXE): $(TOP)/tool/logest.c sqlite3.h
$(LTLINK) -I. -o $@ $(TOP)/tool/logest.c
wordcount$(TEXE): $(TOP)/test/wordcount.c sqlite3.c
$(LTLINK) -o $@ $(TOP)/test/wordcount.c sqlite3.c $(TLIBS)
wordcount$(TEXE): $(TOP)/test/wordcount.c sqlite3.lo
$(LTLINK) -o $@ $(TOP)/test/wordcount.c sqlite3.lo $(TLIBS)
speedtest1$(TEXE): $(TOP)/test/speedtest1.c sqlite3.lo
$(LTLINK) -o $@ $(TOP)/test/speedtest1.c sqlite3.lo $(TLIBS)
speedtest1$(TEXE): $(TOP)/test/speedtest1.c sqlite3.c
$(LTLINK) $(ST_OPT) -o $@ $(TOP)/test/speedtest1.c sqlite3.c $(TLIBS)
KV_OPT += -DSQLITE_DIRECT_OVERFLOW_READ
kvtest$(TEXE): $(TOP)/test/kvtest.c sqlite3.c
$(LTLINK) $(KV_OPT) -o $@ $(TOP)/test/kvtest.c sqlite3.c $(TLIBS)
rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.lo
$(LTLINK) -I. -o $@ $(TOP)/ext/rbu/rbu.c sqlite3.lo $(TLIBS)
@@ -1151,8 +1201,9 @@ loadfts$(EXE): $(TOP)/tool/loadfts.c libsqlite3.la
# symbols that do not begin with "sqlite3_". It is run as part of the
# releasetest.tcl script.
#
VALIDIDS=' sqlite3(changeset|changegroup|session)?_'
checksymbols: sqlite3.lo
nm -g --defined-only sqlite3.o | grep -v " sqlite3_" ; test $$? -ne 0
nm -g --defined-only sqlite3.lo | egrep -v $(VALIDIDS); test $$? -ne 0
echo '0 errors out of 1 tests'
# Build the amalgamation-autoconf package. The amalamgation-tarball target builds
@@ -1215,13 +1266,12 @@ clean:
rm -f lemon$(BEXE) lempar.c parse.* sqlite*.tar.gz
rm -f mkkeywordhash$(BEXE) keywordhash.h
rm -f *.da *.bb *.bbg gmon.out
rm -rf quota2a quota2b quota2c
rm -rf tsrc .target_source
rm -f tclsqlite3$(TEXE)
rm -f testfixture$(TEXE) test.db
rm -f LogEst$(TEXE) fts3view$(TEXE) rollback-test$(TEXE) showdb$(TEXE)
rm -f showjournal$(TEXE) showstat4$(TEXE) showwal$(TEXE) speedtest1$(TEXE)
rm -f wordcount$(TEXE)
rm -f wordcount$(TEXE) changeset$(TEXE)
rm -f sqlite3.dll sqlite3.lib sqlite3.exp sqlite3.def
rm -f sqlite3.c
rm -f sqlite3rc.h
@@ -1234,6 +1284,7 @@ clean:
rm -f fuzzershell fuzzershell.exe
rm -f fuzzcheck fuzzcheck.exe
rm -f sqldiff sqldiff.exe
rm -f dbhash dbhash.exe
rm -f fts5.* fts5parse.*
distclean: clean

286
Makefile.msc
View File

@@ -24,6 +24,27 @@ USE_AMALGAMATION = 1
USE_FULLWARN = 0
!ENDIF
# Set this non-0 to enable treating warnings as errors (-WX, etc) when
# compiling.
#
!IFNDEF USE_FATAL_WARN
USE_FATAL_WARN = 0
!ENDIF
# Set this non-0 to enable full runtime error checks (-RTC1, etc). This
# has no effect if (any) optimizations are enabled.
#
!IFNDEF USE_RUNTIME_CHECKS
USE_RUNTIME_CHECKS = 0
!ENDIF
# Set this non-0 to create a SQLite amalgamation file that excludes the
# various built-in extensions.
#
!IFNDEF MINIMAL_AMALGAMATION
MINIMAL_AMALGAMATION = 0
!ENDIF
# Set this non-0 to use "stdcall" calling convention for the core library
# and shell executable.
#
@@ -198,6 +219,12 @@ DEBUG = 0
OPTIMIZATIONS = 2
!ENDIF
# Set this to non-0 to enable support for the session extension.
#
!IFNDEF SESSION
SESSION = 0
!ENDIF
# Set the source code file to be used by executables and libraries when
# they need the amalgamation.
#
@@ -257,15 +284,50 @@ SQLITE3EXEPDB = /pdb:sqlite3sh.pdb
!ENDIF
!ENDIF
# <<mark>>
# These are the names of the customized Tcl header files used by various parts
# of this makefile when the stdcall calling convention is in use. It is not
# used for any other purpose.
#
!IFNDEF SQLITETCLH
SQLITETCLH = sqlite_tcl.h
!ENDIF
!IFNDEF SQLITETCLDECLSH
SQLITETCLDECLSH = sqlite_tclDecls.h
!ENDIF
# These are the additional targets that the targets that integrate with the
# Tcl library should depend on when compiling, etc.
#
!IFNDEF SQLITE_TCL_DEP
!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0
SQLITE_TCL_DEP = $(SQLITETCLDECLSH) $(SQLITETCLH)
!ELSE
SQLITE_TCL_DEP =
!ENDIF
!ENDIF
# <</mark>>
# These are the "standard" SQLite compilation options used when compiling for
# the Windows platform.
#
!IFNDEF OPT_FEATURE_FLAGS
!IF $(MINIMAL_AMALGAMATION)==0
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
!ENDIF
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1
!ENDIF
# Should the session extension be enabled? If so, add compilation options
# to enable it.
#
!IF $(SESSION)!=0
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_SESSION=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_PREUPDATE_HOOK=1
!ENDIF
# These are the "extended" SQLite compilation options used when compiling for
# the Windows 10 platform.
#
@@ -438,6 +500,12 @@ TCC = $(CC) -nologo -W4 -DINCLUDE_MSVC_H=1 $(CCOPTS) $(TCCOPTS)
TCC = $(CC) -nologo -W3 $(CCOPTS) $(TCCOPTS)
!ENDIF
# Check if warnings should be treated as errors when compiling.
#
!IF $(USE_FATAL_WARN)!=0
TCC = $(TCC) -WX
!ENDIF
TCC = $(TCC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -I$(TOP)\src -fp:precise
RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -I$(TOP)\src $(RCOPTS) $(RCCOPTS)
@@ -450,20 +518,32 @@ RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -I$(TOP)\src $(RCOPTS) $(RCCOPTS)
#
!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0
!IF "$(PLATFORM)"=="x86"
CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
# <<mark>>
TEST_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall -DINCLUDE_SQLITE_TCL_H=1 -DSQLITE_TCLAPI=__cdecl
# <</mark>>
!ELSE
!IFNDEF PLATFORM
CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
# <<mark>>
TEST_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall -DINCLUDE_SQLITE_TCL_H=1 -DSQLITE_TCLAPI=__cdecl
# <</mark>>
!ELSE
CORE_CCONV_OPTS =
SHELL_CCONV_OPTS =
# <<mark>>
TEST_CCONV_OPTS =
# <</mark>>
!ENDIF
!ENDIF
!ELSE
CORE_CCONV_OPTS =
SHELL_CCONV_OPTS =
# <<mark>>
TEST_CCONV_OPTS =
# <</mark>>
!ENDIF
# These are additional compiler options used for the core library.
@@ -607,6 +687,8 @@ TCC = $(TCC) -I$(TOP)\ext\fts3
RCC = $(RCC) -I$(TOP)\ext\fts3
TCC = $(TCC) -I$(TOP)\ext\rtree
RCC = $(RCC) -I$(TOP)\ext\rtree
TCC = $(TCC) -I$(TOP)\ext\session
RCC = $(RCC) -I$(TOP)\ext\session
!ENDIF
# The mksqlite3c.tcl script accepts some options on the command
@@ -614,12 +696,35 @@ RCC = $(RCC) -I$(TOP)\ext\rtree
# options are necessary in order to allow debugging symbols to
# work correctly with Visual Studio when using the amalgamation.
#
!IFNDEF MKSQLITE3C_TOOL
!IF $(MINIMAL_AMALGAMATION)!=0
MKSQLITE3C_TOOL = $(TOP)\tool\mksqlite3c-noext.tcl
!ELSE
MKSQLITE3C_TOOL = $(TOP)\tool\mksqlite3c.tcl
!ENDIF
!ENDIF
!IFNDEF MKSQLITE3C_ARGS
!IF $(DEBUG)>1
MKSQLITE3C_ARGS = --linemacros
!ELSE
MKSQLITE3C_ARGS =
!ENDIF
!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0
MKSQLITE3C_ARGS = $(MKSQLITE3C_ARGS) --useapicall
!ENDIF
!ENDIF
# The mksqlite3h.tcl script accepts some options on the command line.
# When compiling with stdcall support, some of these options are
# necessary.
#
!IFNDEF MKSQLITE3H_ARGS
!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0
MKSQLITE3H_ARGS = --useapicall
!ELSE
MKSQLITE3H_ARGS =
!ENDIF
!ENDIF
# <</mark>>
@@ -641,6 +746,10 @@ RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1
!IF $(DEBUG)>2
TCC = $(TCC) -DSQLITE_DEBUG=1
RCC = $(RCC) -DSQLITE_DEBUG=1
!IF $(DYNAMIC_SHELL)==0
TCC = $(TCC) -DSQLITE_ENABLE_WHERETRACE -DSQLITE_ENABLE_SELECTTRACE
RCC = $(RCC) -DSQLITE_ENABLE_WHERETRACE -DSQLITE_ENABLE_SELECTTRACE
!ENDIF
!ENDIF
!IF $(DEBUG)>4 || $(OSTRACE)!=0
@@ -702,11 +811,11 @@ TCLLIBDIR = c:\tcl\lib
!ENDIF
!IFNDEF LIBTCL
LIBTCL = tcl85.lib
LIBTCL = tcl86.lib
!ENDIF
!IFNDEF LIBTCLSTUB
LIBTCLSTUB = tclstub85.lib
LIBTCLSTUB = tclstub86.lib
!ENDIF
!IFNDEF LIBTCLPATH
@@ -736,7 +845,7 @@ LIBICU = icuuc.lib icuin.lib
# specific Tcl shell to use.
#
!IFNDEF TCLSH_CMD
TCLSH_CMD = tclsh85
TCLSH_CMD = tclsh
!ENDIF
# <</mark>>
@@ -819,6 +928,10 @@ RCC = $(RCC) -D_DEBUG
!IF $(DEBUG)>1 || $(OPTIMIZATIONS)==0
TCC = $(TCC) -Od
BCC = $(BCC) -Od
!IF $(USE_RUNTIME_CHECKS)!=0
TCC = $(TCC) -RTC1
BCC = $(BCC) -RTC1
!ENDIF
!ELSEIF $(OPTIMIZATIONS)>=3
TCC = $(TCC) -Ox
BCC = $(BCC) -Ox
@@ -991,7 +1104,8 @@ LIBOBJS0 = vdbe.lo parse.lo alter.lo analyze.lo attach.lo auth.lo \
mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \
notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \
pager.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
random.lo resolve.lo rowset.lo rtree.lo select.lo sqlite3rbu.lo status.lo \
random.lo resolve.lo rowset.lo rtree.lo \
sqlite3session.lo select.lo sqlite3rbu.lo status.lo \
table.lo threads.lo tokenize.lo treeview.lo trigger.lo \
update.lo util.lo vacuum.lo \
vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
@@ -1178,6 +1292,7 @@ SRC07 = \
$(TOP)\ext\fts3\fts3_write.c \
$(TOP)\ext\icu\icu.c \
$(TOP)\ext\rtree\rtree.c \
$(TOP)\ext\session\sqlite3session.c \
$(TOP)\ext\rbu\sqlite3rbu.c \
$(TOP)\ext\misc\json1.c
@@ -1200,7 +1315,8 @@ SRC09 = \
$(TOP)\ext\fts3\fts3_tokenizer.h \
$(TOP)\ext\icu\sqliteicu.h \
$(TOP)\ext\rtree\rtree.h \
$(TOP)\ext\rbu\sqlite3rbu.h
$(TOP)\ext\rbu\sqlite3rbu.h \
$(TOP)\ext\session\sqlite3session.h
# Generated source code files
#
@@ -1216,6 +1332,16 @@ SRC11 = \
parse.h \
$(SQLITE3H)
# Generated Tcl header files
#
!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0
SRC12 = \
$(SQLITETCLH) \
$(SQLITETCLDECLSH)
!ELSE
SRC12 =
!ENDIF
# All source code files.
#
SRC = $(SRC00) $(SRC01) $(SRC02) $(SRC03) $(SRC04) $(SRC05) $(SRC06) $(SRC07) $(SRC08) $(SRC09) $(SRC10) $(SRC11)
@@ -1239,6 +1365,7 @@ TESTSRC = \
$(TOP)\src\test_blob.c \
$(TOP)\src\test_btree.c \
$(TOP)\src\test_config.c \
$(TOP)\src\test_delete.c \
$(TOP)\src\test_demovfs.c \
$(TOP)\src\test_devsym.c \
$(TOP)\src\test_fs.c \
@@ -1266,13 +1393,16 @@ TESTSRC = \
$(TOP)\src\test_wsd.c \
$(TOP)\ext\fts3\fts3_term.c \
$(TOP)\ext\fts3\fts3_test.c \
$(TOP)\ext\rbu\test_rbu.c
$(TOP)\ext\rbu\test_rbu.c \
$(TOP)\ext\session\test_session.c
# Statically linked extensions.
#
TESTEXT = \
$(TOP)\ext\misc\amatch.c \
$(TOP)\ext\misc\carray.c \
$(TOP)\ext\misc\closure.c \
$(TOP)\ext\misc\csv.c \
$(TOP)\ext\misc\eval.c \
$(TOP)\ext\misc\fileio.c \
$(TOP)\ext\misc\fuzzer.c \
@@ -1283,12 +1413,14 @@ TESTEXT = \
$(TOP)\ext\misc\nextchar.c \
$(TOP)\ext\misc\percentile.c \
$(TOP)\ext\misc\regexp.c \
$(TOP)\ext\misc\remember.c \
$(TOP)\ext\misc\series.c \
$(TOP)\ext\misc\spellfix.c \
$(TOP)\ext\misc\totype.c \
$(TOP)\ext\misc\wholenumber.c
# Source code to the library files needed by the test fixture
# (non-amalgamation)
#
TESTSRC2 = \
$(SRC00) \
@@ -1318,7 +1450,7 @@ HDR = \
parse.h \
$(TOP)\src\pragma.h \
$(SQLITE3H) \
$(TOP)\src\sqlite3ext.h \
sqlite3ext.h \
$(TOP)\src\sqliteInt.h \
$(TOP)\src\sqliteLimit.h \
$(TOP)\src\vdbe.h \
@@ -1347,6 +1479,8 @@ EXTHDR = $(EXTHDR) \
$(TOP)\ext\icu\sqliteicu.h
EXTHDR = $(EXTHDR) \
$(TOP)\ext\rtree\sqlite3rtree.h
EXTHDR = $(EXTHDR) \
$(TOP)\ext\session\sqlite3session.h
# executables needed for testing
#
@@ -1354,7 +1488,8 @@ TESTPROGS = \
testfixture.exe \
$(SQLITE3EXE) \
sqlite3_analyzer.exe \
sqldiff.exe
sqldiff.exe \
dbhash.exe
# Databases containing fuzzer test cases
#
@@ -1362,7 +1497,8 @@ FUZZDATA = \
$(TOP)\test\fuzzdata1.db \
$(TOP)\test\fuzzdata2.db \
$(TOP)\test\fuzzdata3.db \
$(TOP)\test\fuzzdata4.db
$(TOP)\test\fuzzdata4.db \
$(TOP)\test\fuzzdata5.db
# <</mark>>
# Additional compiler options for the shell. These are only effective
@@ -1377,7 +1513,13 @@ SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_
#
MPTESTER_COMPILE_OPTS = -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS5
FUZZERSHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1
FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5
FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ -DSQLITE_MAX_MEMORY=50000000
FUZZCHECK_SRC = $(TOP)\test\fuzzcheck.c $(TOP)\test\ossfuzz.c
OSSSHELL_SRC = $(TOP)\test\ossshell.c $(TOP)\test\ossfuzz.c
DBFUZZ_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION
KV_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_DIRECT_OVERFLOW_READ
DBSELFTEST_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5
ST_COMPILE_OPTS = -DSQLITE_THREADSAFE=0
# Standard options to testfixture.
#
@@ -1420,7 +1562,7 @@ $(SQLITE3DLL): $(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP)
sqlite3.def: libsqlite3.lib
echo EXPORTS > sqlite3.def
dumpbin /all libsqlite3.lib \
| $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3_[^@]*)(?:@\d+)?$$" \1 \
| $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3(?:session|changeset)?_[^@]*)(?:@\d+)?$$" \1 \
| sort >> sqlite3.def
# <</block2>>
@@ -1432,6 +1574,12 @@ $(SQLITE3EXE): $(TOP)\src\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_S
sqldiff.exe: $(TOP)\tool\sqldiff.c $(SQLITE3C) $(SQLITE3H)
$(LTLINK) $(NO_WARN) $(TOP)\tool\sqldiff.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
dbhash.exe: $(TOP)\tool\dbhash.c $(SQLITE3C) $(SQLITE3H)
$(LTLINK) $(NO_WARN) $(TOP)\tool\dbhash.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
scrub.exe: $(TOP)\ext\misc\scrub.c $(SQLITE3C) $(SQLITE3H)
$(LTLINK) $(NO_WARN) $(TOP)\ext\misc\scrub.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
srcck1.exe: $(TOP)\tool\srcck1.c
$(BCC) $(NO_WARN) -Fe$@ $(TOP)\tool\srcck1.c
@@ -1441,8 +1589,14 @@ sourcetest: srcck1.exe sqlite3.c
fuzzershell.exe: $(TOP)\tool\fuzzershell.c $(SQLITE3C) $(SQLITE3H)
$(LTLINK) $(NO_WARN) $(FUZZERSHELL_COMPILE_OPTS) $(TOP)\tool\fuzzershell.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
fuzzcheck.exe: $(TOP)\test\fuzzcheck.c $(SQLITE3C) $(SQLITE3H)
$(LTLINK) $(NO_WARN) $(FUZZCHECK_COMPILE_OPTS) $(TOP)\test\fuzzcheck.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
dbfuzz.exe: $(TOP)\test\dbfuzz.c $(SQLITE3C) $(SQLITE3H)
$(LTLINK) $(NO_WARN) $(DBFUZZ_COMPILE_OPTS) $(TOP)\test\dbfuzz.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
fuzzcheck.exe: $(FUZZCHECK_SRC) $(SQLITE3C) $(SQLITE3H)
$(LTLINK) $(NO_WARN) $(FUZZCHECK_COMPILE_OPTS) $(FUZZCHECK_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
ossshell.exe: $(OSSSHELL_SRC) $(SQLITE3C) $(SQLITE3H)
$(LTLINK) $(NO_WARN) $(FUZZCHECK_COMPILE_OPTS) $(OSSSHELL_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
mptester.exe: $(TOP)\mptest\mptest.c $(SQLITE3C) $(SQLITE3H)
$(LTLINK) $(NO_WARN) $(MPTESTER_COMPILE_OPTS) $(TOP)\mptest\mptest.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
@@ -1467,7 +1621,7 @@ mptest: mptester.exe
# files are automatically generated. This target takes care of
# all that automatic generation.
#
.target_source: $(SRC) $(TOP)\tool\vdbe-compress.tcl fts5.c
.target_source: $(SRC) $(TOP)\tool\vdbe-compress.tcl fts5.c $(SQLITE_TCL_DEP)
-rmdir /Q/S tsrc 2>NUL
-mkdir tsrc
for %i in ($(SRC00)) do copy /Y %i tsrc
@@ -1482,6 +1636,7 @@ mptest: mptester.exe
for %i in ($(SRC09)) do copy /Y %i tsrc
for %i in ($(SRC10)) do copy /Y %i tsrc
for %i in ($(SRC11)) do copy /Y %i tsrc
for %i in ($(SRC12)) do copy /Y %i tsrc
copy /Y fts5.c tsrc
copy /Y fts5.h tsrc
del /Q tsrc\sqlite.h.in tsrc\parse.y 2>NUL
@@ -1489,9 +1644,10 @@ mptest: mptester.exe
move vdbe.new tsrc\vdbe.c
echo > .target_source
sqlite3.c: .target_source sqlite3ext.h $(TOP)\tool\mksqlite3c.tcl
$(TCLSH_CMD) $(TOP)\tool\mksqlite3c.tcl $(MKSQLITE3C_ARGS)
sqlite3.c: .target_source sqlite3ext.h $(MKSQLITE3C_TOOL)
$(TCLSH_CMD) $(MKSQLITE3C_TOOL) $(MKSQLITE3C_ARGS)
copy tsrc\shell.c .
copy $(TOP)\ext\session\sqlite3session.h .
sqlite3-all.c: sqlite3.c $(TOP)\tool\split-sqlite3c.tcl
$(TCLSH_CMD) $(TOP)\tool\split-sqlite3c.tcl
@@ -1762,10 +1918,10 @@ wherecode.lo: $(TOP)\src\wherecode.c $(HDR)
whereexpr.lo: $(TOP)\src\whereexpr.c $(HDR)
$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\whereexpr.c
tclsqlite.lo: $(TOP)\src\tclsqlite.c $(HDR)
tclsqlite.lo: $(TOP)\src\tclsqlite.c $(HDR) $(SQLITE_TCL_DEP)
$(LTCOMPILE) $(NO_WARN) -DUSE_TCL_STUBS=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c
tclsqlite-shell.lo: $(TOP)\src\tclsqlite.c $(HDR)
tclsqlite-shell.lo: $(TOP)\src\tclsqlite.c $(HDR) $(SQLITE_TCL_DEP)
$(LTCOMPILE) $(NO_WARN) -DTCLSH=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c
tclsqlite3.exe: tclsqlite-shell.lo $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS)
@@ -1791,10 +1947,16 @@ parse.c: $(TOP)\src\parse.y lemon.exe $(TOP)\tool\addopcodes.tcl
$(TCLSH_CMD) $(TOP)\tool\addopcodes.tcl parse.h.temp > parse.h
$(SQLITE3H): $(TOP)\src\sqlite.h.in $(TOP)\manifest.uuid $(TOP)\VERSION
$(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP:\=/) > $(SQLITE3H)
$(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP:\=/) > $(SQLITE3H) $(MKSQLITE3H_ARGS)
sqlite3ext.h: .target_source
copy tsrc\sqlite3ext.h .
!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0
type tsrc\sqlite3ext.h | $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "\(\*\)" "(SQLITE_CALLBACK *)" \
| $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "\(\*" "(SQLITE_APICALL *" > sqlite3ext.h
copy /Y sqlite3ext.h tsrc\sqlite3ext.h
!ELSE
copy /Y tsrc\sqlite3ext.h sqlite3ext.h
!ENDIF
mkkeywordhash.exe: $(TOP)\tool\mkkeywordhash.c
$(BCC) $(NO_WARN) -Fe$@ $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) $(OPTS) \
@@ -1870,6 +2032,9 @@ fts3_write.lo: $(TOP)\ext\fts3\fts3_write.c $(HDR) $(EXTHDR)
rtree.lo: $(TOP)\ext\rtree\rtree.c $(HDR) $(EXTHDR)
$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\rtree\rtree.c
sqlite3session.lo: $(TOP)\ext\session\sqlite3session.c $(HDR) $(EXTHDR)
$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\session\sqlite3session.c
# FTS5 things
#
FTS5_SRC = \
@@ -1924,6 +2089,7 @@ TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERVER=1 -DSQLITE_PRIVATE=""
TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_CORE $(NO_WARN)
TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_DEFAULT_PAGE_SIZE=1024
TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) $(TEST_CCONV_OPTS)
TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2)
TESTFIXTURE_SRC1 = $(TESTEXT) $(SQLITE3C)
@@ -1933,7 +2099,27 @@ TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC0)
TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC1)
!ENDIF
testfixture.exe: $(TESTFIXTURE_SRC) $(SQLITE3H) $(LIBRESOBJS) $(HDR)
!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0
sqlite_tclDecls.h:
echo #ifndef SQLITE_TCLAPI > $(SQLITETCLDECLSH)
echo # define SQLITE_TCLAPI >> $(SQLITETCLDECLSH)
echo #endif >> $(SQLITETCLDECLSH)
type "$(TCLINCDIR)\tclDecls.h" \
| $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "^(EXTERN(?: CONST\d+?)?\s+?[^\(]*?\s+?)Tcl_" "\1 SQLITE_TCLAPI Tcl_" \
| $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "^(EXTERN\s+?(?:void|VOID)\s+?)TclFreeObj" "\1 SQLITE_TCLAPI TclFreeObj" \
| $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "\(\*tcl_" "(SQLITE_TCLAPI *tcl_" \
| $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "\(\*tclFreeObj" "(SQLITE_TCLAPI *tclFreeObj" \
| $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "\(\*" "(SQLITE_TCLAPI *" >> $(SQLITETCLDECLSH)
sqlite_tcl.h:
type "$(TCLINCDIR)\tcl.h" | $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact tclDecls.h sqlite_tclDecls.h \
| $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "typedef (.*?)\(Tcl_" "typedef \1 (SQLITE_TCLAPI Tcl_" \
| $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact "void (*freeProc)" "void (SQLITE_TCLAPI *freeProc)" \
| $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact "Tcl_HashEntry *(*findProc)" "Tcl_HashEntry *(SQLITE_TCLAPI *findProc)" \
| $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact "Tcl_HashEntry *(*createProc)" "Tcl_HashEntry *(SQLITE_TCLAPI *createProc)" >> $(SQLITETCLH)
!ENDIF
testfixture.exe: $(TESTFIXTURE_SRC) $(SQLITE3H) $(LIBRESOBJS) $(HDR) $(SQLITE_TCL_DEP)
$(LTLINK) -DSQLITE_NO_SYNC=1 $(TESTFIXTURE_FLAGS) \
-DBUILD_sqlite -I$(TCLINCDIR) \
$(TESTFIXTURE_SRC) \
@@ -1982,7 +2168,7 @@ smoketest: $(TESTPROGS)
@set PATH=$(LIBTCLPATH);$(PATH)
.\testfixture.exe $(TOP)\test\main.test $(TESTOPTS)
sqlite3_analyzer.c: $(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl
sqlite3_analyzer.c: $(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl $(SQLITE_TCL_DEP)
echo #define TCLSH 2 > $@
echo #define SQLITE_ENABLE_DBSTAT_VTAB 1 >> $@
copy $@ + $(SQLITE3C) + $(TOP)\src\tclsqlite.c $@
@@ -1995,6 +2181,10 @@ sqlite3_analyzer.exe: sqlite3_analyzer.c $(LIBRESOBJS)
$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_analyzer.c \
/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
dbdump.exe: $(TOP)\ext\misc\dbdump.c $(SQLITE3C) $(SQLITE3H)
$(LTLINK) $(NO_WARN) -DDBDUMP_STANDALONE $(TOP)\ext\misc\dbdump.c $(SQLITE3C) \
/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS)
testloadext.lo: $(TOP)\src\test_loadext.c
$(LTCOMPILE) $(NO_WARN) -c $(TOP)\src\test_loadext.c
@@ -2002,42 +2192,54 @@ testloadext.dll: testloadext.lo
$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ testloadext.lo
showdb.exe: $(TOP)\tool\showdb.c $(SQLITE3C) $(SQLITE3H)
$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
$(TOP)\tool\showdb.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
showstat4.exe: $(TOP)\tool\showstat4.c $(SQLITE3C) $(SQLITE3H)
$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
$(TOP)\tool\showstat4.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
showjournal.exe: $(TOP)\tool\showjournal.c $(SQLITE3C) $(SQLITE3H)
$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
$(TOP)\tool\showjournal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
showwal.exe: $(TOP)\tool\showwal.c $(SQLITE3C) $(SQLITE3H)
$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
$(TOP)\tool\showwal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
changeset.exe: $(TOP)\ext\session\changeset.c $(SQLITE3C) $(SQLITE3H)
$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
-DSQLITE_ENABLE_SESSION=1 -DSQLITE_ENABLE_PREUPDATE_HOOK=1 \
$(TOP)\ext\session\changeset.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
fts3view.exe: $(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) $(SQLITE3H)
$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
$(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
rollback-test.exe: $(TOP)\tool\rollback-test.c $(SQLITE3C) $(SQLITE3H)
$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
$(TOP)\tool\rollback-test.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
LogEst.exe: $(TOP)\tool\logest.c $(SQLITE3H)
$(LTLINK) $(NO_WARN) -Fe$@ $(TOP)\tool\LogEst.c /link $(LDFLAGS) $(LTLINKOPTS)
$(LTLINK) $(NO_WARN) $(TOP)\tool\LogEst.c /link $(LDFLAGS) $(LTLINKOPTS)
wordcount.exe: $(TOP)\test\wordcount.c $(SQLITE3C) $(SQLITE3H)
$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
$(TOP)\test\wordcount.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
speedtest1.exe: $(TOP)\test\speedtest1.c $(SQLITE3C) $(SQLITE3H)
$(LTLINK) $(NO_WARN) -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
$(LTLINK) $(NO_WARN) $(ST_COMPILE_OPTS) -DSQLITE_OMIT_LOAD_EXTENSION \
$(TOP)\test\speedtest1.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
kvtest.exe: $(TOP)\test\kvtest.c $(SQLITE3C) $(SQLITE3H)
$(LTLINK) $(NO_WARN) $(KV_COMPILE_OPTS) \
$(TOP)\test\kvtest.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
dbselftest.exe: $(TOP)\test\dbselftest.c $(SQLITE3C) $(SQLITE3H)
$(LTLINK) $(NO_WARN) $(DBSELFTEST_COMPILE_OPTS) $(TOP)\test\dbselftest.c $(SQLITE3C)
rbu.exe: $(TOP)\ext\rbu\rbu.c $(TOP)\ext\rbu\sqlite3rbu.c $(SQLITE3C) $(SQLITE3H)
$(LTLINK) $(NO_WARN) -DSQLITE_ENABLE_RBU -Fe$@ \
$(LTLINK) $(NO_WARN) -DSQLITE_ENABLE_RBU \
$(TOP)\ext\rbu\rbu.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
moreclean: clean
@@ -2056,22 +2258,20 @@ clean:
del /Q notasharedlib.* 2>NUL
-rmdir /Q/S .deps 2>NUL
-rmdir /Q/S .libs 2>NUL
-rmdir /Q/S quota2a 2>NUL
-rmdir /Q/S quota2b 2>NUL
-rmdir /Q/S quota2c 2>NUL
-rmdir /Q/S tsrc 2>NUL
del /Q .target_source 2>NUL
del /Q tclsqlite3.exe 2>NUL
del /Q tclsqlite3.exe $(SQLITETCLH) $(SQLITETCLDECLSH) 2>NUL
del /Q testloadext.dll 2>NUL
del /Q testfixture.exe test.db 2>NUL
del /Q LogEst.exe fts3view.exe rollback-test.exe showdb.exe 2>NUL
del /Q LogEst.exe fts3view.exe rollback-test.exe showdb.exe dbdump.exe 2>NUL
del /Q changeset.exe 2>NUL
del /Q showjournal.exe showstat4.exe showwal.exe speedtest1.exe 2>NUL
del /Q mptester.exe wordcount.exe rbu.exe srcck1.exe 2>NUL
del /Q sqlite3.c sqlite3-*.c 2>NUL
del /Q sqlite3rc.h 2>NUL
del /Q shell.c sqlite3ext.h 2>NUL
del /Q shell.c sqlite3ext.h sqlite3session.h 2>NUL
del /Q sqlite3_analyzer.exe sqlite3_analyzer.c 2>NUL
del /Q sqlite-*-output.vsix 2>NUL
del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe 2>NUL
del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe dbhash.exe 2>NUL
del /Q fts5.* fts5parse.* 2>NUL
# <</mark>>

70
README.md
View File

@@ -8,6 +8,48 @@ If you are reading this on a Git mirror someplace, you are doing it wrong.
The [official repository](https://www.sqlite.org/src/) is better. Go there
now.
## Obtaining The Code
SQLite sources are managed using the
[Fossil](https://www.fossil-scm.org/), a distributed version control system
that was specifically designed to support SQLite development.
If you do not want to use Fossil, you can download tarballs or ZIP
archives as follows:
* Lastest trunk check-in:
<https://www.sqlite.org/src/tarball/sqlite.tar.gz> or
<https://www.sqlite.org/src/zip/sqlite.zip>.
* Latest release:
<https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=release> or
<https://www.sqlite.org/src/zip/sqlite.zip?r=release>.
* For other check-ins, substitute an appropriate branch name or
tag or hash prefix for "release" in the URLs of the previous
bullet. Or browse the [timeline](https://www.sqlite.org/src/timeline)
to locate the check-in desired, click on its information page link,
then click on the "Tarball" or "ZIP Archive" links on the information
page.
If you do want to use Fossil to check out the source tree,
first install Fossil version 2.0 or later.
(Source tarballs and precompiled binaries available
[here](https://www.fossil-scm.org/fossil/uv/download.html).)
Then run commands like this:
mkdir ~/sqlite
cd ~/sqlite
fossil clone https://www.sqlite.org/src sqlite.fossil
fossil open sqlite.fossil
After setting up a repository using the steps above, you can always
update to the lastest version using:
fossil update trunk ;# latest trunk check-in
fossil update release ;# latest official release
Or type "fossil ui" to get a web-based user interface.
## Compiling
First create a directory in which to place
@@ -18,13 +60,13 @@ script found at the root of the source tree. Then run "make".
For example:
tar xzf sqlite.tar.gz ;# Unpack the source tree into "sqlite"
mkdir bld ;# Build will occur in a sibling directory
cd bld ;# Change to the build directory
../sqlite/configure ;# Run the configure script
make ;# Run the makefile.
make sqlite3.c ;# Build the "amalgamation" source file
make test ;# Run some tests (requires Tcl)
tar xzf sqlite.tar.gz ;# Unpack the source tree into "sqlite"
mkdir bld ;# Build will occur in a sibling directory
cd bld ;# Change to the build directory
../sqlite/configure ;# Run the configure script
make ;# Run the makefile.
make sqlite3.c ;# Build the "amalgamation" source file
make test ;# Run some tests (requires Tcl)
See the makefile for additional targets.
@@ -43,13 +85,13 @@ with the provided "Makefile.msc" to build one of the supported targets.
For example:
mkdir bld
cd bld
nmake /f Makefile.msc TOP=..\sqlite
nmake /f Makefile.msc sqlite3.c TOP=..\sqlite
nmake /f Makefile.msc sqlite3.dll TOP=..\sqlite
nmake /f Makefile.msc sqlite3.exe TOP=..\sqlite
nmake /f Makefile.msc test TOP=..\sqlite
mkdir bld
cd bld
nmake /f Makefile.msc TOP=..\sqlite
nmake /f Makefile.msc sqlite3.c TOP=..\sqlite
nmake /f Makefile.msc sqlite3.dll TOP=..\sqlite
nmake /f Makefile.msc sqlite3.exe TOP=..\sqlite
nmake /f Makefile.msc test TOP=..\sqlite
There are several build options that can be set via the NMAKE command
line. For example, to build for WinRT, simply add "FOR_WINRT=1" argument

2
VERSION
View File

@@ -1 +1 @@
3.12.0
3.19.0

2
autoconf/Makefile.am
View File

@@ -1,5 +1,5 @@
AM_CFLAGS = @THREADSAFE_FLAGS@ @DYNAMIC_EXTENSION_FLAGS@ @FTS5_FLAGS@ @JSON1_FLAGS@ -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE
AM_CFLAGS = @THREADSAFE_FLAGS@ @DYNAMIC_EXTENSION_FLAGS@ @FTS5_FLAGS@ @JSON1_FLAGS@ @SESSION_FLAGS@ -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE
lib_LTLIBRARIES = libsqlite3.la
libsqlite3_la_SOURCES = sqlite3.c

62
autoconf/Makefile.msc
View File

@@ -24,6 +24,27 @@ TOP = .
USE_FULLWARN = 0
!ENDIF
# Set this non-0 to enable treating warnings as errors (-WX, etc) when
# compiling.
#
!IFNDEF USE_FATAL_WARN
USE_FATAL_WARN = 0
!ENDIF
# Set this non-0 to enable full runtime error checks (-RTC1, etc). This
# has no effect if (any) optimizations are enabled.
#
!IFNDEF USE_RUNTIME_CHECKS
USE_RUNTIME_CHECKS = 0
!ENDIF
# Set this non-0 to create a SQLite amalgamation file that excludes the
# various built-in extensions.
#
!IFNDEF MINIMAL_AMALGAMATION
MINIMAL_AMALGAMATION = 0
!ENDIF
# Set this non-0 to use "stdcall" calling convention for the core library
# and shell executable.
#
@@ -183,6 +204,12 @@ DEBUG = 0
OPTIMIZATIONS = 2
!ENDIF
# Set this to non-0 to enable support for the session extension.
#
!IFNDEF SESSION
SESSION = 0
!ENDIF
# Set the source code file to be used by executables and libraries when
# they need the amalgamation.
#
@@ -242,15 +269,26 @@ SQLITE3EXEPDB = /pdb:sqlite3sh.pdb
!ENDIF
!ENDIF
# These are the "standard" SQLite compilation options used when compiling for
# the Windows platform.
#
!IFNDEF OPT_FEATURE_FLAGS
!IF $(MINIMAL_AMALGAMATION)==0
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
!ENDIF
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1
!ENDIF
# Should the session extension be enabled? If so, add compilation options
# to enable it.
#
!IF $(SESSION)!=0
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_SESSION=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_PREUPDATE_HOOK=1
!ENDIF
# These are the "extended" SQLite compilation options used when compiling for
# the Windows 10 platform.
#
@@ -423,6 +461,12 @@ TCC = $(CC) -nologo -W4 -DINCLUDE_MSVC_H=1 $(CCOPTS) $(TCCOPTS)
TCC = $(CC) -nologo -W3 $(CCOPTS) $(TCCOPTS)
!ENDIF
# Check if warnings should be treated as errors when compiling.
#
!IF $(USE_FATAL_WARN)!=0
TCC = $(TCC) -WX
!ENDIF
TCC = $(TCC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -fp:precise
RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) $(RCOPTS) $(RCCOPTS)
@@ -435,12 +479,12 @@ RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) $(RCOPTS) $(RCCOPTS)
#
!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0
!IF "$(PLATFORM)"=="x86"
CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
!ELSE
!IFNDEF PLATFORM
CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
!ELSE
CORE_CCONV_OPTS =
SHELL_CCONV_OPTS =
@@ -601,6 +645,10 @@ RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1
!IF $(DEBUG)>2
TCC = $(TCC) -DSQLITE_DEBUG=1
RCC = $(RCC) -DSQLITE_DEBUG=1
!IF $(DYNAMIC_SHELL)==0
TCC = $(TCC) -DSQLITE_ENABLE_WHERETRACE -DSQLITE_ENABLE_SELECTTRACE
RCC = $(RCC) -DSQLITE_ENABLE_WHERETRACE -DSQLITE_ENABLE_SELECTTRACE
!ENDIF
!ENDIF
!IF $(DEBUG)>4 || $(OSTRACE)!=0
@@ -726,6 +774,10 @@ RCC = $(RCC) -D_DEBUG
!IF $(DEBUG)>1 || $(OPTIMIZATIONS)==0
TCC = $(TCC) -Od
BCC = $(BCC) -Od
!IF $(USE_RUNTIME_CHECKS)!=0
TCC = $(TCC) -RTC1
BCC = $(BCC) -RTC1
!ENDIF
!ELSEIF $(OPTIMIZATIONS)>=3
TCC = $(TCC) -Ox
BCC = $(BCC) -Ox
@@ -902,7 +954,7 @@ Replace.exe:
sqlite3.def: Replace.exe $(LIBOBJ)
echo EXPORTS > sqlite3.def
dumpbin /all $(LIBOBJ) \
| .\Replace.exe "^\s+/EXPORT:_?(sqlite3_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \
| .\Replace.exe "^\s+/EXPORT:_?(sqlite3(?:session|changeset)?_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \
| sort >> sqlite3.def
$(SQLITE3EXE): $(TOP)\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)

91
autoconf/configure.ac
View File

@@ -30,39 +30,52 @@ AC_FUNC_STRERROR_R
AC_CONFIG_FILES([Makefile sqlite3.pc])
AC_SUBST(BUILD_CFLAGS)
#-----------------------------------------------------------------------
#-------------------------------------------------------------------------
# Two options to enable readline compatible libraries:
#
# --enable-editline
# --enable-readline
#
AC_ARG_ENABLE(editline, [AS_HELP_STRING(
[--enable-editline],
[use BSD libedit])],
[], [enable_editline=yes])
AC_ARG_ENABLE(readline, [AS_HELP_STRING(
[--enable-readline],
[use readline])],
[], [enable_readline=no])
if test x"$enable_editline" != xno ; then
sLIBS=$LIBS
LIBS=""
AC_SEARCH_LIBS([readline],[edit],[enable_readline=no],[enable_editline=no])
READLINE_LIBS=$LIBS
if test x"$LIBS" != "x"; then
AC_DEFINE([HAVE_EDITLINE],1,Define to use BSD editline)
else
unset ac_cv_search_readline
fi
LIBS=$sLIBS
fi
if test x"$enable_readline" != xno ; then
sLIBS=$LIBS
LIBS=""
AC_SEARCH_LIBS(tgetent, curses ncurses ncursesw, [], [])
AC_SEARCH_LIBS(readline, readline, [], [enable_readline=no])
AC_CHECK_FUNCS(readline, [], [])
READLINE_LIBS=$LIBS
LIBS=$sLIBS
fi
# Both are enabled by default. If, after command line processing both are
# still enabled, the script searches for editline first and automatically
# disables readline if it is found. So, to use readline explicitly, the
# user must pass "--disable-editline". To disable command line editing
# support altogether, "--disable-editline --disable-readline".
#
# When searching for either library, check for headers before libraries
# as some distros supply packages that contain libraries but not header
# files, which come as a separate development package.
#
AC_ARG_ENABLE(editline, [AS_HELP_STRING([--enable-editline],[use BSD libedit])])
AC_ARG_ENABLE(readline, [AS_HELP_STRING([--enable-readline],[use readline])])
AS_IF([ test x"$enable_editline" != xno ],[
AC_CHECK_HEADERS([editline/readline.h],[
sLIBS=$LIBS
LIBS=""
AC_SEARCH_LIBS([readline],[edit],[
AC_DEFINE([HAVE_EDITLINE],1,Define to use BSD editline)
READLINE_LIBS="$LIBS -ltinfo"
enable_readline=no
],[],[-ltinfo])
AS_UNSET(ac_cv_search_readline)
LIBS=$sLIBS
])
])
AS_IF([ test x"$enable_readline" != xno ],[
AC_CHECK_HEADERS([readline/readline.h],[
sLIBS=$LIBS
LIBS=""
AC_SEARCH_LIBS(tgetent, termcap curses ncurses ncursesw, [], [])
AC_SEARCH_LIBS(readline,[readline edit], [
AC_DEFINE([HAVE_READLINE],1,Define to use readline or wrapper)
READLINE_LIBS=$LIBS
])
LIBS=$sLIBS
])
])
AC_SUBST(READLINE_LIBS)
#-----------------------------------------------------------------------
@@ -103,7 +116,7 @@ AC_SUBST(DYNAMIC_EXTENSION_FLAGS)
AC_ARG_ENABLE(fts5, [AS_HELP_STRING(
[--enable-fts5], [include fts5 support [default=no]])],
[], [enable_fts5=no])
if test x"$enable_fts5" == "xyes"; then
if test x"$enable_fts5" = "xyes"; then
AC_SEARCH_LIBS(log, m)
FTS5_FLAGS=-DSQLITE_ENABLE_FTS5
fi
@@ -116,12 +129,24 @@ AC_SUBST(FTS5_FLAGS)
AC_ARG_ENABLE(json1, [AS_HELP_STRING(
[--enable-json1], [include json1 support [default=no]])],
[], [enable_json1=no])
if test x"$enable_json1" == "xyes"; then
if test x"$enable_json1" = "xyes"; then
JSON1_FLAGS=-DSQLITE_ENABLE_JSON1
fi
AC_SUBST(JSON1_FLAGS)
#-----------------------------------------------------------------------
#-----------------------------------------------------------------------
# --enable-session
#
AC_ARG_ENABLE(session, [AS_HELP_STRING(
[--enable-session], [enable the session extension [default=no]])],
[], [enable_session=no])
if test x"$enable_session" = "xyes"; then
SESSION_FLAGS="-DSQLITE_ENABLE_SESSION -DSQLITE_ENABLE_PREUPDATE_HOOK"
fi
AC_SUBST(SESSION_FLAGS)
#-----------------------------------------------------------------------
#-----------------------------------------------------------------------
# --enable-static-shell
#
@@ -129,7 +154,7 @@ AC_ARG_ENABLE(static-shell, [AS_HELP_STRING(
[--enable-static-shell],
[statically link libsqlite3 into shell tool [default=yes]])],
[], [enable_static_shell=yes])
if test x"$enable_static_shell" == "xyes"; then
if test x"$enable_static_shell" = "xyes"; then
EXTRA_SHELL_OBJ=sqlite3-sqlite3.$OBJEXT
else
EXTRA_SHELL_OBJ=libsqlite3.la

1
autoconf/tea/configure.ac
View File

@@ -78,7 +78,6 @@ TEA_ADD_LIBS([])
TEA_ADD_CFLAGS([-DSQLITE_ENABLE_FTS3=1])
TEA_ADD_CFLAGS([-DSQLITE_3_SUFFIX_ONLY=1])
TEA_ADD_CFLAGS([-DSQLITE_ENABLE_RTREE=1])
TEA_ADD_CFLAGS([-DSQLITE_OMIT_DEPRECATED=1])
TEA_ADD_STUB_SOURCES([])
TEA_ADD_TCL_SOURCES([])

120
configure vendored
View File

@@ -1,6 +1,6 @@
#! /bin/sh
# Guess values for system-dependent variables and create Makefiles.
# Generated by GNU Autoconf 2.69 for sqlite 3.12.0.
# Generated by GNU Autoconf 2.69 for sqlite 3.19.0.
#
#
# Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -726,8 +726,8 @@ MAKEFLAGS=
# Identity of this package.
PACKAGE_NAME='sqlite'
PACKAGE_TARNAME='sqlite'
PACKAGE_VERSION='3.12.0'
PACKAGE_STRING='sqlite 3.12.0'
PACKAGE_VERSION='3.19.0'
PACKAGE_STRING='sqlite 3.19.0'
PACKAGE_BUGREPORT=''
PACKAGE_URL=''
@@ -903,11 +903,14 @@ with_readline_inc
enable_debug
enable_amalgamation
enable_load_extension
enable_memsys5
enable_memsys3
enable_fts3
enable_fts4
enable_fts5
enable_json1
enable_rtree
enable_session
enable_gcov
'
ac_precious_vars='build_alias
@@ -1460,7 +1463,7 @@ if test "$ac_init_help" = "long"; then
# Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF
\`configure' configures sqlite 3.12.0 to adapt to many kinds of systems.
\`configure' configures sqlite 3.19.0 to adapt to many kinds of systems.
Usage: $0 [OPTION]... [VAR=VALUE]...
@@ -1525,7 +1528,7 @@ fi
if test -n "$ac_init_help"; then
case $ac_init_help in
short | recursive ) echo "Configuration of sqlite 3.12.0:";;
short | recursive ) echo "Configuration of sqlite 3.19.0:";;
esac
cat <<\_ACEOF
@@ -1551,11 +1554,14 @@ Optional Features:
separately
--disable-load-extension
Disable loading of external extensions
--enable-memsys5 Enable MEMSYS5
--enable-memsys3 Enable MEMSYS3
--enable-fts3 Enable the FTS3 extension
--enable-fts4 Enable the FTS4 extension
--enable-fts5 Enable the FTS5 extension
--enable-json1 Enable the JSON1 extension
--enable-rtree Enable the RTREE extension
--enable-session Enable the SESSION extension
--enable-gcov Enable coverage testing using gcov
Optional Packages:
@@ -1646,7 +1652,7 @@ fi
test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then
cat <<\_ACEOF
sqlite configure 3.12.0
sqlite configure 3.19.0
generated by GNU Autoconf 2.69
Copyright (C) 2012 Free Software Foundation, Inc.
@@ -2065,7 +2071,7 @@ cat >config.log <<_ACEOF
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.
It was created by sqlite $as_me 3.12.0, which was
It was created by sqlite $as_me 3.19.0, which was
generated by GNU Autoconf 2.69. Invocation command line was
$ $0 $@
@@ -3923,13 +3929,13 @@ if ${lt_cv_nm_interface+:} false; then :
else
lt_cv_nm_interface="BSD nm"
echo "int some_variable = 0;" > conftest.$ac_ext
(eval echo "\"\$as_me:3926: $ac_compile\"" >&5)
(eval echo "\"\$as_me:3932: $ac_compile\"" >&5)
(eval "$ac_compile" 2>conftest.err)
cat conftest.err >&5
(eval echo "\"\$as_me:3929: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
(eval echo "\"\$as_me:3935: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
(eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out)
cat conftest.err >&5
(eval echo "\"\$as_me:3932: output\"" >&5)
(eval echo "\"\$as_me:3938: output\"" >&5)
cat conftest.out >&5
if $GREP 'External.*some_variable' conftest.out > /dev/null; then
lt_cv_nm_interface="MS dumpbin"
@@ -5135,7 +5141,7 @@ ia64-*-hpux*)
;;
*-*-irix6*)
# Find out which ABI we are using.
echo '#line 5138 "configure"' > conftest.$ac_ext
echo '#line 5144 "configure"' > conftest.$ac_ext
if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5
(eval $ac_compile) 2>&5
ac_status=$?
@@ -6660,11 +6666,11 @@ else
-e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
-e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
-e 's:$: $lt_compiler_flag:'`
(eval echo "\"\$as_me:6663: $lt_compile\"" >&5)
(eval echo "\"\$as_me:6669: $lt_compile\"" >&5)
(eval "$lt_compile" 2>conftest.err)
ac_status=$?
cat conftest.err >&5
echo "$as_me:6667: \$? = $ac_status" >&5
echo "$as_me:6673: \$? = $ac_status" >&5
if (exit $ac_status) && test -s "$ac_outfile"; then
# The compiler can only warn and ignore the option if not recognized
# So say no if there are warnings other than the usual output.
@@ -6999,11 +7005,11 @@ else
-e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
-e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
-e 's:$: $lt_compiler_flag:'`
(eval echo "\"\$as_me:7002: $lt_compile\"" >&5)
(eval echo "\"\$as_me:7008: $lt_compile\"" >&5)
(eval "$lt_compile" 2>conftest.err)
ac_status=$?
cat conftest.err >&5
echo "$as_me:7006: \$? = $ac_status" >&5
echo "$as_me:7012: \$? = $ac_status" >&5
if (exit $ac_status) && test -s "$ac_outfile"; then
# The compiler can only warn and ignore the option if not recognized
# So say no if there are warnings other than the usual output.
@@ -7104,11 +7110,11 @@ else
-e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
-e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
-e 's:$: $lt_compiler_flag:'`
(eval echo "\"\$as_me:7107: $lt_compile\"" >&5)
(eval echo "\"\$as_me:7113: $lt_compile\"" >&5)
(eval "$lt_compile" 2>out/conftest.err)
ac_status=$?
cat out/conftest.err >&5
echo "$as_me:7111: \$? = $ac_status" >&5
echo "$as_me:7117: \$? = $ac_status" >&5
if (exit $ac_status) && test -s out/conftest2.$ac_objext
then
# The compiler can only warn and ignore the option if not recognized
@@ -7159,11 +7165,11 @@ else
-e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
-e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
-e 's:$: $lt_compiler_flag:'`
(eval echo "\"\$as_me:7162: $lt_compile\"" >&5)
(eval echo "\"\$as_me:7168: $lt_compile\"" >&5)
(eval "$lt_compile" 2>out/conftest.err)
ac_status=$?
cat out/conftest.err >&5
echo "$as_me:7166: \$? = $ac_status" >&5
echo "$as_me:7172: \$? = $ac_status" >&5
if (exit $ac_status) && test -s out/conftest2.$ac_objext
then
# The compiler can only warn and ignore the option if not recognized
@@ -9539,7 +9545,7 @@ else
lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
lt_status=$lt_dlunknown
cat > conftest.$ac_ext <<_LT_EOF
#line 9542 "configure"
#line 9548 "configure"
#include "confdefs.h"
#if HAVE_DLFCN_H
@@ -9635,7 +9641,7 @@ else
lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
lt_status=$lt_dlunknown
cat > conftest.$ac_ext <<_LT_EOF
#line 9638 "configure"
#line 9644 "configure"
#include "confdefs.h"
#if HAVE_DLFCN_H
@@ -10755,6 +10761,20 @@ else
fi
fi
# Recent versions of Xcode on Macs hid the tclConfig.sh file
# in a strange place.
if test x"${ac_cv_c_tclconfig}" = x ; then
if test x"$cross_compiling" = xno; then
for i in /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX*.sdk/usr/lib
do
if test -f "$i/tclConfig.sh" ; then
ac_cv_c_tclconfig="$i"
break
fi
done
fi
fi
# then check for a private Tcl installation
if test x"${ac_cv_c_tclconfig}" = x ; then
for i in \
@@ -11232,7 +11252,7 @@ else
fi
if test "${use_debug}" = "yes" ; then
TARGET_DEBUG="-DSQLITE_DEBUG=1"
TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE"
else
TARGET_DEBUG="-DNDEBUG"
fi
@@ -11323,6 +11343,44 @@ else
OPT_FEATURE_FLAGS="-DSQLITE_OMIT_LOAD_EXTENSION=1"
fi
##########
# Do we want to support memsys3 and/or memsys5
#
# Check whether --enable-memsys5 was given.
if test "${enable_memsys5+set}" = set; then :
enableval=$enable_memsys5; enable_memsys5=yes
else
enable_memsys5=no
fi
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support MEMSYS5" >&5
$as_echo_n "checking whether to support MEMSYS5... " >&6; }
if test "${enable_memsys5}" = "yes"; then
OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_MEMSYS5"
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
$as_echo "yes" >&6; }
else
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
fi
# Check whether --enable-memsys3 was given.
if test "${enable_memsys3+set}" = set; then :
enableval=$enable_memsys3; enable_memsys3=yes
else
enable_memsys3=no
fi
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support MEMSYS3" >&5
$as_echo_n "checking whether to support MEMSYS3... " >&6; }
if test "${enable_memsys3}" = "yes" -a "${enable_memsys5}" = "no"; then
OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_MEMSYS3"
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
$as_echo "yes" >&6; }
else
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
fi
#########
# See whether we should enable Full Text Search extensions
# Check whether --enable-fts3 was given.
@@ -11494,6 +11552,20 @@ if test "${enable_rtree}" = "yes" ; then
OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_RTREE"
fi
#########
# See whether we should enable the SESSION extension
# Check whether --enable-session was given.
if test "${enable_session+set}" = set; then :
enableval=$enable_session; enable_session=yes
else
enable_session=no
fi
if test "${enable_session}" = "yes" ; then
OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_SESSION"
OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_PREUPDATE_HOOK"
fi
#########
# attempt to duplicate any OMITS and ENABLES into the $(OPT_FEATURE_FLAGS) parameter
for option in $CFLAGS $CPPFLAGS
@@ -12079,7 +12151,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# report actual input values of CONFIG_FILES etc. instead of their
# values after options handling.
ac_log="
This file was extended by sqlite $as_me 3.12.0, which was
This file was extended by sqlite $as_me 3.19.0, which was
generated by GNU Autoconf 2.69. Invocation command line was
CONFIG_FILES = $CONFIG_FILES
@@ -12145,7 +12217,7 @@ _ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
ac_cs_version="\\
sqlite config.status 3.12.0
sqlite config.status 3.19.0
configured by $0, generated by GNU Autoconf 2.69,
with options \\"\$ac_cs_config\\"

50
configure.ac
View File

@@ -334,6 +334,20 @@ if test "${use_tcl}" = "yes" ; then
fi
fi
# Recent versions of Xcode on Macs hid the tclConfig.sh file
# in a strange place.
if test x"${ac_cv_c_tclconfig}" = x ; then
if test x"$cross_compiling" = xno; then
for i in /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX*.sdk/usr/lib
do
if test -f "$i/tclConfig.sh" ; then
ac_cv_c_tclconfig="$i"
break
fi
done
fi
fi
# then check for a private Tcl installation
if test x"${ac_cv_c_tclconfig}" = x ; then
for i in \
@@ -546,7 +560,7 @@ AC_SEARCH_LIBS(fdatasync, [rt])
AC_ARG_ENABLE(debug, AC_HELP_STRING([--enable-debug],[enable debugging & verbose explain]),
[use_debug=$enableval],[use_debug=no])
if test "${use_debug}" = "yes" ; then
TARGET_DEBUG="-DSQLITE_DEBUG=1"
TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE"
else
TARGET_DEBUG="-DNDEBUG"
fi
@@ -574,6 +588,30 @@ else
OPT_FEATURE_FLAGS="-DSQLITE_OMIT_LOAD_EXTENSION=1"
fi
##########
# Do we want to support memsys3 and/or memsys5
#
AC_ARG_ENABLE(memsys5,
AC_HELP_STRING([--enable-memsys5],[Enable MEMSYS5]),
[enable_memsys5=yes],[enable_memsys5=no])
AC_MSG_CHECKING([whether to support MEMSYS5])
if test "${enable_memsys5}" = "yes"; then
OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_MEMSYS5"
AC_MSG_RESULT([yes])
else
AC_MSG_RESULT([no])
fi
AC_ARG_ENABLE(memsys3,
AC_HELP_STRING([--enable-memsys3],[Enable MEMSYS3]),
[enable_memsys3=yes],[enable_memsys3=no])
AC_MSG_CHECKING([whether to support MEMSYS3])
if test "${enable_memsys3}" = "yes" -a "${enable_memsys5}" = "no"; then
OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_MEMSYS3"
AC_MSG_RESULT([yes])
else
AC_MSG_RESULT([no])
fi
#########
# See whether we should enable Full Text Search extensions
AC_ARG_ENABLE(fts3, AC_HELP_STRING([--enable-fts3],
@@ -615,6 +653,16 @@ if test "${enable_rtree}" = "yes" ; then
OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_RTREE"
fi
#########
# See whether we should enable the SESSION extension
AC_ARG_ENABLE(session, AC_HELP_STRING([--enable-session],
[Enable the SESSION extension]),
[enable_session=yes],[enable_session=no])
if test "${enable_session}" = "yes" ; then
OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_SESSION"
OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_PREUPDATE_HOOK"
fi
#########
# attempt to duplicate any OMITS and ENABLES into the $(OPT_FEATURE_FLAGS) parameter
for option in $CFLAGS $CPPFLAGS

279
doc/lemon.html
View File

@@ -5,14 +5,17 @@
<body bgcolor=white>
<h1 align=center>The Lemon Parser Generator</h1>
<p>Lemon is an LALR(1) parser generator for C or C++.
It does the same job as ``bison'' and ``yacc''.
But lemon is not another bison or yacc clone. It
<p>Lemon is an LALR(1) parser generator for C.
It does the same job as "bison" and "yacc".
But lemon is not a bison or yacc clone. Lemon
uses a different grammar syntax which is designed to
reduce the number of coding errors. Lemon also uses a more
sophisticated parsing engine that is faster than yacc and
bison and which is both reentrant and thread-safe.
Furthermore, Lemon implements features that can be used
reduce the number of coding errors. Lemon also uses a
parsing engine that is faster than yacc and
bison and which is both reentrant and threadsafe.
(Update: Since the previous sentence was written, bison
has also been updated so that it too can generate a
reentrant and threadsafe parser.)
Lemon also implements features that can be used
to eliminate resource leaks, making is suitable for use
in long-running programs such as graphical user interfaces
or embedded controllers.</p>
@@ -44,18 +47,18 @@ one and three files of outputs.
automaton.
</ul>
By default, all three of these output files are generated.
The header file is suppressed if the ``-m'' command-line option is
used and the report file is omitted when ``-q'' is selected.</p>
The header file is suppressed if the "-m" command-line option is
used and the report file is omitted when "-q" is selected.</p>
<p>The grammar specification file uses a ``.y'' suffix, by convention.
<p>The grammar specification file uses a ".y" suffix, by convention.
In the examples used in this document, we'll assume the name of the
grammar file is ``gram.y''. A typical use of Lemon would be the
grammar file is "gram.y". A typical use of Lemon would be the
following command:
<pre>
lemon gram.y
</pre>
This command will generate three output files named ``gram.c'',
``gram.h'' and ``gram.out''.
This command will generate three output files named "gram.c",
"gram.h" and "gram.out".
The first is C code to implement the parser. The second
is the header file that defines numerical values for all
terminal symbols, and the last is the report that explains
@@ -71,39 +74,35 @@ with a brief explanation of what each does by typing
</pre>
As of this writing, the following command-line options are supported:
<ul>
<li><tt>-b</tt>
<li><tt>-c</tt>
<li><tt>-g</tt>
<li><tt>-m</tt>
<li><tt>-q</tt>
<li><tt>-s</tt>
<li><tt>-x</tt>
<li><b>-b</b>
Show only the basis for each parser state in the report file.
<li><b>-c</b>
Do not compress the generated action tables.
<li><b>-D<i>name</i></b>
Define C preprocessor macro <i>name</i>. This macro is useable by
"%ifdef" lines in the grammar file.
<li><b>-g</b>
Do not generate a parser. Instead write the input grammar to standard
output with all comments, actions, and other extraneous text removed.
<li><b>-l</b>
Omit "#line" directives in the generated parser C code.
<li><b>-m</b>
Cause the output C source code to be compatible with the "makeheaders"
program.
<li><b>-p</b>
Display all conflicts that are resolved by
<a href='#precrules'>precedence rules</a>.
<li><b>-q</b>
Suppress generation of the report file.
<li><b>-r</b>
Do not sort or renumber the parser states as part of optimization.
<li><b>-s</b>
Show parser statistics before existing.
<li><b>-T<i>file</i></b>
Use <i>file</i> as the template for the generated C-code parser implementation.
<li><b>-x</b>
Print the Lemon version number.
</ul>
The ``-b'' option reduces the amount of text in the report file by
printing only the basis of each parser state, rather than the full
configuration.
The ``-c'' option suppresses action table compression. Using -c
will make the parser a little larger and slower but it will detect
syntax errors sooner.
The ``-g'' option causes no output files to be generated at all.
Instead, the input grammar file is printed on standard output but
with all comments, actions and other extraneous text deleted. This
is a useful way to get a quick summary of a grammar.
The ``-m'' option causes the output C source file to be compatible
with the ``makeheaders'' program.
Makeheaders is a program that automatically generates header files
from C source code. When the ``-m'' option is used, the header
file is not output since the makeheaders program will take care
of generated all header files automatically.
The ``-q'' option suppresses the report file.
Using ``-s'' causes a brief summary of parser statistics to be
printed. Like this:
<pre>
Parser statistics: 74 terminals, 70 nonterminals, 179 rules
340 states, 2026 parser table entries, 0 conflicts
</pre>
Finally, the ``-x'' option causes Lemon to print its version number
and then stops without attempting to read the grammar or generate a parser.</p>
<h3>The Parser Interface</h3>
@@ -121,12 +120,12 @@ A new parser is created as follows:
</pre>
The ParseAlloc() routine allocates and initializes a new parser and
returns a pointer to it.
The actual data structure used to represent a parser is opaque --
The actual data structure used to represent a parser is opaque &mdash;
its internal structure is not visible or usable by the calling routine.
For this reason, the ParseAlloc() routine returns a pointer to void
rather than a pointer to some particular structure.
The sole argument to the ParseAlloc() routine is a pointer to the
subroutine used to allocate memory. Typically this means ``malloc()''.</p>
subroutine used to allocate memory. Typically this means malloc().</p>
<p>After a program is finished using a parser, it can reclaim all
memory allocated by that parser by calling
@@ -151,13 +150,13 @@ type of the next token in the data stream.
There is one token type for each terminal symbol in the grammar.
The gram.h file generated by Lemon contains #define statements that
map symbolic terminal symbol names into appropriate integer values.
(A value of 0 for the second argument is a special flag to the
parser to indicate that the end of input has been reached.)
A value of 0 for the second argument is a special flag to the
parser to indicate that the end of input has been reached.
The third argument is the value of the given token. By default,
the type of the third argument is integer, but the grammar will
usually redefine this type to be some kind of structure.
Typically the second argument will be a broad category of tokens
such as ``identifier'' or ``number'' and the third argument will
such as "identifier" or "number" and the third argument will
be the name of the identifier or the value of the number.</p>
<p>The Parse() function may have either three or four arguments,
@@ -193,7 +192,7 @@ following:
</pre>
This example shows a user-written routine that parses a file of
text and returns a pointer to the parse tree.
(We've omitted all error-handling from this example to keep it
(All error-handling code is omitted from this example to keep it
simple.)
We assume the existence of some kind of tokenizer which is created
using TokenizerCreate() on line 8 and deleted by TokenizerFree()
@@ -287,7 +286,7 @@ tokens) and it honors the same commenting conventions as C and C++.</p>
<h3>Terminals and Nonterminals</h3>
<p>A terminal symbol (token) is any string of alphanumeric
and underscore characters
and/or underscore characters
that begins with an upper case letter.
A terminal can contain lowercase letters after the first character,
but the usual convention is to make terminals all upper case.
@@ -314,7 +313,7 @@ must have alphanumeric names.</p>
<p>The main component of a Lemon grammar file is a sequence of grammar
rules.
Each grammar rule consists of a nonterminal symbol followed by
the special symbol ``::='' and then a list of terminals and/or nonterminals.
the special symbol "::=" and then a list of terminals and/or nonterminals.
The rule is terminated by a period.
The list of terminals and nonterminals on the right-hand side of the
rule can be empty.
@@ -330,9 +329,9 @@ A typical sequence of grammar rules might look something like this:
</pre>
</p>
<p>There is one non-terminal in this example, ``expr'', and five
terminal symbols or tokens: ``PLUS'', ``TIMES'', ``LPAREN'',
``RPAREN'' and ``VALUE''.</p>
<p>There is one non-terminal in this example, "expr", and five
terminal symbols or tokens: "PLUS", "TIMES", "LPAREN",
"RPAREN" and "VALUE".</p>
<p>Like yacc and bison, Lemon allows the grammar to specify a block
of C code that will be executed whenever a grammar rule is reduced
@@ -348,15 +347,15 @@ For example:
<p>In order to be useful, grammar actions must normally be linked to
their associated grammar rules.
In yacc and bison, this is accomplished by embedding a ``$$'' in the
In yacc and bison, this is accomplished by embedding a "$$" in the
action to stand for the value of the left-hand side of the rule and
symbols ``$1'', ``$2'', and so forth to stand for the value of
symbols "$1", "$2", and so forth to stand for the value of
the terminal or nonterminal at position 1, 2 and so forth on the
right-hand side of the rule.
This idea is very powerful, but it is also very error-prone. The
single most common source of errors in a yacc or bison grammar is
to miscount the number of symbols on the right-hand side of a grammar
rule and say ``$7'' when you really mean ``$8''.</p>
rule and say "$7" when you really mean "$8".</p>
<p>Lemon avoids the need to count grammar symbols by assigning symbolic
names to each symbol in a grammar rule and then using those symbolic
@@ -386,7 +385,7 @@ For example, the rule
<pre>
expr(A) ::= expr(B) PLUS expr(C). { A = B; }
</pre>
will generate an error because the linking symbol ``C'' is used
will generate an error because the linking symbol "C" is used
in the grammar rule but not in the reduce action.</p>
<p>The Lemon notation for linking grammar rules to reduce actions
@@ -394,6 +393,7 @@ also facilitates the use of destructors for reclaiming memory
allocated by the values of terminals and nonterminals on the
right-hand side of a rule.</p>
<a name='precrules'></a>
<h3>Precedence Rules</h3>
<p>Lemon resolves parsing ambiguities in exactly the same way as
@@ -405,7 +405,10 @@ whichever rule comes first in the grammar file.</p>
yacc and bison, Lemon allows a measure of control
over the resolution of paring conflicts using precedence rules.
A precedence value can be assigned to any terminal symbol
using the %left, %right or %nonassoc directives. Terminal symbols
using the
<a href='#pleft'>%left</a>,
<a href='#pright'>%right</a> or
<a href='#pnonassoc'>%nonassoc</a> directives. Terminal symbols
mentioned in earlier directives have a lower precedence that
terminal symbols mentioned in later directives. For example:</p>
@@ -525,7 +528,11 @@ other than that, the order of directives in Lemon is arbitrary.</p>
<li><tt>%default_destructor</tt>
<li><tt>%default_type</tt>
<li><tt>%destructor</tt>
<li><tt>%endif</tt>
<li><tt>%extra_argument</tt>
<li><tt>%fallback</tt>
<li><tt>%ifdef</tt>
<li><tt>%ifndef</tt>
<li><tt>%include</tt>
<li><tt>%left</tt>
<li><tt>%name</tt>
@@ -537,49 +544,57 @@ other than that, the order of directives in Lemon is arbitrary.</p>
<li><tt>%stack_size</tt>
<li><tt>%start_symbol</tt>
<li><tt>%syntax_error</tt>
<li><tt>%token_class</tt>
<li><tt>%token_destructor</tt>
<li><tt>%token_prefix</tt>
<li><tt>%token_type</tt>
<li><tt>%type</tt>
<li><tt>%wildcard</tt>
</ul>
Each of these directives will be described separately in the
following sections:</p>
<a name='pcode'></a>
<h4>The <tt>%code</tt> directive</h4>
<p>The %code directive is used to specify addition C/C++ code that
<p>The %code directive is used to specify addition C code that
is added to the end of the main output file. This is similar to
the %include directive except that %include is inserted at the
beginning of the main output file.</p>
the <a href='#pinclude'>%include</a> directive except that %include
is inserted at the beginning of the main output file.</p>
<p>%code is typically used to include some action routines or perhaps
a tokenizer as part of the output file.</p>
a tokenizer or even the "main()" function
as part of the output file.</p>
<a name='default_destructor'></a>
<h4>The <tt>%default_destructor</tt> directive</h4>
<p>The %default_destructor directive specifies a destructor to
use for non-terminals that do not have their own destructor
specified by a separate %destructor directive. See the documentation
on the %destructor directive below for additional information.</p>
on the <a name='#destructor'>%destructor</a> directive below for
additional information.</p>
<p>In some grammers, many different non-terminal symbols have the
same datatype and hence the same destructor. This directive is
a convenience way to specify the same destructor for all those
non-terminals using a single statement.</p>
<a name='default_type'></a>
<h4>The <tt>%default_type</tt> directive</h4>
<p>The %default_type directive specifies the datatype of non-terminal
symbols that do no have their own datatype defined using a separate
%type directive. See the documentation on %type below for addition
information.</p>
<a href='#ptype'>%type</a> directive.
</p>
<a name='destructor'></a>
<h4>The <tt>%destructor</tt> directive</h4>
<p>The %destructor directive is used to specify a destructor for
a non-terminal symbol.
(See also the %token_destructor directive which is used to
specify a destructor for terminal symbols.)</p>
(See also the <a href='#token_destructor'>%token_destructor</a>
directive which is used to specify a destructor for terminal symbols.)</p>
<p>A non-terminal's destructor is called to dispose of the
non-terminal's value whenever the non-terminal is popped from
@@ -602,26 +617,25 @@ or other resources held by that non-terminal.</p>
</pre>
This example is a bit contrived but it serves to illustrate how
destructors work. The example shows a non-terminal named
``nt'' that holds values of type ``void*''. When the rule for
an ``nt'' reduces, it sets the value of the non-terminal to
"nt" that holds values of type "void*". When the rule for
an "nt" reduces, it sets the value of the non-terminal to
space obtained from malloc(). Later, when the nt non-terminal
is popped from the stack, the destructor will fire and call
free() on this malloced space, thus avoiding a memory leak.
(Note that the symbol ``$$'' in the destructor code is replaced
(Note that the symbol "$$" in the destructor code is replaced
by the value of the non-terminal.)</p>
<p>It is important to note that the value of a non-terminal is passed
to the destructor whenever the non-terminal is removed from the
stack, unless the non-terminal is used in a C-code action. If
the non-terminal is used by C-code, then it is assumed that the
C-code will take care of destroying it if it should really
be destroyed. More commonly, the value is used to build some
C-code will take care of destroying it.
More commonly, the value is used to build some
larger structure and we don't want to destroy it, which is why
the destructor is not called in this circumstance.</p>
<p>By appropriate use of destructors, it is possible to
build a parser using Lemon that can be used within a long-running
program, such as a GUI, that will not leak memory or other resources.
<p>Destructors help avoid memory leaks by automatically freeing
allocated objects when they go out of scope.
To do the same using yacc or bison is much more difficult.</p>
<a name="extraarg"></a>
@@ -638,17 +652,66 @@ and so forth. For example, if the grammar file contains:</p>
</pre></p>
<p>Then the Parse() function generated will have an 4th parameter
of type ``MyStruct*'' and all action routines will have access to
a variable named ``pAbc'' that is the value of the 4th parameter
of type "MyStruct*" and all action routines will have access to
a variable named "pAbc" that is the value of the 4th parameter
in the most recent call to Parse().</p>
<a name='pfallback'></a>
<h4>The <tt>%fallback</tt> directive</h4>
<p>The %fallback directive specifies an alternative meaning for one
or more tokens. The alternative meaning is tried if the original token
would have generated a syntax error.
<p>The %fallback directive was added to support robust parsing of SQL
syntax in <a href="https://www.sqlite.org/">SQLite</a>.
The SQL language contains a large assortment of keywords, each of which
appears as a different token to the language parser. SQL contains so
many keywords, that it can be difficult for programmers to keep up with
them all. Programmers will, therefore, sometimes mistakenly use an
obscure language keyword for an identifier. The %fallback directive
provides a mechanism to tell the parser: "If you are unable to parse
this keyword, try treating it as an identifier instead."
<p>The syntax of %fallback is as follows:
<blockquote>
<tt>%fallback</tt> <i>ID</i> <i>TOKEN...</i> <b>.</b>
</blockquote>
<p>In words, the %fallback directive is followed by a list of token names
terminated by a period. The first token name is the fallback token - the
token to which all the other tokens fall back to. The second and subsequent
arguments are tokens which fall back to the token identified by the first
argument.
<a name='pifdef'></a>
<h4>The <tt>%ifdef</tt>, <tt>%ifndef</tt>, and <tt>%endif</tt> directives.</h4>
<p>The %ifdef, %ifndef, and %endif directives are similar to
#ifdef, #ifndef, and #endif in the C-preprocessor, just not as general.
Each of these directives must begin at the left margin. No whitespace
is allowed between the "%" and the directive name.
<p>Grammar text in between "%ifdef MACRO" and the next nested "%endif" is
ignored unless the "-DMACRO" command-line option is used. Grammar text
betwen "%ifndef MACRO" and the next nested "%endif" is included except when
the "-DMACRO" command-line option is used.
<p>Note that the argument to %ifdef and %ifndef must be a single
preprocessor symbol name, not a general expression. There is no "%else"
directive.
<a name='pinclude'></a>
<h4>The <tt>%include</tt> directive</h4>
<p>The %include directive specifies C code that is included at the
top of the generated parser. You can include any text you want --
the Lemon parser generator copies it blindly. If you have multiple
%include directives in your grammar file the value of the last
%include directive overwrites all the others.</p.
%include directives in your grammar file, their values are concatenated
so that all %include code ultimately appears near the top of the
generated parser, in the same order as it appeared in the grammer.</p>
<p>The %include directive is very handy for getting some extra #include
preprocessor statements at the beginning of the generated parser.
@@ -661,12 +724,13 @@ For example:</p>
<p>This might be needed, for example, if some of the C actions in the
grammar call functions that are prototyed in unistd.h.</p>
<a name='pleft'></a>
<h4>The <tt>%left</tt> directive</h4>
The %left directive is used (along with the %right and
%nonassoc directives) to declare precedences of terminal
symbols. Every terminal symbol whose name appears after
a %left directive but before the next period (``.'') is
The %left directive is used (along with the <a href='#pright'>%right</a> and
<a href='#pnonassoc'>%nonassoc</a> directives) to declare precedences of
terminal symbols. Every terminal symbol whose name appears after
a %left directive but before the next period (".") is
given the same left-associative precedence value. Subsequent
%left directives have higher precedence. For example:</p>
@@ -687,10 +751,11 @@ a large amount of stack space if you make heavy use or right-associative
operators. For this reason, it is recommended that you use %left
rather than %right whenever possible.</p>
<a name='pname'></a>
<h4>The <tt>%name</tt> directive</h4>
<p>By default, the functions generated by Lemon all begin with the
five-character string ``Parse''. You can change this string to something
five-character string "Parse". You can change this string to something
different using the %name directive. For instance:</p>
<p><pre>
@@ -709,16 +774,19 @@ The %name directive allows you to generator two or more different
parsers and link them all into the same executable.
</p>
<a name='pnonassoc'></a>
<h4>The <tt>%nonassoc</tt> directive</h4>
<p>This directive is used to assign non-associative precedence to
one or more terminal symbols. See the section on precedence rules
or on the %left directive for additional information.</p>
one or more terminal symbols. See the section on
<a href='#precrules'>precedence rules</a>
or on the <a href='#pleft'>%left</a> directive for additional information.</p>
<a name='parse_accept'></a>
<h4>The <tt>%parse_accept</tt> directive</h4>
<p>The %parse_accept directive specifies a block of C code that is
executed whenever the parser accepts its input string. To ``accept''
executed whenever the parser accepts its input string. To "accept"
an input string means that the parser was able to process all tokens
without error.</p>
@@ -730,7 +798,7 @@ without error.</p>
}
</pre></p>
<a name='parse_failure'></a>
<h4>The <tt>%parse_failure</tt> directive</h4>
<p>The %parse_failure directive specifies a block of C code that
@@ -745,12 +813,15 @@ only invoked when parsing is unable to continue.</p>
}
</pre></p>
<a name='pright'></a>
<h4>The <tt>%right</tt> directive</h4>
<p>This directive is used to assign right-associative precedence to
one or more terminal symbols. See the section on precedence rules
or on the %left directive for additional information.</p>
one or more terminal symbols. See the section on
<a href='#precrules'>precedence rules</a>
or on the <a href='#pleft'>%left</a> directive for additional information.</p>
<a name='stack_overflow'></a>
<h4>The <tt>%stack_overflow</tt> directive</h4>
<p>The %stack_overflow directive specifies a block of C code that
@@ -779,6 +850,7 @@ Not like this:
list ::= .
</pre>
<a name='stack_size'></a>
<h4>The <tt>%stack_size</tt> directive</h4>
<p>If stack overflow is a problem and you can't resolve the trouble
@@ -791,6 +863,7 @@ with a stack of the requested size. The default value is 100.</p>
%stack_size 2000
</pre></p>
<a name='start_symbol'></a>
<h4>The <tt>%start_symbol</tt> directive</h4>
<p>By default, the start-symbol for the grammar that Lemon generates
@@ -801,6 +874,7 @@ can choose a different start-symbol using the %start_symbol directive.</p>
%start_symbol prog
</pre></p>
<a name='token_destructor'></a>
<h4>The <tt>%token_destructor</tt> directive</h4>
<p>The %destructor directive assigns a destructor to a non-terminal
@@ -813,6 +887,7 @@ the %token_type directive) and so they use a common destructor. Other
than that, the token destructor works just like the non-terminal
destructors.</p>
<a name='token_prefix'></a>
<h4>The <tt>%token_prefix</tt> directive</h4>
<p>Lemon generates #defines that assign small integer constants
@@ -838,6 +913,7 @@ to cause Lemon to produce these symbols instead:
#define TOKEN_PLUS 4
</pre>
<a name='token_type'></a><a name='ptype'></a>
<h4>The <tt>%token_type</tt> and <tt>%type</tt> directives</h4>
<p>These directives are used to specify the data types for values
@@ -853,7 +929,7 @@ token structure. Like this:</p>
</pre></p>
<p>If the data type of terminals is not specified, the default value
is ``int''.</p>
is "void*".</p>
<p>Non-terminal symbols can each have their own data types. Typically
the data type of a non-terminal is a pointer to the root of a parse-tree
@@ -874,6 +950,17 @@ non-terminal whose data type requires 1K of storage, then your 100
entry parser stack will require 100K of heap space. If you are willing
and able to pay that price, fine. You just need to know.</p>
<a name='pwildcard'></a>
<h4>The <tt>%wildcard</tt> directive</h4>
<p>The %wildcard directive is followed by a single token name and a
period. This directive specifies that the identified token should
match any input token.
<p>When the generated parser has the choice of matching an input against
the wildcard token and some other token, the other token is always used.
The wildcard token is only matched if there are no other alternatives.
<h3>Error Processing</h3>
<p>After extensive experimentation over several years, it has been
@@ -885,7 +972,7 @@ first invokes the code specified by the %syntax_error directive, if
any. It then enters its error recovery strategy. The error recovery
strategy is to begin popping the parsers stack until it enters a
state where it is permitted to shift a special non-terminal symbol
named ``error''. It then shifts this non-terminal and continues
named "error". It then shifts this non-terminal and continues
parsing. But the %syntax_error routine will not be called again
until at least three new tokens have been successfully shifted.</p>
@@ -894,7 +981,7 @@ is unable to shift the error symbol, then the %parse_failed routine
is invoked and the parser resets itself to its start state, ready
to begin parsing a new file. This is what will happen at the very
first syntax error, of course, if there are no instances of the
``error'' non-terminal in your grammar.</p>
"error" non-terminal in your grammar.</p>
</body>
</html>

8
ext/README.md Normal file
View File

@@ -0,0 +1,8 @@
## Loadable Extensions
Various [loadable extensions](https://www.sqlite.org/loadext.html) for
SQLite are found in subfolders.
Most subfolders are dedicated to a single loadable extension (for
example FTS5, or RTREE). But the misc/ subfolder contains a collection
of smaller single-file extensions.

2
ext/README.txt
View File

@@ -1,2 +0,0 @@
Version loadable extensions to SQLite are found in subfolders
of this folder.

6
ext/fts2/fts2_tokenizer.c
View File

@@ -99,7 +99,11 @@ static void scalarFunc(
#ifdef SQLITE_TEST
#include <tcl.h>
#if defined(INCLUDE_SQLITE_TCL_H)
# include "sqlite_tcl.h"
#else
# include "tcl.h"
#endif
#include <string.h>
/*

72
ext/fts3/fts3.c
View File

@@ -349,8 +349,9 @@ int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){
** Return the number of bytes read, or 0 on error.
** The value is stored in *v.
*/
int sqlite3Fts3GetVarint(const char *p, sqlite_int64 *v){
const char *pStart = p;
int sqlite3Fts3GetVarint(const char *pBuf, sqlite_int64 *v){
const unsigned char *p = (const unsigned char*)pBuf;
const unsigned char *pStart = p;
u32 a;
u64 b;
int shift;
@@ -492,6 +493,7 @@ static int fts3DisconnectMethod(sqlite3_vtab *pVtab){
assert( p->pSegments==0 );
/* Free any prepared statements held */
sqlite3_finalize(p->pSeekStmt);
for(i=0; i<SizeofArray(p->aStmt); i++){
sqlite3_finalize(p->aStmt[i]);
}
@@ -1363,9 +1365,9 @@ static int fts3InitVtab(
p->pTokenizer = pTokenizer;
p->nMaxPendingData = FTS3_MAX_PENDING_DATA;
p->bHasDocsize = (isFts4 && bNoDocsize==0);
p->bHasStat = isFts4;
p->bFts4 = isFts4;
p->bDescIdx = bDescIdx;
p->bHasStat = (u8)isFts4;
p->bFts4 = (u8)isFts4;
p->bDescIdx = (u8)bDescIdx;
p->nAutoincrmerge = 0xff; /* 0xff means setting unknown */
p->zContentTbl = zContent;
p->zLanguageid = zLanguageid;
@@ -1396,7 +1398,9 @@ static int fts3InitVtab(
char *z;
int n = 0;
z = (char *)sqlite3Fts3NextToken(aCol[iCol], &n);
memcpy(zCsr, z, n);
if( n>0 ){
memcpy(zCsr, z, n);
}
zCsr[n] = '\0';
sqlite3Fts3Dequote(zCsr);
p->azColumn[iCol] = zCsr;
@@ -1680,6 +1684,26 @@ static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
return SQLITE_OK;
}
/*
** Finalize the statement handle at pCsr->pStmt.
**
** Or, if that statement handle is one created by fts3CursorSeekStmt(),
** and the Fts3Table.pSeekStmt slot is currently NULL, save the statement
** pointer there instead of finalizing it.
*/
static void fts3CursorFinalizeStmt(Fts3Cursor *pCsr){
if( pCsr->bSeekStmt ){
Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
if( p->pSeekStmt==0 ){
p->pSeekStmt = pCsr->pStmt;
sqlite3_reset(pCsr->pStmt);
pCsr->pStmt = 0;
}
pCsr->bSeekStmt = 0;
}
sqlite3_finalize(pCsr->pStmt);
}
/*
** Close the cursor. For additional information see the documentation
** on the xClose method of the virtual table interface.
@@ -1687,7 +1711,7 @@ static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){
Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
sqlite3_finalize(pCsr->pStmt);
fts3CursorFinalizeStmt(pCsr);
sqlite3Fts3ExprFree(pCsr->pExpr);
sqlite3Fts3FreeDeferredTokens(pCsr);
sqlite3_free(pCsr->aDoclist);
@@ -1705,20 +1729,23 @@ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){
**
** (or the equivalent for a content=xxx table) and set pCsr->pStmt to
** it. If an error occurs, return an SQLite error code.
**
** Otherwise, set *ppStmt to point to pCsr->pStmt and return SQLITE_OK.
*/
static int fts3CursorSeekStmt(Fts3Cursor *pCsr, sqlite3_stmt **ppStmt){
static int fts3CursorSeekStmt(Fts3Cursor *pCsr){
int rc = SQLITE_OK;
if( pCsr->pStmt==0 ){
Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
char *zSql;
zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist);
if( !zSql ) return SQLITE_NOMEM;
rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
sqlite3_free(zSql);
if( p->pSeekStmt ){
pCsr->pStmt = p->pSeekStmt;
p->pSeekStmt = 0;
}else{
zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist);
if( !zSql ) return SQLITE_NOMEM;
rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
sqlite3_free(zSql);
}
if( rc==SQLITE_OK ) pCsr->bSeekStmt = 1;
}
*ppStmt = pCsr->pStmt;
return rc;
}
@@ -1730,9 +1757,7 @@ static int fts3CursorSeekStmt(Fts3Cursor *pCsr, sqlite3_stmt **ppStmt){
static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){
int rc = SQLITE_OK;
if( pCsr->isRequireSeek ){
sqlite3_stmt *pStmt = 0;
rc = fts3CursorSeekStmt(pCsr, &pStmt);
rc = fts3CursorSeekStmt(pCsr);
if( rc==SQLITE_OK ){
sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId);
pCsr->isRequireSeek = 0;
@@ -3190,7 +3215,7 @@ static int fts3FilterMethod(
assert( iIdx==nVal );
/* In case the cursor has been used before, clear it now. */
sqlite3_finalize(pCsr->pStmt);
fts3CursorFinalizeStmt(pCsr);
sqlite3_free(pCsr->aDoclist);
sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
sqlite3Fts3ExprFree(pCsr->pExpr);
@@ -3258,7 +3283,7 @@ static int fts3FilterMethod(
rc = SQLITE_NOMEM;
}
}else if( eSearch==FTS3_DOCID_SEARCH ){
rc = fts3CursorSeekStmt(pCsr, &pCsr->pStmt);
rc = fts3CursorSeekStmt(pCsr);
if( rc==SQLITE_OK ){
rc = sqlite3_bind_value(pCsr->pStmt, 1, pCons);
}
@@ -3386,8 +3411,10 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){
const u32 nMinMerge = 64; /* Minimum amount of incr-merge work to do */
Fts3Table *p = (Fts3Table*)pVtab;
int rc = sqlite3Fts3PendingTermsFlush(p);
int rc;
i64 iLastRowid = sqlite3_last_insert_rowid(p->db);
rc = sqlite3Fts3PendingTermsFlush(p);
if( rc==SQLITE_OK
&& p->nLeafAdd>(nMinMerge/16)
&& p->nAutoincrmerge && p->nAutoincrmerge!=0xff
@@ -3402,6 +3429,7 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){
if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, p->nAutoincrmerge);
}
sqlite3Fts3SegmentsClose(p);
sqlite3_set_last_insert_rowid(p->db, iLastRowid);
return rc;
}
@@ -3422,7 +3450,7 @@ static int fts3SetHasStat(Fts3Table *p){
if( rc==SQLITE_OK ){
int bHasStat = (sqlite3_step(pStmt)==SQLITE_ROW);
rc = sqlite3_finalize(pStmt);
if( rc==SQLITE_OK ) p->bHasStat = bHasStat;
if( rc==SQLITE_OK ) p->bHasStat = (u8)bHasStat;
}
sqlite3_free(zSql);
}else{

2
ext/fts3/fts3Int.h
View File

@@ -230,6 +230,7 @@ struct Fts3Table {
** statements is run and reset within a single virtual table API call.
*/
sqlite3_stmt *aStmt[40];
sqlite3_stmt *pSeekStmt; /* Cache for fts3CursorSeekStmt() */
char *zReadExprlist;
char *zWriteExprlist;
@@ -299,6 +300,7 @@ struct Fts3Cursor {
i16 eSearch; /* Search strategy (see below) */
u8 isEof; /* True if at End Of Results */
u8 isRequireSeek; /* True if must seek pStmt to %_content row */
u8 bSeekStmt; /* True if pStmt is a seek */
sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */
Fts3Expr *pExpr; /* Parsed MATCH query string */
int iLangid; /* Language being queried for */

19
ext/fts3/fts3_test.c
View File

@@ -18,7 +18,14 @@
** that the sqlite3_tokenizer_module.xLanguage() method is invoked correctly.
*/
#include <tcl.h>
#if defined(INCLUDE_SQLITE_TCL_H)
# include "sqlite_tcl.h"
#else
# include "tcl.h"
# ifndef SQLITE_TCLAPI
# define SQLITE_TCLAPI
# endif
#endif
#include <string.h>
#include <assert.h>
@@ -143,7 +150,7 @@ static int nm_match_count(
/*
** Tclcmd: fts3_near_match DOCUMENT EXPR ?OPTIONS?
*/
static int fts3_near_match_cmd(
static int SQLITE_TCLAPI fts3_near_match_cmd(
ClientData clientData,
Tcl_Interp *interp,
int objc,
@@ -278,7 +285,7 @@ static int fts3_near_match_cmd(
** # Restore initial incr-load settings:
** eval fts3_configure_incr_load $cfg
*/
static int fts3_configure_incr_load_cmd(
static int SQLITE_TCLAPI fts3_configure_incr_load_cmd(
ClientData clientData,
Tcl_Interp *interp,
int objc,
@@ -458,7 +465,7 @@ static int testTokenizerNext(
if( pCsr->iLangid & 0x00000001 ){
for(i=0; i<nToken; i++) pCsr->aBuffer[i] = pToken[i];
}else{
for(i=0; i<nToken; i++) pCsr->aBuffer[i] = testTolower(pToken[i]);
for(i=0; i<nToken; i++) pCsr->aBuffer[i] = (char)testTolower(pToken[i]);
}
pCsr->iToken++;
pCsr->iInput = (int)(p - pCsr->aInput);
@@ -488,7 +495,7 @@ static int testTokenizerLanguage(
}
#endif
static int fts3_test_tokenizer_cmd(
static int SQLITE_TCLAPI fts3_test_tokenizer_cmd(
ClientData clientData,
Tcl_Interp *interp,
int objc,
@@ -517,7 +524,7 @@ static int fts3_test_tokenizer_cmd(
return TCL_OK;
}
static int fts3_test_varint_cmd(
static int SQLITE_TCLAPI fts3_test_varint_cmd(
ClientData clientData,
Tcl_Interp *interp,
int objc,

6
ext/fts3/fts3_tokenizer.c
View File

@@ -224,7 +224,11 @@ int sqlite3Fts3InitTokenizer(
#ifdef SQLITE_TEST
#include <tcl.h>
#if defined(INCLUDE_SQLITE_TCL_H)
# include "sqlite_tcl.h"
#else
# include "tcl.h"
#endif
#include <string.h>
/*

26
ext/fts3/fts3_unicode.c
View File

@@ -136,16 +136,16 @@ static int unicodeAddExceptions(
){
const unsigned char *z = (const unsigned char *)zIn;
const unsigned char *zTerm = &z[nIn];
int iCode;
unsigned int iCode;
int nEntry = 0;
assert( bAlnum==0 || bAlnum==1 );
while( z<zTerm ){
READ_UTF8(z, zTerm, iCode);
assert( (sqlite3FtsUnicodeIsalnum(iCode) & 0xFFFFFFFE)==0 );
if( sqlite3FtsUnicodeIsalnum(iCode)!=bAlnum
&& sqlite3FtsUnicodeIsdiacritic(iCode)==0
assert( (sqlite3FtsUnicodeIsalnum((int)iCode) & 0xFFFFFFFE)==0 );
if( sqlite3FtsUnicodeIsalnum((int)iCode)!=bAlnum
&& sqlite3FtsUnicodeIsdiacritic((int)iCode)==0
){
nEntry++;
}
@@ -162,13 +162,13 @@ static int unicodeAddExceptions(
z = (const unsigned char *)zIn;
while( z<zTerm ){
READ_UTF8(z, zTerm, iCode);
if( sqlite3FtsUnicodeIsalnum(iCode)!=bAlnum
&& sqlite3FtsUnicodeIsdiacritic(iCode)==0
if( sqlite3FtsUnicodeIsalnum((int)iCode)!=bAlnum
&& sqlite3FtsUnicodeIsdiacritic((int)iCode)==0
){
int i, j;
for(i=0; i<nNew && aNew[i]<iCode; i++);
for(i=0; i<nNew && aNew[i]<(int)iCode; i++);
for(j=nNew; j>i; j--) aNew[j] = aNew[j-1];
aNew[i] = iCode;
aNew[i] = (int)iCode;
nNew++;
}
}
@@ -318,7 +318,7 @@ static int unicodeNext(
){
unicode_cursor *pCsr = (unicode_cursor *)pC;
unicode_tokenizer *p = ((unicode_tokenizer *)pCsr->base.pTokenizer);
int iCode = 0;
unsigned int iCode = 0;
char *zOut;
const unsigned char *z = &pCsr->aInput[pCsr->iOff];
const unsigned char *zStart = z;
@@ -330,7 +330,7 @@ static int unicodeNext(
** the input. */
while( z<zTerm ){
READ_UTF8(z, zTerm, iCode);
if( unicodeIsAlnum(p, iCode) ) break;
if( unicodeIsAlnum(p, (int)iCode) ) break;
zStart = z;
}
if( zStart>=zTerm ) return SQLITE_DONE;
@@ -350,7 +350,7 @@ static int unicodeNext(
/* Write the folded case of the last character read to the output */
zEnd = z;
iOut = sqlite3FtsUnicodeFold(iCode, p->bRemoveDiacritic);
iOut = sqlite3FtsUnicodeFold((int)iCode, p->bRemoveDiacritic);
if( iOut ){
WRITE_UTF8(zOut, iOut);
}
@@ -358,8 +358,8 @@ static int unicodeNext(
/* If the cursor is not at EOF, read the next character */
if( z>=zTerm ) break;
READ_UTF8(z, zTerm, iCode);
}while( unicodeIsAlnum(p, iCode)
|| sqlite3FtsUnicodeIsdiacritic(iCode)
}while( unicodeIsAlnum(p, (int)iCode)
|| sqlite3FtsUnicodeIsdiacritic((int)iCode)
);
/* Set the output variables and return. */

21
ext/fts3/fts3_unicode2.c
View File

@@ -127,9 +127,9 @@ int sqlite3FtsUnicodeIsalnum(int c){
0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001,
};
if( c<128 ){
return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 );
}else if( c<(1<<22) ){
if( (unsigned int)c<128 ){
return ( (aAscii[c >> 5] & ((unsigned int)1 << (c & 0x001F)))==0 );
}else if( (unsigned int)c<(1<<22) ){
unsigned int key = (((unsigned int)c)<<10) | 0x000003FF;
int iRes = 0;
int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
@@ -322,16 +322,17 @@ int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){
int ret = c;
assert( c>=0 );
assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 );
if( c<128 ){
if( c>='A' && c<='Z' ) ret = c + ('a' - 'A');
}else if( c<65536 ){
const struct TableEntry *p;
int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
int iLo = 0;
int iRes = -1;
assert( c>aEntry[0].iCode );
while( iHi>=iLo ){
int iTest = (iHi + iLo) / 2;
int cmp = (c - aEntry[iTest].iCode);
@@ -342,14 +343,12 @@ int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){
iHi = iTest-1;
}
}
assert( iRes<0 || c>=aEntry[iRes].iCode );
if( iRes>=0 ){
const struct TableEntry *p = &aEntry[iRes];
if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
assert( ret>0 );
}
assert( iRes>=0 && c>=aEntry[iRes].iCode );
p = &aEntry[iRes];
if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
assert( ret>0 );
}
if( bRemoveDiacritic ) ret = remove_diacritic(ret);

5
ext/fts3/fts3_write.c
View File

@@ -4956,11 +4956,14 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){
** Convert the text beginning at *pz into an integer and return
** its value. Advance *pz to point to the first character past
** the integer.
**
** This function used for parameters to merge= and incrmerge=
** commands.
*/
static int fts3Getint(const char **pz){
const char *z = *pz;
int i = 0;
while( (*z)>='0' && (*z)<='9' ) i = 10*i + *(z++) - '0';
while( (*z)>='0' && (*z)<='9' && i<214748363 ) i = 10*i + *(z++) - '0';
*pz = z;
return i;
}

16
ext/fts3/tool/fts3cov.sh Normal file
View File

@@ -0,0 +1,16 @@
#!/bin/sh
set -e
srcdir=`dirname $(dirname $(dirname $(dirname $0)))`
./testfixture $srcdir/test/fts3.test --output=fts3cov-out.txt
echo ""
for f in `ls $srcdir/ext/fts3/*.c`
do
f=`basename $f`
echo -ne "$f: "
gcov -b $f | grep Taken | sed 's/Taken at least once://'
done

2
ext/fts3/tool/fts3view.c
View File

@@ -398,8 +398,8 @@ static void showSegmentStats(sqlite3 *db, const char *zTab){
if( sqlite3_step(pStmt)==SQLITE_ROW
&& (nLeaf = sqlite3_column_int(pStmt, 0))>0
){
nIdx = sqlite3_column_int(pStmt, 5);
sqlite3_int64 sz;
nIdx = sqlite3_column_int(pStmt, 5);
printf("For level %d:\n", i);
printf(" Number of indexes...................... %9d\n", nIdx);
printf(" Number of leaf segments................ %9d\n", nLeaf);

2
ext/fts3/unicode/mkunicode.tcl
View File

@@ -227,7 +227,7 @@ proc print_isalnum {zFunc lRange} {
an_print_ascii_bitmap $lRange
puts {
if( (unsigned int)c<128 ){
return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 );
return ( (aAscii[c >> 5] & ((unsigned int)1 << (c & 0x001F)))==0 );
}else if( (unsigned int)c<(1<<22) ){
unsigned int key = (((unsigned int)c)<<10) | 0x000003FF;
int iRes = 0;

15
ext/fts5/fts5.h
View File

@@ -143,11 +143,13 @@ struct Fts5PhraseIter {
** ... FROM ftstable WHERE ftstable MATCH $p ORDER BY rowid
**
** with $p set to a phrase equivalent to the phrase iPhrase of the
** current query is executed. For each row visited, the callback function
** passed as the fourth argument is invoked. The context and API objects
** passed to the callback function may be used to access the properties of
** each matched row. Invoking Api.xUserData() returns a copy of the pointer
** passed as the third argument to pUserData.
** current query is executed. Any column filter that applies to
** phrase iPhrase of the current query is included in $p. For each
** row visited, the callback function passed as the fourth argument
** is invoked. The context and API objects passed to the callback
** function may be used to access the properties of each matched row.
** Invoking Api.xUserData() returns a copy of the pointer passed as
** the third argument to pUserData.
**
** If the callback function returns any value other than SQLITE_OK, the
** query is abandoned and the xQueryPhrase function returns immediately.
@@ -316,7 +318,7 @@ struct Fts5ExtensionApi {
** behaviour. The structure methods are expected to function as follows:
**
** xCreate:
** This function is used to allocate and inititalize a tokenizer instance.
** This function is used to allocate and initialize a tokenizer instance.
** A tokenizer instance is required to actually tokenize text.
**
** The first argument passed to this function is a copy of the (void*)
@@ -575,4 +577,3 @@ struct fts5_api {
#endif
#endif /* _FTS5_H */

12
ext/fts5/fts5Int.h
View File

@@ -30,7 +30,9 @@ typedef short i16;
typedef sqlite3_int64 i64;
typedef sqlite3_uint64 u64;
#define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0])))
#ifndef ArraySize
# define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0])))
#endif
#define testcase(x)
#define ALWAYS(x) 1
@@ -47,6 +49,10 @@ typedef sqlite3_uint64 u64;
#endif
/* Truncate very long tokens to this many bytes. Hard limit is
** (65536-1-1-4-9)==65521 bytes. The limiting factor is the 16-bit offset
** field that occurs at the start of each leaf page (see fts5_index.c). */
#define FTS5_MAX_TOKEN_SIZE 32768
/*
** Maximum number of prefix indexes on single FTS5 table. This must be
@@ -480,6 +486,7 @@ int sqlite3Fts5IndexReads(Fts5Index *p);
int sqlite3Fts5IndexReinit(Fts5Index *p);
int sqlite3Fts5IndexOptimize(Fts5Index *p);
int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge);
int sqlite3Fts5IndexReset(Fts5Index *p);
int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
@@ -622,6 +629,7 @@ int sqlite3Fts5StorageDeleteAll(Fts5Storage *p);
int sqlite3Fts5StorageRebuild(Fts5Storage *p);
int sqlite3Fts5StorageOptimize(Fts5Storage *p);
int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge);
int sqlite3Fts5StorageReset(Fts5Storage *p);
/*
** End of interface to code in fts5_storage.c.
@@ -680,7 +688,6 @@ int sqlite3Fts5ExprPopulatePoslists(
Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int
);
void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64);
void sqlite3Fts5ExprClearEof(Fts5Expr*);
int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**);
@@ -732,6 +739,7 @@ void sqlite3Fts5ParseNodeFree(Fts5ExprNode*);
void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNearset*, Fts5Colset*);
Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse*, Fts5Colset*);
void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p);
void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);

232
ext/fts5/fts5_aux.c
View File

@@ -189,7 +189,7 @@ static int fts5HighlightCb(
if( p->iRangeEnd>0 && iPos==p->iRangeEnd ){
fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
p->iOff = iEndOff;
if( iPos<p->iter.iEnd ){
if( iPos>=p->iter.iStart && iPos<p->iter.iEnd ){
fts5HighlightAppend(&rc, p, p->zClose, -1);
}
}
@@ -246,6 +246,118 @@ static void fts5HighlightFunction(
** End of highlight() implementation.
**************************************************************************/
/*
** Context object passed to the fts5SentenceFinderCb() function.
*/
typedef struct Fts5SFinder Fts5SFinder;
struct Fts5SFinder {
int iPos; /* Current token position */
int nFirstAlloc; /* Allocated size of aFirst[] */
int nFirst; /* Number of entries in aFirst[] */
int *aFirst; /* Array of first token in each sentence */
const char *zDoc; /* Document being tokenized */
};
/*
** Add an entry to the Fts5SFinder.aFirst[] array. Grow the array if
** necessary. Return SQLITE_OK if successful, or SQLITE_NOMEM if an
** error occurs.
*/
static int fts5SentenceFinderAdd(Fts5SFinder *p, int iAdd){
if( p->nFirstAlloc==p->nFirst ){
int nNew = p->nFirstAlloc ? p->nFirstAlloc*2 : 64;
int *aNew;
aNew = (int*)sqlite3_realloc(p->aFirst, nNew*sizeof(int));
if( aNew==0 ) return SQLITE_NOMEM;
p->aFirst = aNew;
p->nFirstAlloc = nNew;
}
p->aFirst[p->nFirst++] = iAdd;
return SQLITE_OK;
}
/*
** This function is an xTokenize() callback used by the auxiliary snippet()
** function. Its job is to identify tokens that are the first in a sentence.
** For each such token, an entry is added to the SFinder.aFirst[] array.
*/
static int fts5SentenceFinderCb(
void *pContext, /* Pointer to HighlightContext object */
int tflags, /* Mask of FTS5_TOKEN_* flags */
const char *pToken, /* Buffer containing token */
int nToken, /* Size of token in bytes */
int iStartOff, /* Start offset of token */
int iEndOff /* End offset of token */
){
int rc = SQLITE_OK;
UNUSED_PARAM2(pToken, nToken);
UNUSED_PARAM(iEndOff);
if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){
Fts5SFinder *p = (Fts5SFinder*)pContext;
if( p->iPos>0 ){
int i;
char c = 0;
for(i=iStartOff-1; i>=0; i--){
c = p->zDoc[i];
if( c!=' ' && c!='\t' && c!='\n' && c!='\r' ) break;
}
if( i!=iStartOff-1 && (c=='.' || c==':') ){
rc = fts5SentenceFinderAdd(p, p->iPos);
}
}else{
rc = fts5SentenceFinderAdd(p, 0);
}
p->iPos++;
}
return rc;
}
static int fts5SnippetScore(
const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
Fts5Context *pFts, /* First arg to pass to pApi functions */
int nDocsize, /* Size of column in tokens */
unsigned char *aSeen, /* Array with one element per query phrase */
int iCol, /* Column to score */
int iPos, /* Starting offset to score */
int nToken, /* Max tokens per snippet */
int *pnScore, /* OUT: Score */
int *piPos /* OUT: Adjusted offset */
){
int rc;
int i;
int ip = 0;
int ic = 0;
int iOff = 0;
int iFirst = -1;
int nInst;
int nScore = 0;
int iLast = 0;
rc = pApi->xInstCount(pFts, &nInst);
for(i=0; i<nInst && rc==SQLITE_OK; i++){
rc = pApi->xInst(pFts, i, &ip, &ic, &iOff);
if( rc==SQLITE_OK && ic==iCol && iOff>=iPos && iOff<(iPos+nToken) ){
nScore += (aSeen[ip] ? 1 : 1000);
aSeen[ip] = 1;
if( iFirst<0 ) iFirst = iOff;
iLast = iOff + pApi->xPhraseSize(pFts, ip);
}
}
*pnScore = nScore;
if( piPos ){
int iAdj = iFirst - (nToken - (iLast-iFirst)) / 2;
if( (iAdj+nToken)>nDocsize ) iAdj = nDocsize - nToken;
if( iAdj<0 ) iAdj = 0;
*piPos = iAdj;
}
return rc;
}
/*
** Implementation of snippet() function.
*/
@@ -267,9 +379,10 @@ static void fts5SnippetFunction(
unsigned char *aSeen; /* Array of "seen instance" flags */
int iBestCol; /* Column containing best snippet */
int iBestStart = 0; /* First token of best snippet */
int iBestLast; /* Last token of best snippet */
int nBestScore = 0; /* Score of best snippet */
int nColSize = 0; /* Total size of iBestCol in tokens */
Fts5SFinder sFinder; /* Used to find the beginnings of sentences */
int nCol;
if( nVal!=5 ){
const char *zErr = "wrong number of arguments to function snippet()";
@@ -277,13 +390,13 @@ static void fts5SnippetFunction(
return;
}
nCol = pApi->xColumnCount(pFts);
memset(&ctx, 0, sizeof(HighlightContext));
iCol = sqlite3_value_int(apVal[0]);
ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]);
ctx.zClose = (const char*)sqlite3_value_text(apVal[2]);
zEllips = (const char*)sqlite3_value_text(apVal[3]);
nToken = sqlite3_value_int(apVal[4]);
iBestLast = nToken-1;
iBestCol = (iCol>=0 ? iCol : 0);
nPhrase = pApi->xPhraseCount(pFts);
@@ -291,65 +404,94 @@ static void fts5SnippetFunction(
if( aSeen==0 ){
rc = SQLITE_NOMEM;
}
if( rc==SQLITE_OK ){
rc = pApi->xInstCount(pFts, &nInst);
}
for(i=0; rc==SQLITE_OK && i<nInst; i++){
int ip, iSnippetCol, iStart;
memset(aSeen, 0, nPhrase);
rc = pApi->xInst(pFts, i, &ip, &iSnippetCol, &iStart);
if( rc==SQLITE_OK && (iCol<0 || iSnippetCol==iCol) ){
int nScore = 1000;
int iLast = iStart - 1 + pApi->xPhraseSize(pFts, ip);
int j;
aSeen[ip] = 1;
for(j=i+1; rc==SQLITE_OK && j<nInst; j++){
int ic; int io; int iFinal;
rc = pApi->xInst(pFts, j, &ip, &ic, &io);
iFinal = io + pApi->xPhraseSize(pFts, ip) - 1;
if( rc==SQLITE_OK && ic==iSnippetCol && iLast<iStart+nToken ){
nScore += aSeen[ip] ? 1000 : 1;
aSeen[ip] = 1;
if( iFinal>iLast ) iLast = iFinal;
memset(&sFinder, 0, sizeof(Fts5SFinder));
for(i=0; i<nCol; i++){
if( iCol<0 || iCol==i ){
int nDoc;
int nDocsize;
int ii;
sFinder.iPos = 0;
sFinder.nFirst = 0;
rc = pApi->xColumnText(pFts, i, &sFinder.zDoc, &nDoc);
if( rc!=SQLITE_OK ) break;
rc = pApi->xTokenize(pFts,
sFinder.zDoc, nDoc, (void*)&sFinder,fts5SentenceFinderCb
);
if( rc!=SQLITE_OK ) break;
rc = pApi->xColumnSize(pFts, i, &nDocsize);
if( rc!=SQLITE_OK ) break;
for(ii=0; rc==SQLITE_OK && ii<nInst; ii++){
int ip, ic, io;
int iAdj;
int nScore;
int jj;
rc = pApi->xInst(pFts, ii, &ip, &ic, &io);
if( ic!=i || rc!=SQLITE_OK ) continue;
memset(aSeen, 0, nPhrase);
rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i,
io, nToken, &nScore, &iAdj
);
if( rc==SQLITE_OK && nScore>nBestScore ){
nBestScore = nScore;
iBestCol = i;
iBestStart = iAdj;
nColSize = nDocsize;
}
}
if( rc==SQLITE_OK && nScore>nBestScore ){
iBestCol = iSnippetCol;
iBestStart = iStart;
iBestLast = iLast;
nBestScore = nScore;
if( rc==SQLITE_OK && sFinder.nFirst && nDocsize>nToken ){
for(jj=0; jj<(sFinder.nFirst-1); jj++){
if( sFinder.aFirst[jj+1]>io ) break;
}
if( sFinder.aFirst[jj]<io ){
memset(aSeen, 0, nPhrase);
rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i,
sFinder.aFirst[jj], nToken, &nScore, 0
);
nScore += (sFinder.aFirst[jj]==0 ? 120 : 100);
if( rc==SQLITE_OK && nScore>nBestScore ){
nBestScore = nScore;
iBestCol = i;
iBestStart = sFinder.aFirst[jj];
nColSize = nDocsize;
}
}
}
}
}
}
if( rc==SQLITE_OK ){
rc = pApi->xColumnSize(pFts, iBestCol, &nColSize);
}
if( rc==SQLITE_OK ){
rc = pApi->xColumnText(pFts, iBestCol, &ctx.zIn, &ctx.nIn);
}
if( rc==SQLITE_OK && nColSize==0 ){
rc = pApi->xColumnSize(pFts, iBestCol, &nColSize);
}
if( ctx.zIn ){
if( rc==SQLITE_OK ){
rc = fts5CInstIterInit(pApi, pFts, iBestCol, &ctx.iter);
}
if( (iBestStart+nToken-1)>iBestLast ){
iBestStart -= (iBestStart+nToken-1-iBestLast) / 2;
}
if( iBestStart+nToken>nColSize ){
iBestStart = nColSize - nToken;
}
if( iBestStart<0 ) iBestStart = 0;
ctx.iRangeStart = iBestStart;
ctx.iRangeEnd = iBestStart + nToken - 1;
if( iBestStart>0 ){
fts5HighlightAppend(&rc, &ctx, zEllips, -1);
}
/* Advance iterator ctx.iter so that it points to the first coalesced
** phrase instance at or following position iBestStart. */
while( ctx.iter.iStart>=0 && ctx.iter.iStart<iBestStart && rc==SQLITE_OK ){
rc = fts5CInstIterNext(&ctx.iter);
}
if( rc==SQLITE_OK ){
rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb);
}
@@ -358,15 +500,15 @@ static void fts5SnippetFunction(
}else{
fts5HighlightAppend(&rc, &ctx, zEllips, -1);
}
if( rc==SQLITE_OK ){
sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT);
}else{
sqlite3_result_error_code(pCtx, rc);
}
sqlite3_free(ctx.zOut);
}
if( rc==SQLITE_OK ){
sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT);
}else{
sqlite3_result_error_code(pCtx, rc);
}
sqlite3_free(ctx.zOut);
sqlite3_free(aSeen);
sqlite3_free(sFinder.aFirst);
}
/************************************************************************/

175
ext/fts5/fts5_expr.c
View File

@@ -167,6 +167,7 @@ static int fts5ExprGetToken(
case ',': tok = FTS5_COMMA; break;
case '+': tok = FTS5_PLUS; break;
case '*': tok = FTS5_STAR; break;
case '-': tok = FTS5_MINUS; break;
case '\0': tok = FTS5_EOF; break;
case '"': {
@@ -745,49 +746,61 @@ static int fts5ExprNearTest(
** Initialize all term iterators in the pNear object. If any term is found
** to match no documents at all, return immediately without initializing any
** further iterators.
**
** If an error occurs, return an SQLite error code. Otherwise, return
** SQLITE_OK. It is not considered an error if some term matches zero
** documents.
*/
static int fts5ExprNearInitAll(
Fts5Expr *pExpr,
Fts5ExprNode *pNode
){
Fts5ExprNearset *pNear = pNode->pNear;
int i, j;
int rc = SQLITE_OK;
int i;
assert( pNode->bNomatch==0 );
for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
for(i=0; i<pNear->nPhrase; i++){
Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
for(j=0; j<pPhrase->nTerm; j++){
Fts5ExprTerm *pTerm = &pPhrase->aTerm[j];
Fts5ExprTerm *p;
int bEof = 1;
if( pPhrase->nTerm==0 ){
pNode->bEof = 1;
return SQLITE_OK;
}else{
int j;
for(j=0; j<pPhrase->nTerm; j++){
Fts5ExprTerm *pTerm = &pPhrase->aTerm[j];
Fts5ExprTerm *p;
int bHit = 0;
for(p=pTerm; p && rc==SQLITE_OK; p=p->pSynonym){
if( p->pIter ){
sqlite3Fts5IterClose(p->pIter);
p->pIter = 0;
for(p=pTerm; p; p=p->pSynonym){
int rc;
if( p->pIter ){
sqlite3Fts5IterClose(p->pIter);
p->pIter = 0;
}
rc = sqlite3Fts5IndexQuery(
pExpr->pIndex, p->zTerm, (int)strlen(p->zTerm),
(pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) |
(pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0),
pNear->pColset,
&p->pIter
);
assert( (rc==SQLITE_OK)==(p->pIter!=0) );
if( rc!=SQLITE_OK ) return rc;
if( 0==sqlite3Fts5IterEof(p->pIter) ){
bHit = 1;
}
}
rc = sqlite3Fts5IndexQuery(
pExpr->pIndex, p->zTerm, (int)strlen(p->zTerm),
(pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) |
(pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0),
pNear->pColset,
&p->pIter
);
assert( rc==SQLITE_OK || p->pIter==0 );
if( p->pIter && 0==sqlite3Fts5IterEof(p->pIter) ){
bEof = 0;
}
}
if( bEof ){
pNode->bEof = 1;
return rc;
if( bHit==0 ){
pNode->bEof = 1;
return SQLITE_OK;
}
}
}
}
return rc;
pNode->bEof = 0;
return SQLITE_OK;
}
/*
@@ -920,7 +933,7 @@ static int fts5ExprNodeTest_STRING(
}
}else{
Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter;
if( pIter->iRowid==iLast ) continue;
if( pIter->iRowid==iLast || pIter->bEof ) continue;
bMatch = 0;
if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){
return rc;
@@ -1097,7 +1110,10 @@ static int fts5ExprNodeNext_OR(
|| (bFromValid && fts5RowidCmp(pExpr, p1->iRowid, iFrom)<0)
){
int rc = fts5ExprNodeNext(pExpr, p1, bFromValid, iFrom);
if( rc!=SQLITE_OK ) return rc;
if( rc!=SQLITE_OK ){
pNode->bNomatch = 0;
return rc;
}
}
}
}
@@ -1128,7 +1144,10 @@ static int fts5ExprNodeTest_AND(
if( cmp>0 ){
/* Advance pChild until it points to iLast or laster */
rc = fts5ExprNodeNext(pExpr, pChild, 1, iLast);
if( rc!=SQLITE_OK ) return rc;
if( rc!=SQLITE_OK ){
pAnd->bNomatch = 0;
return rc;
}
}
/* If the child node is now at EOF, so is the parent AND node. Otherwise,
@@ -1167,6 +1186,8 @@ static int fts5ExprNodeNext_AND(
int rc = fts5ExprNodeNext(pExpr, pNode->apChild[0], bFromValid, iFrom);
if( rc==SQLITE_OK ){
rc = fts5ExprNodeTest_AND(pExpr, pNode);
}else{
pNode->bNomatch = 0;
}
return rc;
}
@@ -1209,6 +1230,9 @@ static int fts5ExprNodeNext_NOT(
if( rc==SQLITE_OK ){
rc = fts5ExprNodeTest_NOT(pExpr, pNode);
}
if( rc!=SQLITE_OK ){
pNode->bNomatch = 0;
}
return rc;
}
@@ -1331,7 +1355,10 @@ int sqlite3Fts5ExprFirst(Fts5Expr *p, Fts5Index *pIdx, i64 iFirst, int bDesc){
/* If not at EOF but the current rowid occurs earlier than iFirst in
** the iteration order, move to document iFirst or later. */
if( pRoot->bEof==0 && fts5RowidCmp(p, pRoot->iRowid, iFirst)<0 ){
if( rc==SQLITE_OK
&& 0==pRoot->bEof
&& fts5RowidCmp(p, pRoot->iRowid, iFirst)<0
){
rc = fts5ExprNodeNext(p, pRoot, 1, iFirst);
}
@@ -1493,6 +1520,7 @@ static int fts5ParseTokenize(
/* If an error has already occurred, this is a no-op */
if( pCtx->rc!=SQLITE_OK ) return pCtx->rc;
if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE;
if( pPhrase && pPhrase->nTerm>0 && (tflags & FTS5_TOKEN_COLOCATED) ){
Fts5ExprTerm *pSyn;
@@ -1584,7 +1612,7 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm(
rc = fts5ParseStringFromToken(pToken, &z);
if( rc==SQLITE_OK ){
int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_QUERY : 0);
int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_PREFIX : 0);
int n;
sqlite3Fts5Dequote(z);
n = (int)strlen(z);
@@ -1636,7 +1664,6 @@ int sqlite3Fts5ExprClonePhrase(
){
int rc = SQLITE_OK; /* Return code */
Fts5ExprPhrase *pOrig; /* The phrase extracted from pExpr */
int i; /* Used to iterate through phrase terms */
Fts5Expr *pNew = 0; /* Expression to return via *ppNew */
TokenCtx sCtx = {0,0}; /* Context object for fts5ParseTokenize */
@@ -1654,19 +1681,37 @@ int sqlite3Fts5ExprClonePhrase(
pNew->pRoot->pNear = (Fts5ExprNearset*)sqlite3Fts5MallocZero(&rc,
sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*));
}
if( rc==SQLITE_OK ){
Fts5Colset *pColsetOrig = pOrig->pNode->pNear->pColset;
if( pColsetOrig ){
int nByte = sizeof(Fts5Colset) + (pColsetOrig->nCol-1) * sizeof(int);
Fts5Colset *pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&rc, nByte);
if( pColset ){
memcpy(pColset, pColsetOrig, nByte);
}
pNew->pRoot->pNear->pColset = pColset;
}
}
for(i=0; rc==SQLITE_OK && i<pOrig->nTerm; i++){
int tflags = 0;
Fts5ExprTerm *p;
for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){
const char *zTerm = p->zTerm;
rc = fts5ParseTokenize((void*)&sCtx, tflags, zTerm, (int)strlen(zTerm),
0, 0);
tflags = FTS5_TOKEN_COLOCATED;
}
if( rc==SQLITE_OK ){
sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix;
if( pOrig->nTerm ){
int i; /* Used to iterate through phrase terms */
for(i=0; rc==SQLITE_OK && i<pOrig->nTerm; i++){
int tflags = 0;
Fts5ExprTerm *p;
for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){
const char *zTerm = p->zTerm;
rc = fts5ParseTokenize((void*)&sCtx, tflags, zTerm, (int)strlen(zTerm),
0, 0);
tflags = FTS5_TOKEN_COLOCATED;
}
if( rc==SQLITE_OK ){
sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix;
}
}
}else{
/* This happens when parsing a token or quoted phrase that contains
** no token characters at all. (e.g ... MATCH '""'). */
sCtx.pPhrase = sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprPhrase));
}
if( rc==SQLITE_OK ){
@@ -1781,6 +1826,34 @@ static Fts5Colset *fts5ParseColset(
return pNew;
}
/*
** Allocate and return an Fts5Colset object specifying the inverse of
** the colset passed as the second argument. Free the colset passed
** as the second argument before returning.
*/
Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse *pParse, Fts5Colset *p){
Fts5Colset *pRet;
int nCol = pParse->pConfig->nCol;
pRet = (Fts5Colset*)sqlite3Fts5MallocZero(&pParse->rc,
sizeof(Fts5Colset) + sizeof(int)*nCol
);
if( pRet ){
int i;
int iOld = 0;
for(i=0; i<nCol; i++){
if( iOld>=p->nCol || p->aiCol[iOld]!=i ){
pRet->aiCol[pRet->nCol++] = i;
}else{
iOld++;
}
}
}
sqlite3_free(p);
return pRet;
}
Fts5Colset *sqlite3Fts5ParseColset(
Fts5Parse *pParse, /* Store SQLITE_NOMEM here if required */
Fts5Colset *pColset, /* Existing colset object */
@@ -2495,12 +2568,13 @@ static int fts5ExprPopulatePoslistsCb(
UNUSED_PARAM2(iUnused1, iUnused2);
if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE;
if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++;
for(i=0; i<pExpr->nPhrase; i++){
Fts5ExprTerm *pTerm;
if( p->aPopulator[i].bOk==0 ) continue;
for(pTerm=&pExpr->apExprPhrase[i]->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){
int nTerm = strlen(pTerm->zTerm);
int nTerm = (int)strlen(pTerm->zTerm);
if( (nTerm==nToken || (nTerm<nToken && pTerm->bPrefix))
&& memcmp(pTerm->zTerm, pToken, nTerm)==0
){
@@ -2604,17 +2678,6 @@ void sqlite3Fts5ExprCheckPoslists(Fts5Expr *pExpr, i64 iRowid){
fts5ExprCheckPoslists(pExpr->pRoot, iRowid);
}
static void fts5ExprClearEof(Fts5ExprNode *pNode){
int i;
for(i=0; i<pNode->nChild; i++){
fts5ExprClearEof(pNode->apChild[i]);
}
pNode->bEof = 0;
}
void sqlite3Fts5ExprClearEof(Fts5Expr *pExpr){
fts5ExprClearEof(pExpr->pRoot);
}
/*
** This function is only called for detail=columns tables.
*/

4
ext/fts5/fts5_hash.c
View File

@@ -345,11 +345,11 @@ int sqlite3Fts5HashWrite(
if( pHash->eDetail==FTS5_DETAIL_FULL ){
pPtr[p->nData++] = 0x01;
p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol);
p->iCol = iCol;
p->iCol = (i16)iCol;
p->iPos = 0;
}else{
bNew = 1;
p->iCol = iPos = iCol;
p->iCol = (i16)(iPos = iCol);
}
}

252
ext/fts5/fts5_index.c
View File

@@ -304,6 +304,10 @@ struct Fts5Index {
sqlite3_stmt *pIdxDeleter; /* "DELETE FROM %_idx WHERE segid=? */
sqlite3_stmt *pIdxSelect;
int nRead; /* Total number of blocks read */
sqlite3_stmt *pDataVersion;
i64 iStructVersion; /* data_version when pStruct read */
Fts5Structure *pStruct; /* Current db structure (or NULL) */
};
struct Fts5DoclistIter {
@@ -698,7 +702,6 @@ static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){
return pRet;
}
/*
** Release a reference to data record returned by an earlier call to
** fts5DataRead().
@@ -707,6 +710,18 @@ static void fts5DataRelease(Fts5Data *pData){
sqlite3_free(pData);
}
static Fts5Data *fts5LeafRead(Fts5Index *p, i64 iRowid){
Fts5Data *pRet = fts5DataRead(p, iRowid);
if( pRet ){
if( pRet->szLeaf>pRet->nn ){
p->rc = FTS5_CORRUPT;
fts5DataRelease(pRet);
pRet = 0;
}
}
return pRet;
}
static int fts5IndexPrepareStmt(
Fts5Index *p,
sqlite3_stmt **ppStmt,
@@ -959,6 +974,50 @@ static void fts5StructureExtendLevel(
}
}
static Fts5Structure *fts5StructureReadUncached(Fts5Index *p){
Fts5Structure *pRet = 0;
Fts5Config *pConfig = p->pConfig;
int iCookie; /* Configuration cookie */
Fts5Data *pData;
pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID);
if( p->rc==SQLITE_OK ){
/* TODO: Do we need this if the leaf-index is appended? Probably... */
memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING);
p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet);
if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){
p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie);
}
fts5DataRelease(pData);
if( p->rc!=SQLITE_OK ){
fts5StructureRelease(pRet);
pRet = 0;
}
}
return pRet;
}
static i64 fts5IndexDataVersion(Fts5Index *p){
i64 iVersion = 0;
if( p->rc==SQLITE_OK ){
if( p->pDataVersion==0 ){
p->rc = fts5IndexPrepareStmt(p, &p->pDataVersion,
sqlite3_mprintf("PRAGMA %Q.data_version", p->pConfig->zDb)
);
if( p->rc ) return 0;
}
if( SQLITE_ROW==sqlite3_step(p->pDataVersion) ){
iVersion = sqlite3_column_int64(p->pDataVersion, 0);
}
p->rc = sqlite3_reset(p->pDataVersion);
}
return iVersion;
}
/*
** Read, deserialize and return the structure record.
**
@@ -971,26 +1030,49 @@ static void fts5StructureExtendLevel(
** is called, it is a no-op.
*/
static Fts5Structure *fts5StructureRead(Fts5Index *p){
Fts5Config *pConfig = p->pConfig;
Fts5Structure *pRet = 0; /* Object to return */
int iCookie; /* Configuration cookie */
Fts5Data *pData;
pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID);
if( p->rc ) return 0;
/* TODO: Do we need this if the leaf-index is appended? Probably... */
memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING);
p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet);
if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){
p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie);
if( p->pStruct==0 ){
p->iStructVersion = fts5IndexDataVersion(p);
if( p->rc==SQLITE_OK ){
p->pStruct = fts5StructureReadUncached(p);
}
}
fts5DataRelease(pData);
if( p->rc!=SQLITE_OK ){
fts5StructureRelease(pRet);
pRet = 0;
#if 0
else{
Fts5Structure *pTest = fts5StructureReadUncached(p);
if( pTest ){
int i, j;
assert_nc( p->pStruct->nSegment==pTest->nSegment );
assert_nc( p->pStruct->nLevel==pTest->nLevel );
for(i=0; i<pTest->nLevel; i++){
assert_nc( p->pStruct->aLevel[i].nMerge==pTest->aLevel[i].nMerge );
assert_nc( p->pStruct->aLevel[i].nSeg==pTest->aLevel[i].nSeg );
for(j=0; j<pTest->aLevel[i].nSeg; j++){
Fts5StructureSegment *p1 = &pTest->aLevel[i].aSeg[j];
Fts5StructureSegment *p2 = &p->pStruct->aLevel[i].aSeg[j];
assert_nc( p1->iSegid==p2->iSegid );
assert_nc( p1->pgnoFirst==p2->pgnoFirst );
assert_nc( p1->pgnoLast==p2->pgnoLast );
}
}
fts5StructureRelease(pTest);
}
}
#endif
if( p->rc!=SQLITE_OK ) return 0;
assert( p->iStructVersion!=0 );
assert( p->pStruct!=0 );
fts5StructureRef(p->pStruct);
return p->pStruct;
}
static void fts5StructureInvalidate(Fts5Index *p){
if( p->pStruct ){
fts5StructureRelease(p->pStruct);
p->pStruct = 0;
}
return pRet;
}
/*
@@ -1448,7 +1530,7 @@ static void fts5SegIterNextPage(
pIter->pLeaf = pIter->pNextLeaf;
pIter->pNextLeaf = 0;
}else if( pIter->iLeafPgno<=pSeg->pgnoLast ){
pIter->pLeaf = fts5DataRead(p,
pIter->pLeaf = fts5LeafRead(p,
FTS5_SEGMENT_ROWID(pSeg->iSegid, pIter->iLeafPgno)
);
}else{
@@ -1951,14 +2033,13 @@ static void fts5SegIterNext(
if( pLeaf->nn>pLeaf->szLeaf ){
pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32(
&pLeaf->p[pLeaf->szLeaf], pIter->iEndofDoclist
);
);
}
}
else if( pLeaf->nn>pLeaf->szLeaf ){
pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32(
&pLeaf->p[pLeaf->szLeaf], iOff
);
);
pIter->iLeafOffset = iOff;
pIter->iEndofDoclist = iOff;
bNewTerm = 1;
@@ -1992,6 +2073,7 @@ static void fts5SegIterNext(
*/
int nSz;
assert( p->rc==SQLITE_OK );
assert( pIter->iLeafOffset<=pIter->pLeaf->nn );
fts5FastGetVarint32(pIter->pLeaf->p, pIter->iLeafOffset, nSz);
pIter->bDel = (nSz & 0x0001);
pIter->nPos = nSz>>1;
@@ -2198,6 +2280,11 @@ static void fts5LeafSeek(
iTermOff += nKeep;
iOff = iTermOff;
if( iOff>=n ){
p->rc = FTS5_CORRUPT;
return;
}
/* Read the nKeep field of the next term. */
fts5FastGetVarint32(a, iOff, nKeep);
}
@@ -2250,6 +2337,18 @@ static void fts5LeafSeek(
fts5SegIterLoadNPos(p, pIter);
}
static sqlite3_stmt *fts5IdxSelectStmt(Fts5Index *p){
if( p->pIdxSelect==0 ){
Fts5Config *pConfig = p->pConfig;
fts5IndexPrepareStmt(p, &p->pIdxSelect, sqlite3_mprintf(
"SELECT pgno FROM '%q'.'%q_idx' WHERE "
"segid=? AND term<=? ORDER BY term DESC LIMIT 1",
pConfig->zDb, pConfig->zName
));
}
return p->pIdxSelect;
}
/*
** Initialize the object pIter to point to term pTerm/nTerm within segment
** pSeg. If there is no such term in the index, the iterator is set to EOF.
@@ -2267,6 +2366,7 @@ static void fts5SegIterSeekInit(
int iPg = 1;
int bGe = (flags & FTS5INDEX_QUERY_SCAN);
int bDlidx = 0; /* True if there is a doclist-index */
sqlite3_stmt *pIdxSelect = 0;
assert( bGe==0 || (flags & FTS5INDEX_QUERY_DESC)==0 );
assert( pTerm && nTerm );
@@ -2275,23 +2375,16 @@ static void fts5SegIterSeekInit(
/* This block sets stack variable iPg to the leaf page number that may
** contain term (pTerm/nTerm), if it is present in the segment. */
if( p->pIdxSelect==0 ){
Fts5Config *pConfig = p->pConfig;
fts5IndexPrepareStmt(p, &p->pIdxSelect, sqlite3_mprintf(
"SELECT pgno FROM '%q'.'%q_idx' WHERE "
"segid=? AND term<=? ORDER BY term DESC LIMIT 1",
pConfig->zDb, pConfig->zName
));
}
pIdxSelect = fts5IdxSelectStmt(p);
if( p->rc ) return;
sqlite3_bind_int(p->pIdxSelect, 1, pSeg->iSegid);
sqlite3_bind_blob(p->pIdxSelect, 2, pTerm, nTerm, SQLITE_STATIC);
if( SQLITE_ROW==sqlite3_step(p->pIdxSelect) ){
i64 val = sqlite3_column_int(p->pIdxSelect, 0);
sqlite3_bind_int(pIdxSelect, 1, pSeg->iSegid);
sqlite3_bind_blob(pIdxSelect, 2, pTerm, nTerm, SQLITE_STATIC);
if( SQLITE_ROW==sqlite3_step(pIdxSelect) ){
i64 val = sqlite3_column_int(pIdxSelect, 0);
iPg = (int)(val>>1);
bDlidx = (val & 0x0001);
}
p->rc = sqlite3_reset(p->pIdxSelect);
p->rc = sqlite3_reset(pIdxSelect);
if( iPg<pSeg->pgnoFirst ){
iPg = pSeg->pgnoFirst;
@@ -2748,6 +2841,7 @@ static void fts5MultiIterNext(
i64 iFrom /* Advance at least as far as this */
){
int bUseFrom = bFrom;
assert( pIter->base.bEof==0 );
while( p->rc==SQLITE_OK ){
int iFirst = pIter->aFirst[1].iFirst;
int bNewTerm = 0;
@@ -2974,7 +3068,7 @@ static void fts5ChunkIterate(
break;
}else{
pgno++;
pData = fts5DataRead(p, FTS5_SEGMENT_ROWID(pSeg->pSeg->iSegid, pgno));
pData = fts5LeafRead(p, FTS5_SEGMENT_ROWID(pSeg->pSeg->iSegid, pgno));
if( pData==0 ) break;
pChunk = &pData->p[4];
nChunk = MIN(nRem, pData->szLeaf - 4);
@@ -3118,6 +3212,15 @@ static void fts5IterSetOutputs_Nocolset(Fts5Iter *pIter, Fts5SegIter *pSeg){
}
}
/*
** xSetOutputs callback used when the Fts5Colset object has nCol==0 (match
** against no columns at all).
*/
static void fts5IterSetOutputs_ZeroColset(Fts5Iter *pIter, Fts5SegIter *pSeg){
UNUSED_PARAM(pSeg);
pIter->base.nData = 0;
}
/*
** xSetOutputs callback used by detail=col when there is a column filter
** and there are 100 or more columns. Also called as a fallback from
@@ -3167,7 +3270,7 @@ static void fts5IterSetOutputs_Col100(Fts5Iter *pIter, Fts5SegIter *pSeg){
if( aiCol==aiColEnd ) goto setoutputs_col_out;
}
if( *aiCol==iPrev ){
*aOut++ = (iPrev - iPrevOut) + 2;
*aOut++ = (u8)((iPrev - iPrevOut) + 2);
iPrevOut = iPrev;
}
}
@@ -3223,6 +3326,10 @@ static void fts5IterSetOutputCb(int *pRc, Fts5Iter *pIter){
pIter->xSetOutputs = fts5IterSetOutputs_Nocolset;
}
else if( pIter->pColset->nCol==0 ){
pIter->xSetOutputs = fts5IterSetOutputs_ZeroColset;
}
else if( pConfig->eDetail==FTS5_DETAIL_FULL ){
pIter->xSetOutputs = fts5IterSetOutputs_Full;
}
@@ -3453,18 +3560,46 @@ static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){
if( pStruct->nSegment>=FTS5_MAX_SEGMENT ){
p->rc = SQLITE_FULL;
}else{
while( iSegid==0 ){
int iLvl, iSeg;
sqlite3_randomness(sizeof(u32), (void*)&iSegid);
iSegid = iSegid & ((1 << FTS5_DATA_ID_B)-1);
for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
if( iSegid==pStruct->aLevel[iLvl].aSeg[iSeg].iSegid ){
iSegid = 0;
}
/* FTS5_MAX_SEGMENT is currently defined as 2000. So the following
** array is 63 elements, or 252 bytes, in size. */
u32 aUsed[(FTS5_MAX_SEGMENT+31) / 32];
int iLvl, iSeg;
int i;
u32 mask;
memset(aUsed, 0, sizeof(aUsed));
for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
int iId = pStruct->aLevel[iLvl].aSeg[iSeg].iSegid;
if( iId<=FTS5_MAX_SEGMENT ){
aUsed[(iId-1) / 32] |= 1 << ((iId-1) % 32);
}
}
}
for(i=0; aUsed[i]==0xFFFFFFFF; i++);
mask = aUsed[i];
for(iSegid=0; mask & (1 << iSegid); iSegid++);
iSegid += 1 + i*32;
#ifdef SQLITE_DEBUG
for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
assert( iSegid!=pStruct->aLevel[iLvl].aSeg[iSeg].iSegid );
}
}
assert( iSegid>0 && iSegid<=FTS5_MAX_SEGMENT );
{
sqlite3_stmt *pIdxSelect = fts5IdxSelectStmt(p);
if( p->rc==SQLITE_OK ){
u8 aBlob[2] = {0xff, 0xff};
sqlite3_bind_int(pIdxSelect, 1, iSegid);
sqlite3_bind_blob(pIdxSelect, 2, aBlob, 2, SQLITE_STATIC);
assert( sqlite3_step(pIdxSelect)!=SQLITE_ROW );
p->rc = sqlite3_reset(pIdxSelect);
}
}
#endif
}
}
@@ -3710,6 +3845,9 @@ static void fts5WriteFlushLeaf(Fts5Index *p, Fts5SegWriter *pWriter){
Fts5PageWriter *pPage = &pWriter->writer;
i64 iRowid;
static int nCall = 0;
nCall++;
assert( (pPage->pgidx.n==0)==(pWriter->bFirstTermInPage) );
/* Set the szLeaf header field. */
@@ -3909,7 +4047,9 @@ static void fts5WriteFinish(
fts5WriteFlushLeaf(p, pWriter);
}
*pnLeaf = pLeaf->pgno-1;
fts5WriteFlushBtree(p, pWriter);
if( pLeaf->pgno>1 ){
fts5WriteFlushBtree(p, pWriter);
}
}
fts5BufferFree(&pLeaf->term);
fts5BufferFree(&pLeaf->buf);
@@ -4328,6 +4468,7 @@ static void fts5FlushOneHash(Fts5Index *p){
** for the new level-0 segment. */
pStruct = fts5StructureRead(p);
iSegid = fts5AllocateSegid(p, pStruct);
fts5StructureInvalidate(p);
if( iSegid ){
const int pgsz = p->pConfig->pgsz;
@@ -4512,7 +4653,7 @@ static Fts5Structure *fts5IndexOptimizeStruct(
if( pNew ){
Fts5StructureLevel *pLvl;
int nByte = nSeg * sizeof(Fts5StructureSegment);
nByte = nSeg * sizeof(Fts5StructureSegment);
pNew->nLevel = pStruct->nLevel+1;
pNew->nRef = 1;
pNew->nWriteCounter = pStruct->nWriteCounter;
@@ -4547,6 +4688,7 @@ int sqlite3Fts5IndexOptimize(Fts5Index *p){
assert( p->rc==SQLITE_OK );
fts5IndexFlush(p);
pStruct = fts5StructureRead(p);
fts5StructureInvalidate(p);
if( pStruct ){
pNew = fts5IndexOptimizeStruct(p, pStruct);
@@ -4577,6 +4719,7 @@ int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){
Fts5Structure *pStruct = fts5StructureRead(p);
if( pStruct ){
int nMin = p->pConfig->nUsermerge;
fts5StructureInvalidate(p);
if( nMerge<0 ){
Fts5Structure *pNew = fts5IndexOptimizeStruct(p, pStruct);
fts5StructureRelease(pStruct);
@@ -5004,6 +5147,7 @@ int sqlite3Fts5IndexSync(Fts5Index *p, int bCommit){
int sqlite3Fts5IndexRollback(Fts5Index *p){
fts5CloseReader(p);
fts5IndexDiscardData(p);
fts5StructureInvalidate(p);
/* assert( p->rc==SQLITE_OK ); */
return SQLITE_OK;
}
@@ -5015,6 +5159,7 @@ int sqlite3Fts5IndexRollback(Fts5Index *p){
*/
int sqlite3Fts5IndexReinit(Fts5Index *p){
Fts5Structure s;
fts5StructureInvalidate(p);
memset(&s, 0, sizeof(Fts5Structure));
fts5DataWrite(p, FTS5_AVERAGES_ROWID, (const u8*)"", 0);
fts5StructureWrite(p, &s);
@@ -5073,11 +5218,13 @@ int sqlite3Fts5IndexClose(Fts5Index *p){
int rc = SQLITE_OK;
if( p ){
assert( p->pReader==0 );
fts5StructureInvalidate(p);
sqlite3_finalize(p->pWriter);
sqlite3_finalize(p->pDeleter);
sqlite3_finalize(p->pIdxWriter);
sqlite3_finalize(p->pIdxDeleter);
sqlite3_finalize(p->pIdxSelect);
sqlite3_finalize(p->pDataVersion);
sqlite3Fts5HashFree(p->pHash);
sqlite3_free(p->zDataTbl);
sqlite3_free(p);
@@ -5683,7 +5830,7 @@ static void fts5IndexIntegrityCheckSegment(
** ignore this b-tree entry. Otherwise, load it into memory. */
if( iIdxLeaf<pSeg->pgnoFirst ) continue;
iRow = FTS5_SEGMENT_ROWID(pSeg->iSegid, iIdxLeaf);
pLeaf = fts5DataRead(p, iRow);
pLeaf = fts5LeafRead(p, iRow);
if( pLeaf==0 ) break;
/* Check that the leaf contains at least one term, and that it is equal
@@ -6333,3 +6480,12 @@ int sqlite3Fts5IndexInit(sqlite3 *db){
}
return rc;
}
int sqlite3Fts5IndexReset(Fts5Index *p){
assert( p->pStruct==0 || p->iStructVersion!=0 );
if( fts5IndexDataVersion(p)!=p->iStructVersion ){
fts5StructureInvalidate(p);
}
return fts5IndexReturn(p);
}

40
ext/fts5/fts5_main.c
View File

@@ -597,27 +597,38 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
return SQLITE_OK;
}
static int fts5NewTransaction(Fts5Table *pTab){
Fts5Cursor *pCsr;
for(pCsr=pTab->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){
if( pCsr->base.pVtab==(sqlite3_vtab*)pTab ) return SQLITE_OK;
}
return sqlite3Fts5StorageReset(pTab->pStorage);
}
/*
** Implementation of xOpen method.
*/
static int fts5OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
Fts5Table *pTab = (Fts5Table*)pVTab;
Fts5Config *pConfig = pTab->pConfig;
Fts5Cursor *pCsr; /* New cursor object */
Fts5Cursor *pCsr = 0; /* New cursor object */
int nByte; /* Bytes of space to allocate */
int rc = SQLITE_OK; /* Return code */
int rc; /* Return code */
nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int);
pCsr = (Fts5Cursor*)sqlite3_malloc(nByte);
if( pCsr ){
Fts5Global *pGlobal = pTab->pGlobal;
memset(pCsr, 0, nByte);
pCsr->aColumnSize = (int*)&pCsr[1];
pCsr->pNext = pGlobal->pCsr;
pGlobal->pCsr = pCsr;
pCsr->iCsrId = ++pGlobal->iNextId;
}else{
rc = SQLITE_NOMEM;
rc = fts5NewTransaction(pTab);
if( rc==SQLITE_OK ){
nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int);
pCsr = (Fts5Cursor*)sqlite3_malloc(nByte);
if( pCsr ){
Fts5Global *pGlobal = pTab->pGlobal;
memset(pCsr, 0, nByte);
pCsr->aColumnSize = (int*)&pCsr[1];
pCsr->pNext = pGlobal->pCsr;
pGlobal->pCsr = pCsr;
pCsr->iCsrId = ++pGlobal->iNextId;
}else{
rc = SQLITE_NOMEM;
}
}
*ppCsr = (sqlite3_vtab_cursor*)pCsr;
return rc;
@@ -1175,7 +1186,6 @@ static int fts5FilterMethod(
pCsr->ePlan = FTS5_PLAN_SOURCE;
pCsr->pExpr = pTab->pSortCsr->pExpr;
rc = fts5CursorFirst(pTab, pCsr, bDesc);
sqlite3Fts5ExprClearEof(pCsr->pExpr);
}else if( pMatch ){
const char *zExpr = (const char*)sqlite3_value_text(apVal[0]);
if( zExpr==0 ) zExpr = "";
@@ -1578,8 +1588,8 @@ static int fts5SyncMethod(sqlite3_vtab *pVtab){
** Implementation of xBegin() method.
*/
static int fts5BeginMethod(sqlite3_vtab *pVtab){
UNUSED_PARAM(pVtab); /* Call below is a no-op for NDEBUG builds */
fts5CheckTransactionState((Fts5Table*)pVtab, FTS5_BEGIN, 0);
fts5NewTransaction((Fts5Table*)pVtab);
return SQLITE_OK;
}

40
ext/fts5/fts5_storage.c
View File

@@ -145,6 +145,7 @@ static int fts5StorageGetStmt(
}
*ppStmt = p->aStmt[eStmt];
sqlite3_reset(*ppStmt);
return rc;
}
@@ -246,7 +247,11 @@ int sqlite3Fts5CreateTable(
char *zErr = 0;
rc = fts5ExecPrintf(pConfig->db, &zErr, "CREATE TABLE %Q.'%q_%q'(%s)%s",
pConfig->zDb, pConfig->zName, zPost, zDefn, bWithout?" WITHOUT ROWID":""
pConfig->zDb, pConfig->zName, zPost, zDefn,
#ifndef SQLITE_FTS5_NO_WITHOUT_ROWID
bWithout?" WITHOUT ROWID":
#endif
""
);
if( zErr ){
*pzErr = sqlite3_mprintf(
@@ -368,6 +373,7 @@ static int fts5StorageInsertCallback(
Fts5InsertCtx *pCtx = (Fts5InsertCtx*)pContext;
Fts5Index *pIdx = pCtx->pStorage->pIndex;
UNUSED_PARAM2(iUnused1, iUnused2);
if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE;
if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){
pCtx->szCol++;
}
@@ -539,11 +545,6 @@ int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){
}
}
/* Write the averages record */
if( rc==SQLITE_OK ){
rc = fts5StorageSaveTotals(p);
}
return rc;
}
@@ -639,6 +640,10 @@ int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge){
return sqlite3Fts5IndexMerge(p->pIndex, nMerge);
}
int sqlite3Fts5StorageReset(Fts5Storage *p){
return sqlite3Fts5IndexReset(p->pIndex);
}
/*
** Allocate a new rowid. This is used for "external content" tables when
** a NULL value is inserted into the rowid column. The new rowid is allocated
@@ -743,11 +748,6 @@ int sqlite3Fts5StorageIndexInsert(
}
sqlite3_free(buf.p);
/* Write the averages record */
if( rc==SQLITE_OK ){
rc = fts5StorageSaveTotals(p);
}
return rc;
}
@@ -810,6 +810,7 @@ static int fts5StorageIntegrityCallback(
int iCol;
UNUSED_PARAM2(iUnused1, iUnused2);
if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE;
if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){
pCtx->szCol++;
@@ -1081,12 +1082,17 @@ int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow){
** Flush any data currently held in-memory to disk.
*/
int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit){
if( bCommit && p->bTotalsValid ){
int rc = fts5StorageSaveTotals(p);
p->bTotalsValid = 0;
if( rc!=SQLITE_OK ) return rc;
int rc = SQLITE_OK;
i64 iLastRowid = sqlite3_last_insert_rowid(p->pConfig->db);
if( p->bTotalsValid ){
rc = fts5StorageSaveTotals(p);
if( bCommit ) p->bTotalsValid = 0;
}
return sqlite3Fts5IndexSync(p->pIndex, bCommit);
if( rc==SQLITE_OK ){
rc = sqlite3Fts5IndexSync(p->pIndex, bCommit);
}
sqlite3_set_last_insert_rowid(p->pConfig->db, iLastRowid);
return rc;
}
int sqlite3Fts5StorageRollback(Fts5Storage *p){
@@ -1121,5 +1127,3 @@ int sqlite3Fts5StorageConfigValue(
}
return rc;
}

33
ext/fts5/fts5_tcl.c
View File

@@ -14,7 +14,14 @@
#ifdef SQLITE_TEST
#include <tcl.h>
#if defined(INCLUDE_SQLITE_TCL_H)
# include "sqlite_tcl.h"
#else
# include "tcl.h"
# ifndef SQLITE_TCLAPI
# define SQLITE_TCLAPI
# endif
#endif
#ifdef SQLITE_ENABLE_FTS5
@@ -78,7 +85,7 @@ static int f5tResultToErrorCode(const char *zRes){
return SQLITE_ERROR;
}
static int f5tDbAndApi(
static int SQLITE_TCLAPI f5tDbAndApi(
Tcl_Interp *interp,
Tcl_Obj *pObj,
sqlite3 **ppDb,
@@ -164,7 +171,7 @@ static int xTokenizeCb(
return rc;
}
static int xF5tApi(void*, Tcl_Interp*, int, Tcl_Obj *CONST []);
static int SQLITE_TCLAPI xF5tApi(void*, Tcl_Interp*, int, Tcl_Obj *CONST []);
static int xQueryPhraseCb(
const Fts5ExtensionApi *pApi,
@@ -209,7 +216,7 @@ static void xSetAuxdataDestructor(void *p){
**
** Description...
*/
static int xF5tApi(
static int SQLITE_TCLAPI xF5tApi(
void * clientData,
Tcl_Interp *interp,
int objc,
@@ -602,7 +609,7 @@ static void xF5tDestroy(void *pCtx){
**
** Description...
*/
static int f5tCreateFunction(
static int SQLITE_TCLAPI f5tCreateFunction(
void * clientData,
Tcl_Interp *interp,
int objc,
@@ -672,7 +679,7 @@ static int xTokenizeCb2(
**
** Description...
*/
static int f5tTokenize(
static int SQLITE_TCLAPI f5tTokenize(
void * clientData,
Tcl_Interp *interp,
int objc,
@@ -878,7 +885,7 @@ static int f5tTokenizerTokenize(
/*
** sqlite3_fts5_token ?-colocated? TEXT START END
*/
static int f5tTokenizerReturn(
static int SQLITE_TCLAPI f5tTokenizerReturn(
void * clientData,
Tcl_Interp *interp,
int objc,
@@ -949,7 +956,7 @@ static void f5tDelTokenizer(void *pCtx){
** SCRIPT2 should invoke the [sqlite3_fts5_token] command once for each
** token within the tokenized text.
*/
static int f5tCreateTokenizer(
static int SQLITE_TCLAPI f5tCreateTokenizer(
ClientData clientData,
Tcl_Interp *interp,
int objc,
@@ -992,7 +999,7 @@ static int f5tCreateTokenizer(
return TCL_OK;
}
static void xF5tFree(ClientData clientData){
static void SQLITE_TCLAPI xF5tFree(ClientData clientData){
ckfree(clientData);
}
@@ -1001,7 +1008,7 @@ static void xF5tFree(ClientData clientData){
**
** Set or clear the global "may-be-corrupt" flag. Return the old value.
*/
static int f5tMayBeCorrupt(
static int SQLITE_TCLAPI f5tMayBeCorrupt(
void * clientData,
Tcl_Interp *interp,
int objc,
@@ -1033,7 +1040,7 @@ static unsigned int f5t_fts5HashKey(int nSlot, const char *p, int n){
return (h % nSlot);
}
static int f5tTokenHash(
static int SQLITE_TCLAPI f5tTokenHash(
void * clientData,
Tcl_Interp *interp,
int objc,
@@ -1058,7 +1065,7 @@ static int f5tTokenHash(
return TCL_OK;
}
static int f5tRegisterMatchinfo(
static int SQLITE_TCLAPI f5tRegisterMatchinfo(
void * clientData,
Tcl_Interp *interp,
int objc,
@@ -1083,7 +1090,7 @@ static int f5tRegisterMatchinfo(
return TCL_OK;
}
static int f5tRegisterTok(
static int SQLITE_TCLAPI f5tRegisterTok(
void * clientData,
Tcl_Interp *interp,
int objc,

13
ext/fts5/fts5_vocab.c
View File

@@ -279,8 +279,19 @@ static int fts5VocabBestIndexMethod(
}
}
pInfo->idxNum = idxNum;
/* This virtual table always delivers results in ascending order of
** the "term" column (column 0). So if the user has requested this
** specifically - "ORDER BY term" or "ORDER BY term ASC" - set the
** sqlite3_index_info.orderByConsumed flag to tell the core the results
** are already in sorted order. */
if( pInfo->nOrderBy==1
&& pInfo->aOrderBy[0].iColumn==0
&& pInfo->aOrderBy[0].desc==0
){
pInfo->orderByConsumed = 1;
}
pInfo->idxNum = idxNum;
return SQLITE_OK;
}

8
ext/fts5/fts5parse.y
View File

@@ -120,10 +120,17 @@ cnearset(A) ::= colset(X) COLON nearset(Y). {
%type colsetlist {Fts5Colset*}
%destructor colsetlist { sqlite3_free($$); }
colset(A) ::= MINUS LCP colsetlist(X) RCP. {
A = sqlite3Fts5ParseColsetInvert(pParse, X);
}
colset(A) ::= LCP colsetlist(X) RCP. { A = X; }
colset(A) ::= STRING(X). {
A = sqlite3Fts5ParseColset(pParse, 0, &X);
}
colset(A) ::= MINUS STRING(X). {
A = sqlite3Fts5ParseColset(pParse, 0, &X);
A = sqlite3Fts5ParseColsetInvert(pParse, A);
}
colsetlist(A) ::= colsetlist(Y) STRING(X). {
A = sqlite3Fts5ParseColset(pParse, Y, &X); }
@@ -131,7 +138,6 @@ colsetlist(A) ::= STRING(X). {
A = sqlite3Fts5ParseColset(pParse, 0, &X);
}
%type nearset {Fts5ExprNearset*}
%type nearphrases {Fts5ExprNearset*}
%destructor nearset { sqlite3Fts5ParseNearsetFree($$); }

9
ext/fts5/test/fts5aa.test
View File

@@ -432,9 +432,16 @@ proc funk {} {
}
db func funk funk
# This test case corrupts the structure record within the first invocation
# of function funk(). Which used to cause the bm25() function to throw an
# exception. But since bm25() can now used the cached structure record,
# it never sees the corruption introduced by funk() and so the following
# statement no longer fails.
#
do_catchsql_test 16.2 {
SELECT funk(), bm25(n1), funk() FROM n1 WHERE n1 MATCH 'a+b+c+d'
} {1 {SQL logic error or missing database}}
} {0 {{} -1e-06 {}}}
# {1 {SQL logic error or missing database}}
#-------------------------------------------------------------------------
#

91
ext/fts5/test/fts5af.test
View File

@@ -72,42 +72,56 @@ foreach {tn doc res} {
2.2 {o X o o o o o o} {o [X] o o o o o...}
2.3 {o o X o o o o o} {o o [X] o o o o...}
2.4 {o o o X o o o o} {o o o [X] o o o...}
2.5 {o o o o X o o o} {...o o o [X] o o o}
2.6 {o o o o o X o o} {...o o o o [X] o o}
2.7 {o o o o o o X o} {...o o o o o [X] o}
2.5 {o o o o X o o o} {o o o o [X] o o...}
2.6 {o o o o o X o o} {o o o o o [X] o...}
2.7 {o o o o o o X o} {o o o o o o [X]...}
2.8 {o o o o o o o X} {...o o o o o o [X]}
2.9 {o o o o o o o X o} {...o o o o o [X] o}
2.10 {o o o o o o o X o o} {...o o o o [X] o o}
2.11 {o o o o o o o X o o o} {...o o o [X] o o o}
2.12 {o o o o o o o X o o o o} {...o o o [X] o o o...}
3.1 {X o o o o o o o o} {[X] o o o o o o...}
3.2 {o X o o o o o o o} {o [X] o o o o o...}
3.3 {o o X o o o o o o} {o o [X] o o o o...}
3.4 {o o o X o o o o o} {o o o [X] o o o...}
3.5 {o o o o X o o o o} {...o o o [X] o o o...}
3.6 {o o o o o X o o o} {...o o o [X] o o o}
3.7 {o o o o o o X o o} {...o o o o [X] o o}
3.8 {o o o o o o o X o} {...o o o o o [X] o}
3.9 {o o o o o o o o X} {...o o o o o o [X]}
3.5 {o o o o o o o X o o o o} {...o o o [X] o o o...}
3.6 {o o o o o o o o X o o o} {...o o o [X] o o o}
3.7 {o o o o o o o o o X o o} {...o o o o [X] o o}
3.8 {o o o o o o o o o o X o} {...o o o o o [X] o}
3.9 {o o o o o o o o o o o X} {...o o o o o o [X]}
4.1 {X o o o o o X o o} {[X] o o o o o [X]...}
4.2 {o X o o o o o X o} {...[X] o o o o o [X]...}
4.3 {o o X o o o o o X} {...[X] o o o o o [X]}
4.2 {o o o o o o o X o o o o o X o} {...[X] o o o o o [X]...}
4.3 {o o o o o o o o X o o o o o X} {...[X] o o o o o [X]}
5.1 {X o o o o X o o o} {[X] o o o o [X] o...}
5.2 {o X o o o o X o o} {...[X] o o o o [X] o...}
5.3 {o o X o o o o X o} {...[X] o o o o [X] o}
5.4 {o o o X o o o o X} {...o [X] o o o o [X]}
5.2 {o o o o o o o X o o o o X o o} {...[X] o o o o [X] o...}
5.3 {o o o o o o o o X o o o o X o} {...[X] o o o o [X] o}
5.4 {o o o o o o o o o X o o o o X} {...o [X] o o o o [X]}
6.1 {X o o o X o o o} {[X] o o o [X] o o...}
6.2 {o X o o o X o o o} {o [X] o o o [X] o...}
6.3 {o o X o o o X o o} {...o [X] o o o [X] o...}
6.4 {o o o X o o o X o} {...o [X] o o o [X] o}
6.5 {o o o o X o o o X} {...o o [X] o o o [X]}
6.3 {o o o o o o o X o o o X o o} {...o [X] o o o [X] o...}
6.4 {o o o o o o o o X o o o X o} {...o [X] o o o [X] o}
6.5 {o o o o o o o o o X o o o X} {...o o [X] o o o [X]}
7.1 {X o o X o o o o o} {[X] o o [X] o o o...}
7.2 {o X o o X o o o o} {o [X] o o [X] o o...}
7.3 {o o X o o X o o o} {...o [X] o o [X] o o...}
7.4 {o o o X o o X o o} {...o [X] o o [X] o o}
7.5 {o o o o X o o X o} {...o o [X] o o [X] o}
7.6 {o o o o o X o o X} {...o o o [X] o o [X]}
7.3 {o o o o o o o X o o X o o o} {...o [X] o o [X] o o...}
7.4 {o o o o o o o o X o o X o o} {...o [X] o o [X] o o}
7.5 {o o o o o o o o o X o o X o} {...o o [X] o o [X] o}
7.6 {o o o o o o o o o o X o o X} {...o o o [X] o o [X]}
8.1 {o o o o o o o o o X o o o o o o o o o o o o o o o o X X X o o o}
{...o o [X] [X] [X] o o...}
8.2 {o o o o o o o. o o X o o o o o o o o o o o o o o o o X X X o o o}
{...o o [X] o o o o...}
8.3 {o o o o X o o o o o o o o o o o o o o o o o o o o o X X X o o o}
{o o o o [X] o o...}
} {
do_snippet_test 1.$tn $doc X $res
}
@@ -124,24 +138,43 @@ if {[detail_is_full]} {
2.1 {X Y o o o o o o} {[X Y] o o o o o...}
2.2 {o X Y o o o o o} {o [X Y] o o o o...}
2.3 {o o X Y o o o o} {o o [X Y] o o o...}
2.4 {o o o X Y o o o} {...o o [X Y] o o o}
2.5 {o o o o X Y o o} {...o o o [X Y] o o}
2.6 {o o o o o X Y o} {...o o o o [X Y] o}
2.7 {o o o o o o X Y} {...o o o o o [X Y]}
2.4 {o o o o o o o X Y o o o} {...o o [X Y] o o o}
2.5 {o o o o o o o o X Y o o} {...o o o [X Y] o o}
2.6 {o o o o o o o o o X Y o} {...o o o o [X Y] o}
2.7 {o o o o o o o o o o X Y} {...o o o o o [X Y]}
3.1 {X Y o o o o o o o} {[X Y] o o o o o...}
3.2 {o X Y o o o o o o} {o [X Y] o o o o...}
3.3 {o o X Y o o o o o} {o o [X Y] o o o...}
3.4 {o o o X Y o o o o} {...o o [X Y] o o o...}
3.5 {o o o o X Y o o o} {...o o [X Y] o o o}
3.6 {o o o o o X Y o o} {...o o o [X Y] o o}
3.7 {o o o o o o X Y o} {...o o o o [X Y] o}
3.8 {o o o o o o o X Y} {...o o o o o [X Y]}
3.4 {o o o o o o o X Y o o o o} {...o o [X Y] o o o...}
3.5 {o o o o o o o o X Y o o o} {...o o [X Y] o o o}
3.6 {o o o o o o o o o X Y o o} {...o o o [X Y] o o}
3.7 {o o o o o o o o o o X Y o} {...o o o o [X Y] o}
3.8 {o o o o o o o o o o o X Y} {...o o o o o [X Y]}
} {
do_snippet_test 2.$tn $doc "X + Y" $res
}
}
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE x1 USING fts5(a, b);
INSERT INTO x1 VALUES('xyz', '1 2 3 4 5 6 7 8 9 10 11 12 13');
SELECT snippet(x1, 1, '[', ']', '...', 5) FROM x1('xyz');
} {
{1 2 3 4 5...}
}
do_execsql_test 5.0 {
CREATE VIRTUAL TABLE p1 USING fts5(a, b);
INSERT INTO p1 VALUES(
'x a a a a a a a a a a',
'a a a a a a a a a a a a a a a a a a a x'
);
}
do_execsql_test 5.1 {
SELECT snippet(p1, 0, '[', ']', '...', 6) FROM p1('x');
} {{[x] a a a a a...}}
} ;# foreach_detail_mode
finish_test

32
ext/fts5/test/fts5aux.test
View File

@@ -246,5 +246,37 @@ foreach {tn lRow res} {
} $res
}
#-------------------------------------------------------------------------
# Test the built-in bm25() demo.
#
reset_db
do_execsql_test 9.1 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b);
INSERT INTO t1 VALUES('a', NULL); -- 1
INSERT INTO t1 VALUES('a', NULL); -- 2
INSERT INTO t1 VALUES('a', NULL); -- 3
INSERT INTO t1 VALUES('a', NULL); -- 4
INSERT INTO t1 VALUES('a', NULL); -- 5
INSERT INTO t1 VALUES('a', NULL); -- 6
INSERT INTO t1 VALUES('a', NULL); -- 7
INSERT INTO t1 VALUES('a', NULL); -- 8
INSERT INTO t1 VALUES(NULL, 'a a b'); -- 9
INSERT INTO t1 VALUES(NULL, 'b b a'); -- 10
}
do_execsql_test 9.2 {
SELECT rowid FROM t1('a AND b') ORDER BY rank;
} {
10 9
}
do_execsql_test 9.3 {
SELECT rowid FROM t1('b:a AND b:b') ORDER BY rank;
} {
9 10
}
finish_test

59
ext/fts5/test/fts5colset.test Normal file
View File

@@ -0,0 +1,59 @@
# 2016 August 10
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library. The
# focus of this script is testing the FTS5 module.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5colset
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
foreach_detail_mode $::testprefix {
if {[detail_is_none]} continue
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, d, detail=%DETAIL%);
INSERT INTO t1 VALUES('a', 'b', 'c', 'd'); -- 1
INSERT INTO t1 VALUES('d', 'a', 'b', 'c'); -- 2
INSERT INTO t1 VALUES('c', 'd', 'a', 'b'); -- 3
INSERT INTO t1 VALUES('b', 'c', 'd', 'a'); -- 4
}
foreach {tn q res} {
1 "a" {1 2 3 4}
2 "{a} : a" {1}
3 "-{a} : a" {2 3 4}
4 "- {a c} : a" {2 4}
5 " - {d d c} : a" {1 2}
6 "- {d c b a} : a" {}
7 "-{\"a\"} : b" {1 2 3}
8 "- c : a" {1 2 4}
9 "-c : a" {1 2 4}
10 "-\"c\" : a" {1 2 4}
} {
breakpoint
do_execsql_test 1.$tn {
SELECT rowid FROM t1($q)
} $res
}
}
finish_test

6
ext/fts5/test/fts5corrupt2.test
View File

@@ -37,7 +37,7 @@ do_execsql_test 1.0 {
}
set mask [expr 31 << 31]
if 1 {
if 0 {
# Test 1:
#
@@ -84,6 +84,8 @@ foreach {tno stmt} {
}
}
}
# Using the same database as the 1.* tests.
#
# Run N-1 tests, where N is the number of bytes in the rightmost leaf page
@@ -212,8 +214,6 @@ foreach {tn nCut} {
# do_test 4.$tn.x { expr $nCorrupt>0 } 1
}
}
set doc [string repeat "A B C " 1000]
do_execsql_test 5.0 {
CREATE VIRTUAL TABLE x5 USING fts5(tt);

4
ext/fts5/test/fts5corrupt3.test
View File

@@ -179,6 +179,10 @@ for {set i 1} {1} {incr i} {
if {$end<=$i} break
lset var end [expr $end - $i]
set struct [binary format c* $var]
db close
sqlite3 db test.db
db eval {
BEGIN;
UPDATE t1_data SET block = $struct WHERE id=10;

67
ext/fts5/test/fts5determin.test Normal file
View File

@@ -0,0 +1,67 @@
# 2016 March 21
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library. The
# focus of this script is testing the FTS5 module.
#
# Specifically, that the fts5 module is deterministic. At one point, when
# segment ids were allocated using sqlite3_randomness(), this was not the
# case.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5aa
return_if_no_fts5
proc do_determin_test {tn} {
uplevel [list
do_execsql_test $tn {
SELECT (SELECT md5sum(id, block) FROM t1_data)==
(SELECT md5sum(id, block) FROM t2_data),
(SELECT md5sum(id, block) FROM t1_data)==
(SELECT md5sum(id, block) FROM t3_data)
} {1 1}
]
}
foreach_detail_mode $::testprefix {
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix="1 2", detail=%DETAIL%);
CREATE VIRTUAL TABLE t2 USING fts5(a, b, prefix="1 2", detail=%DETAIL%);
CREATE VIRTUAL TABLE t3 USING fts5(a, b, prefix="1 2", detail=%DETAIL%);
}
do_test 1.1 {
foreach t {t1 t2 t3} {
execsql [string map [list TBL $t] {
INSERT INTO TBL VALUES('a b c', 'd e f');
INSERT INTO TBL VALUES('c1 c2 c3', 'c1 c2 c3');
INSERT INTO TBL VALUES('xyzxyzxyz', 'xyzxyzxyz');
}]
}
} {}
do_determin_test 1.2
do_test 1.3 {
foreach t {t1 t2 t3} {
execsql [string map [list TBL $t] {
INSERT INTO TBL(TBL) VALUES('optimize');
}]
}
} {}
do_determin_test 1.4
}
finish_test

4
ext/fts5/test/fts5dlidx.test
View File

@@ -178,7 +178,7 @@ do_execsql_test 3.2 {
ORDER BY rowid DESC;
} {16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1}
do_execsql_test 3.2 {
do_execsql_test 3.3 {
INSERT INTO abc(abc) VALUES('integrity-check');
INSERT INTO abc(abc) VALUES('optimize');
INSERT INTO abc(abc) VALUES('integrity-check');
@@ -187,7 +187,7 @@ do_execsql_test 3.2 {
set v [lindex $vocab 0]
set i 0
foreach v $vocab {
do_execsql_test 3.3.[incr i] {
do_execsql_test 3.4.[incr i] {
SELECT rowid FROM abc WHERE abc MATCH $v
} {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16}
}

17
ext/fts5/test/fts5eb.test
View File

@@ -63,6 +63,23 @@ do_catchsql_test 2.1 {
SELECT fts5_expr_tcl()
} {1 {wrong number of arguments to function fts5_expr_tcl}}
do_execsql_test 3.0 {
CREATE VIRTUAL TABLE e1 USING fts5(text, tokenize = 'porter unicode61');
INSERT INTO e1 VALUES ("just a few words with a / inside");
}
do_execsql_test 3.1 {
SELECT rowid, bm25(e1) FROM e1 WHERE e1 MATCH '"just"' ORDER BY rank;
} {1 -1e-06}
do_execsql_test 3.2 {
SELECT rowid FROM e1 WHERE e1 MATCH '"/" OR "just"'
} 1
do_execsql_test 3.3 {
SELECT rowid, bm25(e1) FROM e1 WHERE e1 MATCH '"/" OR "just"' ORDER BY rank;
} {1 -1e-06}
finish_test

2
ext/fts5/test/fts5fault4.test
View File

@@ -86,7 +86,7 @@ set ::res [db eval {SELECT rowid, x1 FROM x1 WHERE x1 MATCH '*reads'}]
do_faultsim_test 4 -faults oom-* -body {
db eval {SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads'}
} -test {
faultsim_test_result {0 {0 {} 4}}
faultsim_test_result {0 {0 {} 3}}
}
#-------------------------------------------------------------------------

28
ext/fts5/test/fts5faultB.test
View File

@@ -78,6 +78,34 @@ do_faultsim_test 2.4 -faults oom* -body {
faultsim_test_result {0 {{3 2} {2 3}}}
}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 3.0 {
CREATE VIRTUAL TABLE x1 USING fts5(z);
}
do_faultsim_test 3.1 -faults oom* -body {
execsql {
SELECT rowid FROM x1('c') WHERE rowid>1;
}
} -test {
faultsim_test_result {0 {}}
}
do_execsql_test 3.2 {
INSERT INTO x1 VALUES('a b c');
INSERT INTO x1 VALUES('b c d');
INSERT INTO x1 VALUES('c d e');
INSERT INTO x1 VALUES('d e f');
}
do_faultsim_test 3.3 -faults oom* -body {
execsql {
SELECT rowid FROM x1('c') WHERE rowid>1;
}
} -test {
faultsim_test_result {0 {2 3}}
}
finish_test

87
ext/fts5/test/fts5faultD.test Normal file
View File

@@ -0,0 +1,87 @@
# 2016 February 2
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
#
# This file is focused on OOM errors.
#
source [file join [file dirname [info script]] fts5_common.tcl]
source $testdir/malloc_common.tcl
set testprefix fts5faultA
# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
foreach_detail_mode $testprefix {
if {"%DETAIL%"=="none"} continue
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE o1 USING fts5(a, b, c, detail=%DETAIL%);
INSERT INTO o1(o1, rank) VALUES('pgsz', 32);
WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<60 )
INSERT INTO o1 SELECT 'A', 'B', 'C' FROM s;
WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<60 )
INSERT INTO o1 SELECT 'C', 'A', 'B' FROM s;
WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<60 )
INSERT INTO o1 SELECT 'B', 'C', 'A' FROM s;
}
do_faultsim_test 1 -faults int* -prep {
sqlite3 db test.db
} -body {
execsql { SELECT count(*) FROM o1('a') }
} -test {
faultsim_test_result {0 180} {1 {vtable constructor failed: o1}}
}
do_faultsim_test 2 -faults int* -prep {
sqlite3 db test.db
} -body {
execsql { SELECT * FROM o1('a:a AND {b c}:b') ORDER BY rank }
expr 1
} -test {
faultsim_test_result {0 1} {1 {vtable constructor failed: o1}}
}
do_faultsim_test 3 -faults int* -prep {
sqlite3 db test.db
} -body {
execsql { SELECT * FROM o1('{b c}:b NOT a:a') ORDER BY rank }
expr 1
} -test {
faultsim_test_result {0 1} {1 {vtable constructor failed: o1}}
}
do_faultsim_test 4 -faults int* -prep {
sqlite3 db test.db
} -body {
execsql { SELECT * FROM o1('b:b OR a:a') }
expr 1
} -test {
faultsim_test_result {0 1} {1 {vtable constructor failed: o1}}
}
do_faultsim_test 5 -faults int* -prep {
sqlite3 db test.db
} -body {
execsql { SELECT count(*) FROM o1('c:b') }
expr 1
} -test {
faultsim_test_result {0 1} {1 {vtable constructor failed: o1}}
}
}
finish_test

73
ext/fts5/test/fts5lastrowid.test Normal file
View File

@@ -0,0 +1,73 @@
# 2017 Feb 27
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# Tests of the last_insert_rowid functionality with fts5.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5lastrowid
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(str);
}
do_execsql_test 1.1 {
INSERT INTO t1 VALUES('one string');
INSERT INTO t1 VALUES('two string');
INSERT INTO t1 VALUES('three string');
SELECT last_insert_rowid();
} {3}
do_execsql_test 1.2 {
BEGIN;
INSERT INTO t1 VALUES('one string');
INSERT INTO t1 VALUES('two string');
INSERT INTO t1 VALUES('three string');
COMMIT;
SELECT last_insert_rowid();
} {6}
do_execsql_test 1.3 {
INSERT INTO t1(rowid, str) VALUES(-22, 'some more text');
SELECT last_insert_rowid();
} {-22}
do_execsql_test 1.4 {
BEGIN;
INSERT INTO t1(rowid, str) VALUES(45, 'some more text');
INSERT INTO t1(rowid, str) VALUES(46, 'some more text');
INSERT INTO t1(rowid, str) VALUES(222, 'some more text');
SELECT last_insert_rowid();
COMMIT;
SELECT last_insert_rowid();
} {222 222}
do_execsql_test 1.5 {
CREATE TABLE x1(x);
INSERT INTO x1 VALUES('john'), ('paul'), ('george'), ('ringo');
INSERT INTO t1 SELECT x FROM x1;
SELECT last_insert_rowid();
} {226}
do_execsql_test 1.6 {
INSERT INTO t1(rowid, str) SELECT rowid+10, x FROM x1;
SELECT last_insert_rowid();
} {14}
finish_test

48
ext/fts5/test/fts5multiclient.test Normal file
View File

@@ -0,0 +1,48 @@
# 2016 March 17
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
#
source [file join [file dirname [info script]] fts5_common.tcl]
source $testdir/lock_common.tcl
set testprefix fts5multiclient
return_if_no_fts5
foreach_detail_mode $testprefix {
do_multiclient_test tn {
do_test 1.$tn.1 {
sql1 { CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%) }
sql1 { INSERT INTO t1 VALUES('a b c') }
sql2 { SELECT rowid FROM t1('b') }
} {1}
do_test 1.$tn.2 {
sql2 { INSERT INTO t1 VALUES('a b c') }
sql1 { SELECT rowid FROM t1('b') }
} {1 2}
do_test 1.$tn.3 {
sql2 { INSERT INTO t1 VALUES('a b c') }
sql1 { SELECT rowid FROM t1('b') }
} {1 2 3}
do_test 1.$tn.4 {
sql2 { INSERT INTO t1 VALUES('a b c') }
sql1 { INSERT INTO t1 VALUES('a b c') }
sql3 { INSERT INTO t1(t1) VALUES('integrity-check') }
} {}
};# do_multiclient_test
};# foreach_detail_mode
finish_test

2
ext/fts5/test/fts5prefix.test
View File

@@ -9,7 +9,7 @@
#
#***********************************************************************
#
# This file containst tests focused on prefix indexes.
# This file contains tests focused on prefix indexes.
#
source [file join [file dirname [info script]] fts5_common.tcl]

54
ext/fts5/test/fts5rank.test
View File

@@ -91,7 +91,61 @@ do_test 2.7 {
} {1 3 2}
#--------------------------------------------------------------------------
# At one point there was a problem with queries such as:
#
# ... MATCH 'x OR y' ORDER BY rank;
#
# if there were zero occurrences of token 'y' in the dataset. The
# following tests verify that that problem has been addressed.
#
foreach_detail_mode $::testprefix {
do_execsql_test 3.1.0 {
CREATE VIRTUAL TABLE y1 USING fts5(z, detail=%DETAIL%);
INSERT INTO y1 VALUES('test xyz');
INSERT INTO y1 VALUES('test test xyz test');
INSERT INTO y1 VALUES('test test xyz');
}
do_execsql_test 3.1.1 {
SELECT rowid FROM y1('test OR tset');
} {1 2 3}
do_execsql_test 3.1.2 {
SELECT rowid FROM y1('test OR tset') ORDER BY bm25(y1)
} {2 3 1}
do_execsql_test 3.1.3 {
SELECT rowid FROM y1('test OR tset') ORDER BY +rank
} {2 3 1}
do_execsql_test 3.1.4 {
SELECT rowid FROM y1('test OR tset') ORDER BY rank
} {2 3 1}
do_execsql_test 3.1.5 {
SELECT rowid FROM y1('test OR xyz') ORDER BY rank
} {3 2 1}
do_execsql_test 3.2.1 {
CREATE VIRTUAL TABLE z1 USING fts5(a, detail=%DETAIL%);
INSERT INTO z1 VALUES('wrinkle in time');
SELECT * FROM z1 WHERE z1 MATCH 'wrinkle in time OR a wrinkle in time';
} {{wrinkle in time}}
}
do_execsql_test 4.1 {
DROP TABLE IF EXISTS VTest;
CREATE virtual TABLE VTest USING FTS5(
Title, AUthor, tokenize ='porter unicode61 remove_diacritics 1',
columnsize='1', detail=full
);
INSERT INTO VTest (Title, Author) VALUES ('wrinkle in time', 'Bill Smith');
SELECT * FROM VTest WHERE
VTest MATCH 'wrinkle in time OR a wrinkle in time' ORDER BY rank;
} {{wrinkle in time} {Bill Smith}}

28
ext/fts5/test/fts5simple.test
View File

@@ -265,7 +265,7 @@ do_test 11.0 {
execsql "
INSERT INTO t4 VALUES('a b c \x1A');
INSERT INTO t4 VALUES('a b c d\x1A');
INSERT INTO t4 VALUES('a b c \x1Ad');
INSERT INTO t4 VALUES('a b c \x1Ag');
INSERT INTO t4 VALUES('a b c d');
"
} {}
@@ -340,7 +340,7 @@ do_test 14.2 {
#-------------------------------------------------------------------------
db func rnddoc fts5_rnddoc
do_execsql_test 4.0 {
do_execsql_test 14.3 {
CREATE VIRTUAL TABLE x1 USING fts5(x);
INSERT INTO x1(x1, rank) VALUES('pgsz', 32);
@@ -348,9 +348,9 @@ do_execsql_test 4.0 {
INSERT INTO x1 SELECT rnddoc(5) FROM ii;
}
do_execsql_test 4.1 {
do_execsql_test 14.4 {
SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads'
} {0 {} 4}
} {0 {} 3}
#-------------------------------------------------------------------------
reset_db
@@ -448,4 +448,24 @@ do_execsql_test 20.2 {
INSERT INTO x1(x1) VALUES('integrity-check');
} {}
#-------------------------------------------------------------------------
reset_db
set doc "a b [string repeat x 100000]"
do_execsql_test 21.0 {
CREATE VIRTUAL TABLE x1 USING fts5(x);
INSERT INTO x1(rowid, x) VALUES(11111, $doc);
INSERT INTO x1(rowid, x) VALUES(11112, $doc);
}
do_execsql_test 21.1 {
INSERT INTO x1(x1) VALUES('integrity-check');
}
do_execsql_test 21.2 {
SELECT rowid FROM x1($doc);
} {11111 11112}
do_execsql_test 21.3 {
DELETE FROM x1 WHERE rowid=11111;
INSERT INTO x1(x1) VALUES('integrity-check');
SELECT rowid FROM x1($doc);
} {11112}
finish_test

35
ext/fts5/test/fts5simple2.test
View File

@@ -332,6 +332,41 @@ do_execsql_test 16.0 {
DELETE FROM t2;
}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 17.0 {
CREATE VIRTUAL TABLE t2 USING fts5(x, y);
BEGIN;
INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb');
INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb');
INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb');
COMMIT;
}
do_execsql_test 17.1 { SELECT * FROM t2('y:a*') WHERE rowid BETWEEN 10 AND 20 }
do_execsql_test 17.2 {
BEGIN;
INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb');
SELECT * FROM t2('y:a*') WHERE rowid BETWEEN 10 AND 20 ;
}
do_execsql_test 17.3 {
COMMIT
}
reset_db
do_execsql_test 17.4 {
CREATE VIRTUAL TABLE t2 USING fts5(x, y);
BEGIN;
INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb');
INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb');
SELECT * FROM t2('y:a*') WHERE rowid>66;
}
do_execsql_test 17.5 { SELECT * FROM t2('x:b* OR y:a*') }
do_execsql_test 17.5 { COMMIT ; SELECT * FROM t2('x:b* OR y:a*') }
do_execsql_test 17.6 {
SELECT * FROM t2('x:b* OR y:a*') WHERE rowid>55
}
#db eval {SELECT rowid, fts5_decode_none(rowid, block) aS r FROM t2_data} {puts $r}
finish_test

34
ext/fts5/test/fts5simple3.test
View File

@@ -80,6 +80,40 @@ do_execsql_test 3.0 {
SELECT * FROM x3('x OR y OR z');
}
#-------------------------------------------------------------------------
# Test that a crash occuring when the second or subsequent tokens in a
# phrase matched zero rows has been fixed.
#
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x);
INSERT INTO t1 VALUES('ab');
INSERT INTO t1 VALUES('cd');
INSERT INTO t1 VALUES('ab cd');
INSERT INTO t1 VALUES('ab cdXXX');
INSERT INTO t1 VALUES('abXXX cd');
}
do_execsql_test 4.1 {
SELECT * FROM t1('"ab cd" OR "ab cd" *');
} {{ab cd} {ab cdXXX}}
do_execsql_test 4.2 {
SELECT * FROM t1('"xy zz" OR "ab cd" *');
} {{ab cd} {ab cdXXX}}
do_execsql_test 4.3 {
SELECT * FROM t1('"xy zz" OR "xy zz" *');
}
do_execsql_test 4.4 {
SELECT * FROM t1('"ab cd" OR "xy zz" *');
} {{ab cd}}
do_execsql_test 4.5 {
CREATE VIRTUAL TABLE t2 USING fts5(x);
INSERT INTO t2 VALUES('ab');
INSERT INTO t2 VALUES('cd');
INSERT INTO t2 VALUES('ef');
}
do_execsql_test 4.6 {
SELECT * FROM t2('ab + xyz');
}
finish_test

2
ext/fts5/test/fts5synonym.test
View File

@@ -152,7 +152,7 @@ foreach {tn expr res} {
1 {abc} {"abc"}
2 {one} {"one"|"i"|"1"}
3 {3} {"3"|"iii"|"three"}
4 {3*} {"3"|"iii"|"three" *}
4 {3*} {"3" *}
} {
do_execsql_test 4.1.$tn {
SELECT fts5_expr($expr, 'tokenize=tclnum')

39
ext/fts5/test/fts5tokenizer.test
View File

@@ -262,5 +262,44 @@ do_execsql_test 8.3 {
brown dog fox jump lazi over quick the
}
#-------------------------------------------------------------------------
# Check that the FTS5_TOKENIZE_PREFIX flag is passed to the tokenizer
# implementation.
#
reset_db
proc tcl_create {args} { return "tcl_tokenize" }
sqlite3_fts5_create_tokenizer db tcl tcl_create
set ::flags [list]
proc tcl_tokenize {tflags text} {
lappend ::flags $tflags
foreach {w iStart iEnd} [fts5_tokenize_split $text] {
sqlite3_fts5_token $w $iStart $iEnd
}
}
do_execsql_test 9.1.1 {
CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=tcl);
INSERT INTO t1 VALUES('abc');
INSERT INTO t1 VALUES('xyz');
} {}
do_test 9.1.2 { set ::flags } {document document}
set ::flags [list]
do_execsql_test 9.2.1 { SELECT * FROM t1('abc'); } {abc}
do_test 9.2.2 { set ::flags } {query}
set ::flags [list]
do_execsql_test 9.3.1 { SELECT * FROM t1('ab*'); } {abc}
do_test 9.3.2 { set ::flags } {prefixquery}
set ::flags [list]
do_execsql_test 9.4.1 { SELECT * FROM t1('"abc xyz" *'); } {}
do_test 9.4.2 { set ::flags } {prefixquery}
set ::flags [list]
do_execsql_test 9.5.1 { SELECT * FROM t1('"abc xyz*"'); } {}
do_test 9.5.2 { set ::flags } {query}
finish_test

8
ext/fts5/test/fts5unicode2.test
View File

@@ -160,12 +160,12 @@ foreach {tn query snippet} {
the maximum x value.
}
4 "rollback" {
...[ROLLBACK]. Instead, the pending statement
will return SQLITE_ABORT upon next access after the [ROLLBACK].
Pending statements no longer block [ROLLBACK]. Instead, the pending
statement will return SQLITE_ABORT upon...
}
5 "rOllback" {
...[ROLLBACK]. Instead, the pending statement
will return SQLITE_ABORT upon next access after the [ROLLBACK].
Pending statements no longer block [ROLLBACK]. Instead, the pending
statement will return SQLITE_ABORT upon...
}
6 "lang*" {
Added support for the FTS4 [languageid] option.

38
ext/fts5/test/fts5vocab.test
View File

@@ -442,8 +442,44 @@ if {[detail_is_none]} {
}
sqlite3_fts5_may_be_corrupt 0
}
#-------------------------------------------------------------------------
# Test that both "ORDER BY term" and "ORDER BY term DESC" work.
#
reset_db
do_execsql_test 9.1 {
CREATE VIRTUAL TABLE x1 USING fts5(x);
INSERT INTO x1 VALUES('def ABC ghi');
INSERT INTO x1 VALUES('DEF abc GHI');
}
do_execsql_test 9.2 {
CREATE VIRTUAL TABLE rrr USING fts5vocab(x1, row);
SELECT * FROM rrr
} {
abc 2 2 def 2 2 ghi 2 2
}
do_execsql_test 9.3 {
SELECT * FROM rrr ORDER BY term ASC
} {
abc 2 2 def 2 2 ghi 2 2
}
do_execsql_test 9.4 {
SELECT * FROM rrr ORDER BY term DESC
} {
ghi 2 2 def 2 2 abc 2 2
}
do_test 9.5 {
set e2 [db eval { EXPLAIN SELECT * FROM rrr ORDER BY term ASC }]
expr [lsearch $e2 SorterSort]<0
} 1
do_test 9.6 {
set e2 [db eval { EXPLAIN SELECT * FROM rrr ORDER BY term DESC }]
expr [lsearch $e2 SorterSort]<0
} 0
finish_test

5
ext/fts5/tool/fts5txt2db.tcl
View File

@@ -17,6 +17,7 @@ proc process_cmdline {} {
{detail "full" "Fts5 detail mode to use"}
{repeat 1 "Load each file this many times"}
{prefix "" "Fts prefix= option"}
{trans 1 "True to use a transaction"}
database
file...
} {
@@ -214,7 +215,7 @@ foreach c [lrange $cols 1 end] {
}
append sql ")"
db eval BEGIN
if {$A(trans)} { db eval BEGIN }
while {$i < $N} {
foreach c $cols s $A(colsize) {
set R($c) [lrange $tokens $i [expr $i+$s-1]]
@@ -222,7 +223,7 @@ db eval BEGIN
}
db eval $sql
}
db eval COMMIT
if {$A(trans)} { db eval COMMIT }

74
ext/icu/icu.c
View File

@@ -349,20 +349,22 @@ static void icuRegexpFunc(sqlite3_context *p, int nArg, sqlite3_value **apArg){
** of upper() or lower().
**
** lower('I', 'en_us') -> 'i'
** lower('I', 'tr_tr') -> 'ı' (small dotless i)
** lower('I', 'tr_tr') -> '\u131' (small dotless i)
**
** http://www.icu-project.org/userguide/posix.html#case_mappings
*/
static void icuCaseFunc16(sqlite3_context *p, int nArg, sqlite3_value **apArg){
const UChar *zInput;
UChar *zOutput = 0;
int nInput;
int nOut;
const UChar *zInput; /* Pointer to input string */
UChar *zOutput = 0; /* Pointer to output buffer */
int nInput; /* Size of utf-16 input string in bytes */
int nOut; /* Size of output buffer in bytes */
int cnt;
int bToUpper; /* True for toupper(), false for tolower() */
UErrorCode status;
const char *zLocale = 0;
assert(nArg==1 || nArg==2);
bToUpper = (sqlite3_user_data(p)!=0);
if( nArg==2 ){
zLocale = (const char *)sqlite3_value_text(apArg[1]);
}
@@ -386,19 +388,23 @@ static void icuCaseFunc16(sqlite3_context *p, int nArg, sqlite3_value **apArg){
}
zOutput = zNew;
status = U_ZERO_ERROR;
if( sqlite3_user_data(p) ){
if( bToUpper ){
nOut = 2*u_strToUpper(zOutput,nOut/2,zInput,nInput/2,zLocale,&status);
}else{
nOut = 2*u_strToLower(zOutput,nOut/2,zInput,nInput/2,zLocale,&status);
}
if( !U_SUCCESS(status) ){
if( status==U_BUFFER_OVERFLOW_ERROR ) continue;
icuFunctionError(p,
sqlite3_user_data(p) ? "u_strToUpper" : "u_strToLower", status);
return;
if( U_SUCCESS(status) ){
sqlite3_result_text16(p, zOutput, nOut, xFree);
}else if( status==U_BUFFER_OVERFLOW_ERROR ){
assert( cnt==0 );
continue;
}else{
icuFunctionError(p, bToUpper ? "u_strToUpper" : "u_strToLower", status);
}
return;
}
sqlite3_result_text16(p, zOutput, nOut, xFree);
assert( 0 ); /* Unreachable */
}
/*
@@ -487,38 +493,36 @@ static void icuLoadCollation(
** Register the ICU extension functions with database db.
*/
int sqlite3IcuInit(sqlite3 *db){
struct IcuScalar {
static const struct IcuScalar {
const char *zName; /* Function name */
int nArg; /* Number of arguments */
int enc; /* Optimal text encoding */
void *pContext; /* sqlite3_user_data() context */
unsigned char nArg; /* Number of arguments */
unsigned short enc; /* Optimal text encoding */
unsigned char iContext; /* sqlite3_user_data() context */
void (*xFunc)(sqlite3_context*,int,sqlite3_value**);
} scalars[] = {
{"regexp", 2, SQLITE_ANY, 0, icuRegexpFunc},
{"lower", 1, SQLITE_UTF16, 0, icuCaseFunc16},
{"lower", 2, SQLITE_UTF16, 0, icuCaseFunc16},
{"upper", 1, SQLITE_UTF16, (void*)1, icuCaseFunc16},
{"upper", 2, SQLITE_UTF16, (void*)1, icuCaseFunc16},
{"lower", 1, SQLITE_UTF8, 0, icuCaseFunc16},
{"lower", 2, SQLITE_UTF8, 0, icuCaseFunc16},
{"upper", 1, SQLITE_UTF8, (void*)1, icuCaseFunc16},
{"upper", 2, SQLITE_UTF8, (void*)1, icuCaseFunc16},
{"like", 2, SQLITE_UTF8, 0, icuLikeFunc},
{"like", 3, SQLITE_UTF8, 0, icuLikeFunc},
{"icu_load_collation", 2, SQLITE_UTF8, (void*)db, icuLoadCollation},
{"icu_load_collation", 2, SQLITE_UTF8, 1, icuLoadCollation},
{"regexp", 2, SQLITE_ANY|SQLITE_DETERMINISTIC, 0, icuRegexpFunc},
{"lower", 1, SQLITE_UTF16|SQLITE_DETERMINISTIC, 0, icuCaseFunc16},
{"lower", 2, SQLITE_UTF16|SQLITE_DETERMINISTIC, 0, icuCaseFunc16},
{"upper", 1, SQLITE_UTF16|SQLITE_DETERMINISTIC, 1, icuCaseFunc16},
{"upper", 2, SQLITE_UTF16|SQLITE_DETERMINISTIC, 1, icuCaseFunc16},
{"lower", 1, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuCaseFunc16},
{"lower", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuCaseFunc16},
{"upper", 1, SQLITE_UTF8|SQLITE_DETERMINISTIC, 1, icuCaseFunc16},
{"upper", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 1, icuCaseFunc16},
{"like", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuLikeFunc},
{"like", 3, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuLikeFunc},
};
int rc = SQLITE_OK;
int i;
for(i=0; rc==SQLITE_OK && i<(int)(sizeof(scalars)/sizeof(scalars[0])); i++){
struct IcuScalar *p = &scalars[i];
const struct IcuScalar *p = &scalars[i];
rc = sqlite3_create_function(
db, p->zName, p->nArg, p->enc, p->pContext, p->xFunc, 0, 0
db, p->zName, p->nArg, p->enc,
p->iContext ? (void*)db : (void*)0,
p->xFunc, 0, 0
);
}

40
ext/misc/README.md Normal file
View File

@@ -0,0 +1,40 @@
## Miscellaneous Extensions
This folder contains a collection of smaller loadable extensions.
See <https://www.sqlite.org/loadext.html> for instructions on how
to compile and use loadable extensions.
Each extension in this folder is implemented in a single file of C code.
Each source file contains a description in its header comment. See the
header comments for details about each extension. Additional notes are
as follows:
* **carray.c** &mdash; This module implements the
[carray](https://www.sqlite.org/carray.html) table-valued function.
It is a good example of how to go about implementing a custom
[table-valued function](https://www.sqlite.org/vtab.html#tabfunc2).
* **dbdump.c** &mdash; This is not actually a loadable extension, but
rather a library that implements an approximate equivalent to the
".dump" command of the
[command-line shell](https://www.sqlite.org/cli.html).
* **memvfs.c** &mdash; This file implements a custom
[VFS](https://www.sqlite.org/vfs.html) that stores an entire database
file in a single block of RAM. It serves as a good example of how
to implement a simple custom VFS.
* **rot13.c** &mdash; This file implements the very simple rot13()
substitution function. This file makes a good template for implementing
new custom SQL functions for SQLite.
* **series.c** &mdash; This is an implementation of the
"generate_series" [virtual table](https://www.sqlite.org/vtab.html).
It can make a good template for new custom virtual table implementations.
* **shathree.c** &mdash; An implementation of the sha3() and
sha3_query() SQL functions. The file is named "shathree.c" instead
of "sha3.c" because the default entry point names in SQLite are based
on the source filename with digits removed, so if we used the name
"sha3.c" then the entry point would conflict with the prior "sha1.c"
extension.

8
ext/misc/amatch.c
View File

@@ -625,10 +625,10 @@ static int amatchLoadOneRule(
}else{
memset(pRule, 0, sizeof(*pRule));
pRule->zFrom = &pRule->zTo[nTo+1];
pRule->nFrom = nFrom;
pRule->nFrom = (amatch_len)nFrom;
memcpy(pRule->zFrom, zFrom, nFrom+1);
memcpy(pRule->zTo, zTo, nTo+1);
pRule->nTo = nTo;
pRule->nTo = (amatch_len)nTo;
pRule->rCost = rCost;
pRule->iLang = (int)iLang;
}
@@ -1081,7 +1081,7 @@ static void amatchAddWord(
pWord->rCost = rCost;
pWord->iSeq = pCur->nWord++;
amatchWriteCost(pWord);
pWord->nMatch = nMatch;
pWord->nMatch = (short)nMatch;
pWord->pNext = pCur->pAllWords;
pCur->pAllWords = pWord;
pWord->sCost.zKey = pWord->zCost;
@@ -1162,7 +1162,7 @@ static int amatchNext(sqlite3_vtab_cursor *cur){
#endif
nWord = (int)strlen(pWord->zWord+2);
if( nWord+20>nBuf ){
nBuf = nWord+100;
nBuf = (char)(nWord+100);
zBuf = sqlite3_realloc(zBuf, nBuf);
if( zBuf==0 ) return SQLITE_NOMEM;
}

364
ext/misc/carray.c Normal file
View File

@@ -0,0 +1,364 @@
/*
** 2016-06-29
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
**
** This file demonstrates how to create a table-valued-function that
** returns the values in a C-language array.
** Examples:
**
** SELECT * FROM carray($ptr,5)
**
** The query above returns 5 integers contained in a C-language array
** at the address $ptr. $ptr is a pointer to the array of integers that
** has been cast to an integer.
**
** There is an optional third parameter to determine the datatype of
** the C-language array. Allowed values of the third parameter are
** 'int32', 'int64', 'double', 'char*'. Example:
**
** SELECT * FROM carray($ptr,10,'char*');
**
** HOW IT WORKS
**
** The carray "function" is really a virtual table with the
** following schema:
**
** CREATE TABLE carray(
** value,
** pointer HIDDEN,
** count HIDDEN,
** ctype TEXT HIDDEN
** );
**
** If the hidden columns "pointer" and "count" are unconstrained, then
** the virtual table has no rows. Otherwise, the virtual table interprets
** the integer value of "pointer" as a pointer to the array and "count"
** as the number of elements in the array. The virtual table steps through
** the array, element by element.
*/
#include "sqlite3ext.h"
SQLITE_EXTENSION_INIT1
#include <assert.h>
#include <string.h>
#ifndef SQLITE_OMIT_VIRTUALTABLE
/*
** Allowed datatypes
*/
#define CARRAY_INT32 0
#define CARRAY_INT64 1
#define CARRAY_DOUBLE 2
#define CARRAY_TEXT 3
/*
** Names of types
*/
static const char *azType[] = { "int32", "int64", "double", "char*" };
/* carray_cursor is a subclass of sqlite3_vtab_cursor which will
** serve as the underlying representation of a cursor that scans
** over rows of the result
*/
typedef struct carray_cursor carray_cursor;
struct carray_cursor {
sqlite3_vtab_cursor base; /* Base class - must be first */
sqlite3_int64 iRowid; /* The rowid */
sqlite3_int64 iPtr; /* Pointer to array of values */
sqlite3_int64 iCnt; /* Number of integers in the array */
unsigned char eType; /* One of the CARRAY_type values */
};
/*
** The carrayConnect() method is invoked to create a new
** carray_vtab that describes the carray virtual table.
**
** Think of this routine as the constructor for carray_vtab objects.
**
** All this routine needs to do is:
**
** (1) Allocate the carray_vtab object and initialize all fields.
**
** (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the
** result set of queries against carray will look like.
*/
static int carrayConnect(
sqlite3 *db,
void *pAux,
int argc, const char *const*argv,
sqlite3_vtab **ppVtab,
char **pzErr
){
sqlite3_vtab *pNew;
int rc;
/* Column numbers */
#define CARRAY_COLUMN_VALUE 0
#define CARRAY_COLUMN_POINTER 1
#define CARRAY_COLUMN_COUNT 2
#define CARRAY_COLUMN_CTYPE 3
rc = sqlite3_declare_vtab(db,
"CREATE TABLE x(value,pointer hidden,count hidden,ctype hidden)");
if( rc==SQLITE_OK ){
pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) );
if( pNew==0 ) return SQLITE_NOMEM;
memset(pNew, 0, sizeof(*pNew));
}
return rc;
}
/*
** This method is the destructor for carray_cursor objects.
*/
static int carrayDisconnect(sqlite3_vtab *pVtab){
sqlite3_free(pVtab);
return SQLITE_OK;
}
/*
** Constructor for a new carray_cursor object.
*/
static int carrayOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
carray_cursor *pCur;
pCur = sqlite3_malloc( sizeof(*pCur) );
if( pCur==0 ) return SQLITE_NOMEM;
memset(pCur, 0, sizeof(*pCur));
*ppCursor = &pCur->base;
return SQLITE_OK;
}
/*
** Destructor for a carray_cursor.
*/
static int carrayClose(sqlite3_vtab_cursor *cur){
sqlite3_free(cur);
return SQLITE_OK;
}
/*
** Advance a carray_cursor to its next row of output.
*/
static int carrayNext(sqlite3_vtab_cursor *cur){
carray_cursor *pCur = (carray_cursor*)cur;
pCur->iRowid++;
return SQLITE_OK;
}
/*
** Return values of columns for the row at which the carray_cursor
** is currently pointing.
*/
static int carrayColumn(
sqlite3_vtab_cursor *cur, /* The cursor */
sqlite3_context *ctx, /* First argument to sqlite3_result_...() */
int i /* Which column to return */
){
carray_cursor *pCur = (carray_cursor*)cur;
sqlite3_int64 x = 0;
switch( i ){
case CARRAY_COLUMN_POINTER: x = pCur->iPtr; break;
case CARRAY_COLUMN_COUNT: x = pCur->iCnt; break;
case CARRAY_COLUMN_CTYPE: {
sqlite3_result_text(ctx, azType[pCur->eType], -1, SQLITE_STATIC);
return SQLITE_OK;
}
default: {
switch( pCur->eType ){
case CARRAY_INT32: {
int *p = (int*)pCur->iPtr;
sqlite3_result_int(ctx, p[pCur->iRowid-1]);
return SQLITE_OK;
}
case CARRAY_INT64: {
sqlite3_int64 *p = (sqlite3_int64*)pCur->iPtr;
sqlite3_result_int64(ctx, p[pCur->iRowid-1]);
return SQLITE_OK;
}
case CARRAY_DOUBLE: {
double *p = (double*)pCur->iPtr;
sqlite3_result_double(ctx, p[pCur->iRowid-1]);
return SQLITE_OK;
}
case CARRAY_TEXT: {
const char **p = (const char**)pCur->iPtr;
sqlite3_result_text(ctx, p[pCur->iRowid-1], -1, SQLITE_TRANSIENT);
return SQLITE_OK;
}
}
}
}
sqlite3_result_int64(ctx, x);
return SQLITE_OK;
}
/*
** Return the rowid for the current row. In this implementation, the
** rowid is the same as the output value.
*/
static int carrayRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
carray_cursor *pCur = (carray_cursor*)cur;
*pRowid = pCur->iRowid;
return SQLITE_OK;
}
/*
** Return TRUE if the cursor has been moved off of the last
** row of output.
*/
static int carrayEof(sqlite3_vtab_cursor *cur){
carray_cursor *pCur = (carray_cursor*)cur;
return pCur->iRowid>pCur->iCnt;
}
/*
** This method is called to "rewind" the carray_cursor object back
** to the first row of output.
*/
static int carrayFilter(
sqlite3_vtab_cursor *pVtabCursor,
int idxNum, const char *idxStr,
int argc, sqlite3_value **argv
){
carray_cursor *pCur = (carray_cursor *)pVtabCursor;
if( idxNum ){
pCur->iPtr = sqlite3_value_int64(argv[0]);
pCur->iCnt = sqlite3_value_int64(argv[1]);
if( idxNum<3 ){
pCur->eType = CARRAY_INT32;
}else{
unsigned char i;
const char *zType = (const char*)sqlite3_value_text(argv[2]);
for(i=0; i<sizeof(azType)/sizeof(azType[0]); i++){
if( sqlite3_stricmp(zType, azType[i])==0 ) break;
}
if( i>=sizeof(azType)/sizeof(azType[0]) ){
pVtabCursor->pVtab->zErrMsg = sqlite3_mprintf(
"unknown datatype: %Q", zType);
return SQLITE_ERROR;
}else{
pCur->eType = i;
}
}
}else{
pCur->iPtr = 0;
pCur->iCnt = 0;
}
pCur->iRowid = 1;
return SQLITE_OK;
}
/*
** SQLite will invoke this method one or more times while planning a query
** that uses the carray virtual table. This routine needs to create
** a query plan for each invocation and compute an estimated cost for that
** plan.
**
** In this implementation idxNum is used to represent the
** query plan. idxStr is unused.
**
** idxNum is 2 if the pointer= and count= constraints exist,
** 3 if the ctype= constraint also exists, and is 0 otherwise.
** If idxNum is 0, then carray becomes an empty table.
*/
static int carrayBestIndex(
sqlite3_vtab *tab,
sqlite3_index_info *pIdxInfo
){
int i; /* Loop over constraints */
int ptrIdx = -1; /* Index of the pointer= constraint, or -1 if none */
int cntIdx = -1; /* Index of the count= constraint, or -1 if none */
int ctypeIdx = -1; /* Index of the ctype= constraint, or -1 if none */
const struct sqlite3_index_constraint *pConstraint;
pConstraint = pIdxInfo->aConstraint;
for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){
if( pConstraint->usable==0 ) continue;
if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;
switch( pConstraint->iColumn ){
case CARRAY_COLUMN_POINTER:
ptrIdx = i;
break;
case CARRAY_COLUMN_COUNT:
cntIdx = i;
break;
case CARRAY_COLUMN_CTYPE:
ctypeIdx = i;
break;
}
}
if( ptrIdx>=0 && cntIdx>=0 ){
pIdxInfo->aConstraintUsage[ptrIdx].argvIndex = 1;
pIdxInfo->aConstraintUsage[ptrIdx].omit = 1;
pIdxInfo->aConstraintUsage[cntIdx].argvIndex = 2;
pIdxInfo->aConstraintUsage[cntIdx].omit = 1;
pIdxInfo->estimatedCost = (double)1;
pIdxInfo->estimatedRows = 100;
pIdxInfo->idxNum = 2;
if( ctypeIdx>=0 ){
pIdxInfo->aConstraintUsage[ctypeIdx].argvIndex = 3;
pIdxInfo->aConstraintUsage[ctypeIdx].omit = 1;
pIdxInfo->idxNum = 3;
}
}else{
pIdxInfo->estimatedCost = (double)2147483647;
pIdxInfo->estimatedRows = 2147483647;
pIdxInfo->idxNum = 0;
}
return SQLITE_OK;
}
/*
** This following structure defines all the methods for the
** carray virtual table.
*/
static sqlite3_module carrayModule = {
0, /* iVersion */
0, /* xCreate */
carrayConnect, /* xConnect */
carrayBestIndex, /* xBestIndex */
carrayDisconnect, /* xDisconnect */
0, /* xDestroy */
carrayOpen, /* xOpen - open a cursor */
carrayClose, /* xClose - close a cursor */
carrayFilter, /* xFilter - configure scan constraints */
carrayNext, /* xNext - advance a cursor */
carrayEof, /* xEof - check for end of scan */
carrayColumn, /* xColumn - read data */
carrayRowid, /* xRowid - read data */
0, /* xUpdate */
0, /* xBegin */
0, /* xSync */
0, /* xCommit */
0, /* xRollback */
0, /* xFindMethod */
0, /* xRename */
};
#endif /* SQLITE_OMIT_VIRTUALTABLE */
#ifdef _WIN32
__declspec(dllexport)
#endif
int sqlite3_carray_init(
sqlite3 *db,
char **pzErrMsg,
const sqlite3_api_routines *pApi
){
int rc = SQLITE_OK;
SQLITE_EXTENSION_INIT2(pApi);
#ifndef SQLITE_OMIT_VIRTUALTABLE
rc = sqlite3_create_module(db, "carray", &carrayModule, 0);
#endif
return rc;
}

869
ext/misc/csv.c Normal file
View File

@@ -0,0 +1,869 @@
/*
** 2016-05-28
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** This file contains the implementation of an SQLite virtual table for
** reading CSV files.
**
** Usage:
**
** .load ./csv
** CREATE VIRTUAL TABLE temp.csv USING csv(filename=FILENAME);
** SELECT * FROM csv;
**
** The columns are named "c1", "c2", "c3", ... by default. But the
** application can define its own CREATE TABLE statement as an additional
** parameter. For example:
**
** CREATE VIRTUAL TABLE temp.csv2 USING csv(
** filename = "../http.log",
** schema = "CREATE TABLE x(date,ipaddr,url,referrer,userAgent)"
** );
**
** Instead of specifying a file, the text of the CSV can be loaded using
** the data= parameter.
**
** If the columns=N parameter is supplied, then the CSV file is assumed to have
** N columns. If the columns parameter is omitted, the CSV file is opened
** as soon as the virtual table is constructed and the first row of the CSV
** is read in order to count the tables.
**
** Some extra debugging features (used for testing virtual tables) are available
** if this module is compiled with -DSQLITE_TEST.
*/
#include <sqlite3ext.h>
SQLITE_EXTENSION_INIT1
#include <string.h>
#include <stdlib.h>
#include <assert.h>
#include <stdarg.h>
#include <ctype.h>
#include <stdio.h>
#ifndef SQLITE_OMIT_VIRTUALTABLE
/*
** A macro to hint to the compiler that a function should not be
** inlined.
*/
#if defined(__GNUC__)
# define CSV_NOINLINE __attribute__((noinline))
#elif defined(_MSC_VER) && _MSC_VER>=1310
# define CSV_NOINLINE __declspec(noinline)
#else
# define CSV_NOINLINE
#endif
/* Max size of the error message in a CsvReader */
#define CSV_MXERR 200
/* Size of the CsvReader input buffer */
#define CSV_INBUFSZ 1024
/* A context object used when read a CSV file. */
typedef struct CsvReader CsvReader;
struct CsvReader {
FILE *in; /* Read the CSV text from this input stream */
char *z; /* Accumulated text for a field */
int n; /* Number of bytes in z */
int nAlloc; /* Space allocated for z[] */
int nLine; /* Current line number */
char cTerm; /* Character that terminated the most recent field */
size_t iIn; /* Next unread character in the input buffer */
size_t nIn; /* Number of characters in the input buffer */
char *zIn; /* The input buffer */
char zErr[CSV_MXERR]; /* Error message */
};
/* Initialize a CsvReader object */
static void csv_reader_init(CsvReader *p){
p->in = 0;
p->z = 0;
p->n = 0;
p->nAlloc = 0;
p->nLine = 0;
p->nIn = 0;
p->zIn = 0;
p->zErr[0] = 0;
}
/* Close and reset a CsvReader object */
static void csv_reader_reset(CsvReader *p){
if( p->in ){
fclose(p->in);
sqlite3_free(p->zIn);
}
sqlite3_free(p->z);
csv_reader_init(p);
}
/* Report an error on a CsvReader */
static void csv_errmsg(CsvReader *p, const char *zFormat, ...){
va_list ap;
va_start(ap, zFormat);
sqlite3_vsnprintf(CSV_MXERR, p->zErr, zFormat, ap);
va_end(ap);
}
/* Open the file associated with a CsvReader
** Return the number of errors.
*/
static int csv_reader_open(
CsvReader *p, /* The reader to open */
const char *zFilename, /* Read from this filename */
const char *zData /* ... or use this data */
){
if( zFilename ){
p->zIn = sqlite3_malloc( CSV_INBUFSZ );
if( p->zIn==0 ){
csv_errmsg(p, "out of memory");
return 1;
}
p->in = fopen(zFilename, "rb");
if( p->in==0 ){
csv_reader_reset(p);
csv_errmsg(p, "cannot open '%s' for reading", zFilename);
return 1;
}
}else{
assert( p->in==0 );
p->zIn = (char*)zData;
p->nIn = strlen(zData);
}
return 0;
}
/* The input buffer has overflowed. Refill the input buffer, then
** return the next character
*/
static CSV_NOINLINE int csv_getc_refill(CsvReader *p){
size_t got;
assert( p->iIn>=p->nIn ); /* Only called on an empty input buffer */
assert( p->in!=0 ); /* Only called if reading froma file */
got = fread(p->zIn, 1, CSV_INBUFSZ, p->in);
if( got==0 ) return EOF;
p->nIn = got;
p->iIn = 1;
return p->zIn[0];
}
/* Return the next character of input. Return EOF at end of input. */
static int csv_getc(CsvReader *p){
if( p->iIn >= p->nIn ){
if( p->in!=0 ) return csv_getc_refill(p);
return EOF;
}
return p->zIn[p->iIn++];
}
/* Increase the size of p->z and append character c to the end.
** Return 0 on success and non-zero if there is an OOM error */
static CSV_NOINLINE int csv_resize_and_append(CsvReader *p, char c){
char *zNew;
int nNew = p->nAlloc*2 + 100;
zNew = sqlite3_realloc64(p->z, nNew);
if( zNew ){
p->z = zNew;
p->nAlloc = nNew;
p->z[p->n++] = c;
return 0;
}else{
csv_errmsg(p, "out of memory");
return 1;
}
}
/* Append a single character to the CsvReader.z[] array.
** Return 0 on success and non-zero if there is an OOM error */
static int csv_append(CsvReader *p, char c){
if( p->n>=p->nAlloc-1 ) return csv_resize_and_append(p, c);
p->z[p->n++] = c;
return 0;
}
/* Read a single field of CSV text. Compatible with rfc4180 and extended
** with the option of having a separator other than ",".
**
** + Input comes from p->in.
** + Store results in p->z of length p->n. Space to hold p->z comes
** from sqlite3_malloc64().
** + Keep track of the line number in p->nLine.
** + Store the character that terminates the field in p->cTerm. Store
** EOF on end-of-file.
**
** Return "" at EOF. Return 0 on an OOM error.
*/
static char *csv_read_one_field(CsvReader *p){
int c;
p->n = 0;
c = csv_getc(p);
if( c==EOF ){
p->cTerm = EOF;
return "";
}
if( c=='"' ){
int pc, ppc;
int startLine = p->nLine;
pc = ppc = 0;
while( 1 ){
c = csv_getc(p);
if( c<='"' || pc=='"' ){
if( c=='\n' ) p->nLine++;
if( c=='"' ){
if( pc=='"' ){
pc = 0;
continue;
}
}
if( (c==',' && pc=='"')
|| (c=='\n' && pc=='"')
|| (c=='\n' && pc=='\r' && ppc=='"')
|| (c==EOF && pc=='"')
){
do{ p->n--; }while( p->z[p->n]!='"' );
p->cTerm = (char)c;
break;
}
if( pc=='"' && c!='\r' ){
csv_errmsg(p, "line %d: unescaped %c character", p->nLine, '"');
break;
}
if( c==EOF ){
csv_errmsg(p, "line %d: unterminated %c-quoted field\n",
startLine, '"');
p->cTerm = (char)c;
break;
}
}
if( csv_append(p, (char)c) ) return 0;
ppc = pc;
pc = c;
}
}else{
while( c>',' || (c!=EOF && c!=',' && c!='\n') ){
if( csv_append(p, (char)c) ) return 0;
c = csv_getc(p);
}
if( c=='\n' ){
p->nLine++;
if( p->n>0 && p->z[p->n-1]=='\r' ) p->n--;
}
p->cTerm = (char)c;
}
if( p->z ) p->z[p->n] = 0;
return p->z;
}
/* Forward references to the various virtual table methods implemented
** in this file. */
static int csvtabCreate(sqlite3*, void*, int, const char*const*,
sqlite3_vtab**,char**);
static int csvtabConnect(sqlite3*, void*, int, const char*const*,
sqlite3_vtab**,char**);
static int csvtabBestIndex(sqlite3_vtab*,sqlite3_index_info*);
static int csvtabDisconnect(sqlite3_vtab*);
static int csvtabOpen(sqlite3_vtab*, sqlite3_vtab_cursor**);
static int csvtabClose(sqlite3_vtab_cursor*);
static int csvtabFilter(sqlite3_vtab_cursor*, int idxNum, const char *idxStr,
int argc, sqlite3_value **argv);
static int csvtabNext(sqlite3_vtab_cursor*);
static int csvtabEof(sqlite3_vtab_cursor*);
static int csvtabColumn(sqlite3_vtab_cursor*,sqlite3_context*,int);
static int csvtabRowid(sqlite3_vtab_cursor*,sqlite3_int64*);
/* An instance of the CSV virtual table */
typedef struct CsvTable {
sqlite3_vtab base; /* Base class. Must be first */
char *zFilename; /* Name of the CSV file */
char *zData; /* Raw CSV data in lieu of zFilename */
long iStart; /* Offset to start of data in zFilename */
int nCol; /* Number of columns in the CSV file */
unsigned int tstFlags; /* Bit values used for testing */
} CsvTable;
/* Allowed values for tstFlags */
#define CSVTEST_FIDX 0x0001 /* Pretend that constrained searchs cost less*/
/* A cursor for the CSV virtual table */
typedef struct CsvCursor {
sqlite3_vtab_cursor base; /* Base class. Must be first */
CsvReader rdr; /* The CsvReader object */
char **azVal; /* Value of the current row */
int *aLen; /* Length of each entry */
sqlite3_int64 iRowid; /* The current rowid. Negative for EOF */
} CsvCursor;
/* Transfer error message text from a reader into a CsvTable */
static void csv_xfer_error(CsvTable *pTab, CsvReader *pRdr){
sqlite3_free(pTab->base.zErrMsg);
pTab->base.zErrMsg = sqlite3_mprintf("%s", pRdr->zErr);
}
/*
** This method is the destructor fo a CsvTable object.
*/
static int csvtabDisconnect(sqlite3_vtab *pVtab){
CsvTable *p = (CsvTable*)pVtab;
sqlite3_free(p->zFilename);
sqlite3_free(p->zData);
sqlite3_free(p);
return SQLITE_OK;
}
/* Skip leading whitespace. Return a pointer to the first non-whitespace
** character, or to the zero terminator if the string has only whitespace */
static const char *csv_skip_whitespace(const char *z){
while( isspace((unsigned char)z[0]) ) z++;
return z;
}
/* Remove trailing whitespace from the end of string z[] */
static void csv_trim_whitespace(char *z){
size_t n = strlen(z);
while( n>0 && isspace((unsigned char)z[n]) ) n--;
z[n] = 0;
}
/* Dequote the string */
static void csv_dequote(char *z){
int j;
char cQuote = z[0];
size_t i, n;
if( cQuote!='\'' && cQuote!='"' ) return;
n = strlen(z);
if( n<2 || z[n-1]!=z[0] ) return;
for(i=1, j=0; i<n-1; i++){
if( z[i]==cQuote && z[i+1]==cQuote ) i++;
z[j++] = z[i];
}
z[j] = 0;
}
/* Check to see if the string is of the form: "TAG = VALUE" with optional
** whitespace before and around tokens. If it is, return a pointer to the
** first character of VALUE. If it is not, return NULL.
*/
static const char *csv_parameter(const char *zTag, int nTag, const char *z){
z = csv_skip_whitespace(z);
if( strncmp(zTag, z, nTag)!=0 ) return 0;
z = csv_skip_whitespace(z+nTag);
if( z[0]!='=' ) return 0;
return csv_skip_whitespace(z+1);
}
/* Decode a parameter that requires a dequoted string.
**
** Return 1 if the parameter is seen, or 0 if not. 1 is returned
** even if there is an error. If an error occurs, then an error message
** is left in p->zErr. If there are no errors, p->zErr[0]==0.
*/
static int csv_string_parameter(
CsvReader *p, /* Leave the error message here, if there is one */
const char *zParam, /* Parameter we are checking for */
const char *zArg, /* Raw text of the virtual table argment */
char **pzVal /* Write the dequoted string value here */
){
const char *zValue;
zValue = csv_parameter(zParam,(int)strlen(zParam),zArg);
if( zValue==0 ) return 0;
p->zErr[0] = 0;
if( *pzVal ){
csv_errmsg(p, "more than one '%s' parameter", zParam);
return 1;
}
*pzVal = sqlite3_mprintf("%s", zValue);
if( *pzVal==0 ){
csv_errmsg(p, "out of memory");
return 1;
}
csv_trim_whitespace(*pzVal);
csv_dequote(*pzVal);
return 1;
}
/* Return 0 if the argument is false and 1 if it is true. Return -1 if
** we cannot really tell.
*/
static int csv_boolean(const char *z){
if( sqlite3_stricmp("yes",z)==0
|| sqlite3_stricmp("on",z)==0
|| sqlite3_stricmp("true",z)==0
|| (z[0]=='1' && z[1]==0)
){
return 1;
}
if( sqlite3_stricmp("no",z)==0
|| sqlite3_stricmp("off",z)==0
|| sqlite3_stricmp("false",z)==0
|| (z[0]=='0' && z[1]==0)
){
return 0;
}
return -1;
}
/*
** Parameters:
** filename=FILENAME Name of file containing CSV content
** data=TEXT Direct CSV content.
** schema=SCHEMA Alternative CSV schema.
** header=YES|NO First row of CSV defines the names of
** columns if "yes". Default "no".
** columns=N Assume the CSV file contains N columns.
**
** Only available if compiled with SQLITE_TEST:
**
** testflags=N Bitmask of test flags. Optional
**
** If schema= is omitted, then the columns are named "c0", "c1", "c2",
** and so forth. If columns=N is omitted, then the file is opened and
** the number of columns in the first row is counted to determine the
** column count. If header=YES, then the first row is skipped.
*/
static int csvtabConnect(
sqlite3 *db,
void *pAux,
int argc, const char *const*argv,
sqlite3_vtab **ppVtab,
char **pzErr
){
CsvTable *pNew = 0; /* The CsvTable object to construct */
int bHeader = -1; /* header= flags. -1 means not seen yet */
int rc = SQLITE_OK; /* Result code from this routine */
int i, j; /* Loop counters */
#ifdef SQLITE_TEST
int tstFlags = 0; /* Value for testflags=N parameter */
#endif
int nCol = -99; /* Value of the columns= parameter */
CsvReader sRdr; /* A CSV file reader used to store an error
** message and/or to count the number of columns */
static const char *azParam[] = {
"filename", "data", "schema",
};
char *azPValue[3]; /* Parameter values */
# define CSV_FILENAME (azPValue[0])
# define CSV_DATA (azPValue[1])
# define CSV_SCHEMA (azPValue[2])
assert( sizeof(azPValue)==sizeof(azParam) );
memset(&sRdr, 0, sizeof(sRdr));
memset(azPValue, 0, sizeof(azPValue));
for(i=3; i<argc; i++){
const char *z = argv[i];
const char *zValue;
for(j=0; j<sizeof(azParam)/sizeof(azParam[0]); j++){
if( csv_string_parameter(&sRdr, azParam[j], z, &azPValue[j]) ) break;
}
if( j<sizeof(azParam)/sizeof(azParam[0]) ){
if( sRdr.zErr[0] ) goto csvtab_connect_error;
}else
if( (zValue = csv_parameter("header",6,z))!=0 ){
int x;
if( bHeader>=0 ){
csv_errmsg(&sRdr, "more than one 'header' parameter");
goto csvtab_connect_error;
}
x = csv_boolean(zValue);
if( x==1 ){
bHeader = 1;
}else if( x==0 ){
bHeader = 0;
}else{
csv_errmsg(&sRdr, "unrecognized argument to 'header': %s", zValue);
goto csvtab_connect_error;
}
}else
#ifdef SQLITE_TEST
if( (zValue = csv_parameter("testflags",9,z))!=0 ){
tstFlags = (unsigned int)atoi(zValue);
}else
#endif
if( (zValue = csv_parameter("columns",7,z))!=0 ){
if( nCol>0 ){
csv_errmsg(&sRdr, "more than one 'columns' parameter");
goto csvtab_connect_error;
}
nCol = atoi(zValue);
if( nCol<=0 ){
csv_errmsg(&sRdr, "must have at least one column");
goto csvtab_connect_error;
}
}else
{
csv_errmsg(&sRdr, "unrecognized parameter '%s'", z);
goto csvtab_connect_error;
}
}
if( (CSV_FILENAME==0)==(CSV_DATA==0) ){
csv_errmsg(&sRdr, "must either filename= or data= but not both");
goto csvtab_connect_error;
}
if( nCol<=0 && csv_reader_open(&sRdr, CSV_FILENAME, CSV_DATA) ){
goto csvtab_connect_error;
}
pNew = sqlite3_malloc( sizeof(*pNew) );
*ppVtab = (sqlite3_vtab*)pNew;
if( pNew==0 ) goto csvtab_connect_oom;
memset(pNew, 0, sizeof(*pNew));
if( nCol>0 ){
pNew->nCol = nCol;
}else{
do{
const char *z = csv_read_one_field(&sRdr);
if( z==0 ) goto csvtab_connect_oom;
pNew->nCol++;
}while( sRdr.cTerm==',' );
}
pNew->zFilename = CSV_FILENAME; CSV_FILENAME = 0;
pNew->zData = CSV_DATA; CSV_DATA = 0;
#ifdef SQLITE_TEST
pNew->tstFlags = tstFlags;
#endif
pNew->iStart = bHeader==1 ? ftell(sRdr.in) : 0;
csv_reader_reset(&sRdr);
if( CSV_SCHEMA==0 ){
char *zSep = "";
CSV_SCHEMA = sqlite3_mprintf("CREATE TABLE x(");
if( CSV_SCHEMA==0 ) goto csvtab_connect_oom;
for(i=0; i<pNew->nCol; i++){
CSV_SCHEMA = sqlite3_mprintf("%z%sc%d TEXT",CSV_SCHEMA, zSep, i);
zSep = ",";
}
CSV_SCHEMA = sqlite3_mprintf("%z);", CSV_SCHEMA);
}
rc = sqlite3_declare_vtab(db, CSV_SCHEMA);
if( rc ) goto csvtab_connect_error;
for(i=0; i<sizeof(azPValue)/sizeof(azPValue[0]); i++){
sqlite3_free(azPValue[i]);
}
return SQLITE_OK;
csvtab_connect_oom:
rc = SQLITE_NOMEM;
csv_errmsg(&sRdr, "out of memory");
csvtab_connect_error:
if( pNew ) csvtabDisconnect(&pNew->base);
for(i=0; i<sizeof(azPValue)/sizeof(azPValue[0]); i++){
sqlite3_free(azPValue[i]);
}
if( sRdr.zErr[0] ){
sqlite3_free(*pzErr);
*pzErr = sqlite3_mprintf("%s", sRdr.zErr);
}
csv_reader_reset(&sRdr);
if( rc==SQLITE_OK ) rc = SQLITE_ERROR;
return rc;
}
/*
** Reset the current row content held by a CsvCursor.
*/
static void csvtabCursorRowReset(CsvCursor *pCur){
CsvTable *pTab = (CsvTable*)pCur->base.pVtab;
int i;
for(i=0; i<pTab->nCol; i++){
sqlite3_free(pCur->azVal[i]);
pCur->azVal[i] = 0;
pCur->aLen[i] = 0;
}
}
/*
** The xConnect and xCreate methods do the same thing, but they must be
** different so that the virtual table is not an eponymous virtual table.
*/
static int csvtabCreate(
sqlite3 *db,
void *pAux,
int argc, const char *const*argv,
sqlite3_vtab **ppVtab,
char **pzErr
){
return csvtabConnect(db, pAux, argc, argv, ppVtab, pzErr);
}
/*
** Destructor for a CsvCursor.
*/
static int csvtabClose(sqlite3_vtab_cursor *cur){
CsvCursor *pCur = (CsvCursor*)cur;
csvtabCursorRowReset(pCur);
csv_reader_reset(&pCur->rdr);
sqlite3_free(cur);
return SQLITE_OK;
}
/*
** Constructor for a new CsvTable cursor object.
*/
static int csvtabOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
CsvTable *pTab = (CsvTable*)p;
CsvCursor *pCur;
size_t nByte;
nByte = sizeof(*pCur) + (sizeof(char*)+sizeof(int))*pTab->nCol;
pCur = sqlite3_malloc64( nByte );
if( pCur==0 ) return SQLITE_NOMEM;
memset(pCur, 0, nByte);
pCur->azVal = (char**)&pCur[1];
pCur->aLen = (int*)&pCur->azVal[pTab->nCol];
*ppCursor = &pCur->base;
if( csv_reader_open(&pCur->rdr, pTab->zFilename, pTab->zData) ){
csv_xfer_error(pTab, &pCur->rdr);
return SQLITE_ERROR;
}
return SQLITE_OK;
}
/*
** Advance a CsvCursor to its next row of input.
** Set the EOF marker if we reach the end of input.
*/
static int csvtabNext(sqlite3_vtab_cursor *cur){
CsvCursor *pCur = (CsvCursor*)cur;
CsvTable *pTab = (CsvTable*)cur->pVtab;
int i = 0;
char *z;
do{
z = csv_read_one_field(&pCur->rdr);
if( z==0 ){
csv_xfer_error(pTab, &pCur->rdr);
break;
}
if( i<pTab->nCol ){
if( pCur->aLen[i] < pCur->rdr.n+1 ){
char *zNew = sqlite3_realloc64(pCur->azVal[i], pCur->rdr.n+1);
if( zNew==0 ){
csv_errmsg(&pCur->rdr, "out of memory");
csv_xfer_error(pTab, &pCur->rdr);
break;
}
pCur->azVal[i] = zNew;
pCur->aLen[i] = pCur->rdr.n+1;
}
memcpy(pCur->azVal[i], z, pCur->rdr.n+1);
i++;
}
}while( pCur->rdr.cTerm==',' );
while( i<pTab->nCol ){
sqlite3_free(pCur->azVal[i]);
pCur->azVal[i] = 0;
pCur->aLen[i] = 0;
i++;
}
if( z==0 || pCur->rdr.cTerm==EOF ){
pCur->iRowid = -1;
}else{
pCur->iRowid++;
}
return SQLITE_OK;
}
/*
** Return values of columns for the row at which the CsvCursor
** is currently pointing.
*/
static int csvtabColumn(
sqlite3_vtab_cursor *cur, /* The cursor */
sqlite3_context *ctx, /* First argument to sqlite3_result_...() */
int i /* Which column to return */
){
CsvCursor *pCur = (CsvCursor*)cur;
CsvTable *pTab = (CsvTable*)cur->pVtab;
if( i>=0 && i<pTab->nCol && pCur->azVal[i]!=0 ){
sqlite3_result_text(ctx, pCur->azVal[i], -1, SQLITE_STATIC);
}
return SQLITE_OK;
}
/*
** Return the rowid for the current row.
*/
static int csvtabRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
CsvCursor *pCur = (CsvCursor*)cur;
*pRowid = pCur->iRowid;
return SQLITE_OK;
}
/*
** Return TRUE if the cursor has been moved off of the last
** row of output.
*/
static int csvtabEof(sqlite3_vtab_cursor *cur){
CsvCursor *pCur = (CsvCursor*)cur;
return pCur->iRowid<0;
}
/*
** Only a full table scan is supported. So xFilter simply rewinds to
** the beginning.
*/
static int csvtabFilter(
sqlite3_vtab_cursor *pVtabCursor,
int idxNum, const char *idxStr,
int argc, sqlite3_value **argv
){
CsvCursor *pCur = (CsvCursor*)pVtabCursor;
CsvTable *pTab = (CsvTable*)pVtabCursor->pVtab;
pCur->iRowid = 0;
if( pCur->rdr.in==0 ){
assert( pCur->rdr.zIn==pTab->zData );
assert( pTab->iStart>=0 );
assert( (size_t)pTab->iStart<=pCur->rdr.nIn );
pCur->rdr.iIn = pTab->iStart;
}else{
fseek(pCur->rdr.in, pTab->iStart, SEEK_SET);
pCur->rdr.iIn = 0;
pCur->rdr.nIn = 0;
}
return csvtabNext(pVtabCursor);
}
/*
** Only a forward full table scan is supported. xBestIndex is mostly
** a no-op. If CSVTEST_FIDX is set, then the presence of equality
** constraints lowers the estimated cost, which is fiction, but is useful
** for testing certain kinds of virtual table behavior.
*/
static int csvtabBestIndex(
sqlite3_vtab *tab,
sqlite3_index_info *pIdxInfo
){
pIdxInfo->estimatedCost = 1000000;
#ifdef SQLITE_TEST
if( (((CsvTable*)tab)->tstFlags & CSVTEST_FIDX)!=0 ){
/* The usual (and sensible) case is to always do a full table scan.
** The code in this branch only runs when testflags=1. This code
** generates an artifical and unrealistic plan which is useful
** for testing virtual table logic but is not helpful to real applications.
**
** Any ==, LIKE, or GLOB constraint is marked as usable by the virtual
** table (even though it is not) and the cost of running the virtual table
** is reduced from 1 million to just 10. The constraints are *not* marked
** as omittable, however, so the query planner should still generate a
** plan that gives a correct answer, even if they plan is not optimal.
*/
int i;
int nConst = 0;
for(i=0; i<pIdxInfo->nConstraint; i++){
unsigned char op;
if( pIdxInfo->aConstraint[i].usable==0 ) continue;
op = pIdxInfo->aConstraint[i].op;
if( op==SQLITE_INDEX_CONSTRAINT_EQ
|| op==SQLITE_INDEX_CONSTRAINT_LIKE
|| op==SQLITE_INDEX_CONSTRAINT_GLOB
){
pIdxInfo->estimatedCost = 10;
pIdxInfo->aConstraintUsage[nConst].argvIndex = nConst+1;
nConst++;
}
}
}
#endif
return SQLITE_OK;
}
static sqlite3_module CsvModule = {
0, /* iVersion */
csvtabCreate, /* xCreate */
csvtabConnect, /* xConnect */
csvtabBestIndex, /* xBestIndex */
csvtabDisconnect, /* xDisconnect */
csvtabDisconnect, /* xDestroy */
csvtabOpen, /* xOpen - open a cursor */
csvtabClose, /* xClose - close a cursor */
csvtabFilter, /* xFilter - configure scan constraints */
csvtabNext, /* xNext - advance a cursor */
csvtabEof, /* xEof - check for end of scan */
csvtabColumn, /* xColumn - read data */
csvtabRowid, /* xRowid - read data */
0, /* xUpdate */
0, /* xBegin */
0, /* xSync */
0, /* xCommit */
0, /* xRollback */
0, /* xFindMethod */
0, /* xRename */
};
#ifdef SQLITE_TEST
/*
** For virtual table testing, make a version of the CSV virtual table
** available that has an xUpdate function. But the xUpdate always returns
** SQLITE_READONLY since the CSV file is not really writable.
*/
static int csvtabUpdate(sqlite3_vtab *p,int n,sqlite3_value**v,sqlite3_int64*x){
return SQLITE_READONLY;
}
static sqlite3_module CsvModuleFauxWrite = {
0, /* iVersion */
csvtabCreate, /* xCreate */
csvtabConnect, /* xConnect */
csvtabBestIndex, /* xBestIndex */
csvtabDisconnect, /* xDisconnect */
csvtabDisconnect, /* xDestroy */
csvtabOpen, /* xOpen - open a cursor */
csvtabClose, /* xClose - close a cursor */
csvtabFilter, /* xFilter - configure scan constraints */
csvtabNext, /* xNext - advance a cursor */
csvtabEof, /* xEof - check for end of scan */
csvtabColumn, /* xColumn - read data */
csvtabRowid, /* xRowid - read data */
csvtabUpdate, /* xUpdate */
0, /* xBegin */
0, /* xSync */
0, /* xCommit */
0, /* xRollback */
0, /* xFindMethod */
0, /* xRename */
};
#endif /* SQLITE_TEST */
#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */
#ifdef _WIN32
__declspec(dllexport)
#endif
/*
** This routine is called when the extension is loaded. The new
** CSV virtual table module is registered with the calling database
** connection.
*/
int sqlite3_csv_init(
sqlite3 *db,
char **pzErrMsg,
const sqlite3_api_routines *pApi
){
#ifndef SQLITE_OMIT_VIRTUALTABLE
int rc;
SQLITE_EXTENSION_INIT2(pApi);
rc = sqlite3_create_module(db, "csv", &CsvModule, 0);
#ifdef SQLITE_TEST
if( rc==SQLITE_OK ){
rc = sqlite3_create_module(db, "csv_wr", &CsvModuleFauxWrite, 0);
}
#endif
return rc;
#else
return SQLITE_OK;
#endif
}

717
ext/misc/dbdump.c Normal file
View File

@@ -0,0 +1,717 @@
/*
** 2016-03-13
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** This file implements a C-language subroutine that converts the content
** of an SQLite database into UTF-8 text SQL statements that can be used
** to exactly recreate the original database. ROWID values are preserved.
**
** A prototype of the implemented subroutine is this:
**
** int sqlite3_db_dump(
** sqlite3 *db,
** const char *zSchema,
** const char *zTable,
** void (*xCallback)(void*, const char*),
** void *pArg
** );
**
** The db parameter is the database connection. zSchema is the schema within
** that database which is to be dumped. Usually the zSchema is "main" but
** can also be "temp" or any ATTACH-ed database. If zTable is not NULL, then
** only the content of that one table is dumped. If zTable is NULL, then all
** tables are dumped.
**
** The generate text is passed to xCallback() in multiple calls. The second
** argument to xCallback() is a copy of the pArg parameter. The first
** argument is some of the output text that this routine generates. The
** signature to xCallback() is designed to make it compatible with fputs().
**
** The sqlite3_db_dump() subroutine returns SQLITE_OK on success or some error
** code if it encounters a problem.
**
** If this file is compiled with -DDBDUMP_STANDALONE then a "main()" routine
** is included so that this routine becomes a command-line utility. The
** command-line utility takes two or three arguments which are the name
** of the database file, the schema, and optionally the table, forming the
** first three arguments of a single call to the library routine.
*/
#include "sqlite3.h"
#include <stdarg.h>
#include <string.h>
#include <ctype.h>
/*
** The state of the dump process.
*/
typedef struct DState DState;
struct DState {
sqlite3 *db; /* The database connection */
int nErr; /* Number of errors seen so far */
int rc; /* Error code */
int writableSchema; /* True if in writable_schema mode */
int (*xCallback)(const char*,void*); /* Send output here */
void *pArg; /* Argument to xCallback() */
};
/*
** A variable length string to which one can append text.
*/
typedef struct DText DText;
struct DText {
char *z; /* The text */
int n; /* Number of bytes of content in z[] */
int nAlloc; /* Number of bytes allocated to z[] */
};
/*
** Initialize and destroy a DText object
*/
static void initText(DText *p){
memset(p, 0, sizeof(*p));
}
static void freeText(DText *p){
sqlite3_free(p->z);
initText(p);
}
/* zIn is either a pointer to a NULL-terminated string in memory obtained
** from malloc(), or a NULL pointer. The string pointed to by zAppend is
** added to zIn, and the result returned in memory obtained from malloc().
** zIn, if it was not NULL, is freed.
**
** If the third argument, quote, is not '\0', then it is used as a
** quote character for zAppend.
*/
static void appendText(DText *p, char const *zAppend, char quote){
int len;
int i;
int nAppend = (int)(strlen(zAppend) & 0x3fffffff);
len = nAppend+p->n+1;
if( quote ){
len += 2;
for(i=0; i<nAppend; i++){
if( zAppend[i]==quote ) len++;
}
}
if( p->n+len>=p->nAlloc ){
char *zNew;
p->nAlloc = p->nAlloc*2 + len + 20;
zNew = sqlite3_realloc(p->z, p->nAlloc);
if( zNew==0 ){
freeText(p);
return;
}
p->z = zNew;
}
if( quote ){
char *zCsr = p->z+p->n;
*zCsr++ = quote;
for(i=0; i<nAppend; i++){
*zCsr++ = zAppend[i];
if( zAppend[i]==quote ) *zCsr++ = quote;
}
*zCsr++ = quote;
p->n = (int)(zCsr - p->z);
*zCsr = '\0';
}else{
memcpy(p->z+p->n, zAppend, nAppend);
p->n += nAppend;
p->z[p->n] = '\0';
}
}
/*
** Attempt to determine if identifier zName needs to be quoted, either
** because it contains non-alphanumeric characters, or because it is an
** SQLite keyword. Be conservative in this estimate: When in doubt assume
** that quoting is required.
**
** Return '"' if quoting is required. Return 0 if no quoting is required.
*/
static char quoteChar(const char *zName){
/* All SQLite keywords, in alphabetical order */
static const char *azKeywords[] = {
"ABORT", "ACTION", "ADD", "AFTER", "ALL", "ALTER", "ANALYZE", "AND", "AS",
"ASC", "ATTACH", "AUTOINCREMENT", "BEFORE", "BEGIN", "BETWEEN", "BY",
"CASCADE", "CASE", "CAST", "CHECK", "COLLATE", "COLUMN", "COMMIT",
"CONFLICT", "CONSTRAINT", "CREATE", "CROSS", "CURRENT_DATE",
"CURRENT_TIME", "CURRENT_TIMESTAMP", "DATABASE", "DEFAULT", "DEFERRABLE",
"DEFERRED", "DELETE", "DESC", "DETACH", "DISTINCT", "DROP", "EACH",
"ELSE", "END", "ESCAPE", "EXCEPT", "EXCLUSIVE", "EXISTS", "EXPLAIN",
"FAIL", "FOR", "FOREIGN", "FROM", "FULL", "GLOB", "GROUP", "HAVING", "IF",
"IGNORE", "IMMEDIATE", "IN", "INDEX", "INDEXED", "INITIALLY", "INNER",
"INSERT", "INSTEAD", "INTERSECT", "INTO", "IS", "ISNULL", "JOIN", "KEY",
"LEFT", "LIKE", "LIMIT", "MATCH", "NATURAL", "NO", "NOT", "NOTNULL",
"NULL", "OF", "OFFSET", "ON", "OR", "ORDER", "OUTER", "PLAN", "PRAGMA",
"PRIMARY", "QUERY", "RAISE", "RECURSIVE", "REFERENCES", "REGEXP",
"REINDEX", "RELEASE", "RENAME", "REPLACE", "RESTRICT", "RIGHT",
"ROLLBACK", "ROW", "SAVEPOINT", "SELECT", "SET", "TABLE", "TEMP",
"TEMPORARY", "THEN", "TO", "TRANSACTION", "TRIGGER", "UNION", "UNIQUE",
"UPDATE", "USING", "VACUUM", "VALUES", "VIEW", "VIRTUAL", "WHEN", "WHERE",
"WITH", "WITHOUT",
};
int i, lwr, upr, mid, c;
if( !isalpha((unsigned char)zName[0]) && zName[0]!='_' ) return '"';
for(i=0; zName[i]; i++){
if( !isalnum((unsigned char)zName[i]) && zName[i]!='_' ) return '"';
}
lwr = 0;
upr = sizeof(azKeywords)/sizeof(azKeywords[0]) - 1;
while( lwr<=upr ){
mid = (lwr+upr)/2;
c = sqlite3_stricmp(azKeywords[mid], zName);
if( c==0 ) return '"';
if( c<0 ){
lwr = mid+1;
}else{
upr = mid-1;
}
}
return 0;
}
/*
** Release memory previously allocated by tableColumnList().
*/
static void freeColumnList(char **azCol){
int i;
for(i=1; azCol[i]; i++){
sqlite3_free(azCol[i]);
}
/* azCol[0] is a static string */
sqlite3_free(azCol);
}
/*
** Return a list of pointers to strings which are the names of all
** columns in table zTab. The memory to hold the names is dynamically
** allocated and must be released by the caller using a subsequent call
** to freeColumnList().
**
** The azCol[0] entry is usually NULL. However, if zTab contains a rowid
** value that needs to be preserved, then azCol[0] is filled in with the
** name of the rowid column.
**
** The first regular column in the table is azCol[1]. The list is terminated
** by an entry with azCol[i]==0.
*/
static char **tableColumnList(DState *p, const char *zTab){
char **azCol = 0;
sqlite3_stmt *pStmt = 0;
char *zSql;
int nCol = 0;
int nAlloc = 0;
int nPK = 0; /* Number of PRIMARY KEY columns seen */
int isIPK = 0; /* True if one PRIMARY KEY column of type INTEGER */
int preserveRowid = 1;
int rc;
zSql = sqlite3_mprintf("PRAGMA table_info=%Q", zTab);
if( zSql==0 ) return 0;
rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
sqlite3_free(zSql);
if( rc ) return 0;
while( sqlite3_step(pStmt)==SQLITE_ROW ){
if( nCol>=nAlloc-2 ){
char **azNew;
nAlloc = nAlloc*2 + nCol + 10;
azNew = sqlite3_realloc(azCol, nAlloc*sizeof(azCol[0]));
if( azNew==0 ) goto col_oom;
azCol = azNew;
azCol[0] = 0;
}
azCol[++nCol] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1));
if( azCol[nCol]==0 ) goto col_oom;
if( sqlite3_column_int(pStmt, 5) ){
nPK++;
if( nPK==1
&& sqlite3_stricmp((const char*)sqlite3_column_text(pStmt,2),
"INTEGER")==0
){
isIPK = 1;
}else{
isIPK = 0;
}
}
}
sqlite3_finalize(pStmt);
pStmt = 0;
azCol[nCol+1] = 0;
/* The decision of whether or not a rowid really needs to be preserved
** is tricky. We never need to preserve a rowid for a WITHOUT ROWID table
** or a table with an INTEGER PRIMARY KEY. We are unable to preserve
** rowids on tables where the rowid is inaccessible because there are other
** columns in the table named "rowid", "_rowid_", and "oid".
*/
if( isIPK ){
/* If a single PRIMARY KEY column with type INTEGER was seen, then it
** might be an alise for the ROWID. But it might also be a WITHOUT ROWID
** table or a INTEGER PRIMARY KEY DESC column, neither of which are
** ROWID aliases. To distinguish these cases, check to see if
** there is a "pk" entry in "PRAGMA index_list". There will be
** no "pk" index if the PRIMARY KEY really is an alias for the ROWID.
*/
zSql = sqlite3_mprintf("SELECT 1 FROM pragma_index_list(%Q)"
" WHERE origin='pk'", zTab);
if( zSql==0 ) goto col_oom;
rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
sqlite3_free(zSql);
if( rc ){
freeColumnList(azCol);
return 0;
}
rc = sqlite3_step(pStmt);
sqlite3_finalize(pStmt);
pStmt = 0;
preserveRowid = rc==SQLITE_ROW;
}
if( preserveRowid ){
/* Only preserve the rowid if we can find a name to use for the
** rowid */
static char *azRowid[] = { "rowid", "_rowid_", "oid" };
int i, j;
for(j=0; j<3; j++){
for(i=1; i<=nCol; i++){
if( sqlite3_stricmp(azRowid[j],azCol[i])==0 ) break;
}
if( i>nCol ){
/* At this point, we know that azRowid[j] is not the name of any
** ordinary column in the table. Verify that azRowid[j] is a valid
** name for the rowid before adding it to azCol[0]. WITHOUT ROWID
** tables will fail this last check */
int rc;
rc = sqlite3_table_column_metadata(p->db,0,zTab,azRowid[j],0,0,0,0,0);
if( rc==SQLITE_OK ) azCol[0] = azRowid[j];
break;
}
}
}
return azCol;
col_oom:
sqlite3_finalize(pStmt);
freeColumnList(azCol);
p->nErr++;
p->rc = SQLITE_NOMEM;
return 0;
}
/*
** Send mprintf-formatted content to the output callback.
*/
static void output_formatted(DState *p, const char *zFormat, ...){
va_list ap;
char *z;
va_start(ap, zFormat);
z = sqlite3_vmprintf(zFormat, ap);
va_end(ap);
p->xCallback(z, p->pArg);
sqlite3_free(z);
}
/*
** Output the given string as a quoted string using SQL quoting conventions.
**
** The "\n" and "\r" characters are converted to char(10) and char(13)
** to prevent them from being transformed by end-of-line translators.
*/
static void output_quoted_string(DState *p, const unsigned char *z){
int i;
char c;
int inQuote = 0;
int bStarted = 0;
for(i=0; (c = z[i])!=0 && c!='\'' && c!='\n' && c!='\r'; i++){}
if( c==0 ){
output_formatted(p, "'%s'", z);
return;
}
while( *z ){
for(i=0; (c = z[i])!=0 && c!='\n' && c!='\r' && c!='\''; i++){}
if( c=='\'' ) i++;
if( i ){
if( !inQuote ){
if( bStarted ) p->xCallback("||", p->pArg);
p->xCallback("'", p->pArg);
inQuote = 1;
}
output_formatted(p, "%.*s", i, z);
z += i;
bStarted = 1;
}
if( c=='\'' ){
p->xCallback("'", p->pArg);
continue;
}
if( inQuote ){
p->xCallback("'", p->pArg);
inQuote = 0;
}
if( c==0 ){
break;
}
for(i=0; (c = z[i])=='\r' || c=='\n'; i++){
if( bStarted ) p->xCallback("||", p->pArg);
output_formatted(p, "char(%d)", c);
bStarted = 1;
}
z += i;
}
if( inQuote ) p->xCallback("'", p->pArg);
}
/*
** This is an sqlite3_exec callback routine used for dumping the database.
** Each row received by this callback consists of a table name,
** the table type ("index" or "table") and SQL to create the table.
** This routine should print text sufficient to recreate the table.
*/
static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){
int rc;
const char *zTable;
const char *zType;
const char *zSql;
DState *p = (DState*)pArg;
sqlite3_stmt *pStmt;
(void)azCol;
if( nArg!=3 ) return 1;
zTable = azArg[0];
zType = azArg[1];
zSql = azArg[2];
if( strcmp(zTable, "sqlite_sequence")==0 ){
p->xCallback("DELETE FROM sqlite_sequence;\n", p->pArg);
}else if( sqlite3_strglob("sqlite_stat?", zTable)==0 ){
p->xCallback("ANALYZE sqlite_master;\n", p->pArg);
}else if( strncmp(zTable, "sqlite_", 7)==0 ){
return 0;
}else if( strncmp(zSql, "CREATE VIRTUAL TABLE", 20)==0 ){
if( !p->writableSchema ){
p->xCallback("PRAGMA writable_schema=ON;\n", p->pArg);
p->writableSchema = 1;
}
output_formatted(p,
"INSERT INTO sqlite_master(type,name,tbl_name,rootpage,sql)"
"VALUES('table','%q','%q',0,'%q');",
zTable, zTable, zSql);
return 0;
}else{
if( sqlite3_strglob("CREATE TABLE ['\"]*", zSql)==0 ){
p->xCallback("CREATE TABLE IF NOT EXISTS ", p->pArg);
p->xCallback(zSql+13, p->pArg);
}else{
p->xCallback(zSql, p->pArg);
}
p->xCallback(";\n", p->pArg);
}
if( strcmp(zType, "table")==0 ){
DText sSelect;
DText sTable;
char **azCol;
int i;
int nCol;
azCol = tableColumnList(p, zTable);
if( azCol==0 ) return 0;
initText(&sTable);
appendText(&sTable, "INSERT INTO ", 0);
/* Always quote the table name, even if it appears to be pure ascii,
** in case it is a keyword. Ex: INSERT INTO "table" ... */
appendText(&sTable, zTable, quoteChar(zTable));
/* If preserving the rowid, add a column list after the table name.
** In other words: "INSERT INTO tab(rowid,a,b,c,...) VALUES(...)"
** instead of the usual "INSERT INTO tab VALUES(...)".
*/
if( azCol[0] ){
appendText(&sTable, "(", 0);
appendText(&sTable, azCol[0], 0);
for(i=1; azCol[i]; i++){
appendText(&sTable, ",", 0);
appendText(&sTable, azCol[i], quoteChar(azCol[i]));
}
appendText(&sTable, ")", 0);
}
appendText(&sTable, " VALUES(", 0);
/* Build an appropriate SELECT statement */
initText(&sSelect);
appendText(&sSelect, "SELECT ", 0);
if( azCol[0] ){
appendText(&sSelect, azCol[0], 0);
appendText(&sSelect, ",", 0);
}
for(i=1; azCol[i]; i++){
appendText(&sSelect, azCol[i], quoteChar(azCol[i]));
if( azCol[i+1] ){
appendText(&sSelect, ",", 0);
}
}
nCol = i;
if( azCol[0]==0 ) nCol--;
freeColumnList(azCol);
appendText(&sSelect, " FROM ", 0);
appendText(&sSelect, zTable, quoteChar(zTable));
rc = sqlite3_prepare_v2(p->db, sSelect.z, -1, &pStmt, 0);
if( rc!=SQLITE_OK ){
p->nErr++;
if( p->rc==SQLITE_OK ) p->rc = rc;
}else{
while( SQLITE_ROW==sqlite3_step(pStmt) ){
p->xCallback(sTable.z, p->pArg);
for(i=0; i<nCol; i++){
if( i ) p->xCallback(",", p->pArg);
switch( sqlite3_column_type(pStmt,i) ){
case SQLITE_INTEGER: {
output_formatted(p, "%lld", sqlite3_column_int64(pStmt,i));
break;
}
case SQLITE_FLOAT: {
double r = sqlite3_column_double(pStmt,i);
output_formatted(p, "%!.20g", r);
break;
}
case SQLITE_NULL: {
p->xCallback("NULL", p->pArg);
break;
}
case SQLITE_TEXT: {
output_quoted_string(p, sqlite3_column_text(pStmt,i));
break;
}
case SQLITE_BLOB: {
int nByte = sqlite3_column_bytes(pStmt,i);
unsigned char *a = (unsigned char*)sqlite3_column_blob(pStmt,i);
int j;
p->xCallback("x'", p->pArg);
for(j=0; j<nByte; j++){
char zWord[3];
zWord[0] = "0123456789abcdef"[(a[j]>>4)&15];
zWord[1] = "0123456789abcdef"[a[j]&15];
zWord[2] = 0;
p->xCallback(zWord, p->pArg);
}
p->xCallback("'", p->pArg);
break;
}
}
}
p->xCallback(");\n", p->pArg);
}
}
sqlite3_finalize(pStmt);
freeText(&sTable);
freeText(&sSelect);
}
return 0;
}
/*
** Execute a query statement that will generate SQL output. Print
** the result columns, comma-separated, on a line and then add a
** semicolon terminator to the end of that line.
**
** If the number of columns is 1 and that column contains text "--"
** then write the semicolon on a separate line. That way, if a
** "--" comment occurs at the end of the statement, the comment
** won't consume the semicolon terminator.
*/
static void output_sql_from_query(
DState *p, /* Query context */
const char *zSelect, /* SELECT statement to extract content */
...
){
sqlite3_stmt *pSelect;
int rc;
int nResult;
int i;
const char *z;
char *zSql;
va_list ap;
va_start(ap, zSelect);
zSql = sqlite3_vmprintf(zSelect, ap);
va_end(ap);
if( zSql==0 ){
p->rc = SQLITE_NOMEM;
p->nErr++;
return;
}
rc = sqlite3_prepare_v2(p->db, zSql, -1, &pSelect, 0);
sqlite3_free(zSql);
if( rc!=SQLITE_OK || !pSelect ){
output_formatted(p, "/**** ERROR: (%d) %s *****/\n", rc,
sqlite3_errmsg(p->db));
p->nErr++;
return;
}
rc = sqlite3_step(pSelect);
nResult = sqlite3_column_count(pSelect);
while( rc==SQLITE_ROW ){
z = (const char*)sqlite3_column_text(pSelect, 0);
p->xCallback(z, p->pArg);
for(i=1; i<nResult; i++){
p->xCallback(",", p->pArg);
p->xCallback((const char*)sqlite3_column_text(pSelect,i), p->pArg);
}
if( z==0 ) z = "";
while( z[0] && (z[0]!='-' || z[1]!='-') ) z++;
if( z[0] ){
p->xCallback("\n;\n", p->pArg);
}else{
p->xCallback(";\n", p->pArg);
}
rc = sqlite3_step(pSelect);
}
rc = sqlite3_finalize(pSelect);
if( rc!=SQLITE_OK ){
output_formatted(p, "/**** ERROR: (%d) %s *****/\n", rc,
sqlite3_errmsg(p->db));
if( (rc&0xff)!=SQLITE_CORRUPT ) p->nErr++;
}
}
/*
** Run zQuery. Use dump_callback() as the callback routine so that
** the contents of the query are output as SQL statements.
**
** If we get a SQLITE_CORRUPT error, rerun the query after appending
** "ORDER BY rowid DESC" to the end.
*/
static void run_schema_dump_query(
DState *p,
const char *zQuery,
...
){
char *zErr = 0;
char *z;
va_list ap;
va_start(ap, zQuery);
z = sqlite3_vmprintf(zQuery, ap);
va_end(ap);
sqlite3_exec(p->db, z, dump_callback, p, &zErr);
sqlite3_free(z);
if( zErr ){
output_formatted(p, "/****** %s ******/\n", zErr);
sqlite3_free(zErr);
p->nErr++;
zErr = 0;
}
}
/*
** Convert an SQLite database into SQL statements that will recreate that
** database.
*/
int sqlite3_db_dump(
sqlite3 *db, /* The database connection */
const char *zSchema, /* Which schema to dump. Usually "main". */
const char *zTable, /* Which table to dump. NULL means everything. */
int (*xCallback)(const char*,void*), /* Output sent to this callback */
void *pArg /* Second argument of the callback */
){
DState x;
memset(&x, 0, sizeof(x));
x.rc = sqlite3_exec(db, "BEGIN", 0, 0, 0);
if( x.rc ) return x.rc;
x.db = db;
x.xCallback = xCallback;
x.pArg = pArg;
xCallback("PRAGMA foreign_keys=OFF;\nBEGIN TRANSACTION;\n", pArg);
if( zTable==0 ){
run_schema_dump_query(&x,
"SELECT name, type, sql FROM \"%w\".sqlite_master "
"WHERE sql NOT NULL AND type=='table' AND name!='sqlite_sequence'",
zSchema
);
run_schema_dump_query(&x,
"SELECT name, type, sql FROM \"%w\".sqlite_master "
"WHERE name=='sqlite_sequence'", zSchema
);
output_sql_from_query(&x,
"SELECT sql FROM sqlite_master "
"WHERE sql NOT NULL AND type IN ('index','trigger','view')", 0
);
}else{
run_schema_dump_query(&x,
"SELECT name, type, sql FROM \"%w\".sqlite_master "
"WHERE tbl_name=%Q COLLATE nocase AND type=='table'"
" AND sql NOT NULL",
zSchema, zTable
);
output_sql_from_query(&x,
"SELECT sql FROM \"%w\".sqlite_master "
"WHERE sql NOT NULL"
" AND type IN ('index','trigger','view')"
" AND tbl_name=%Q COLLATE nocase",
zSchema, zTable
);
}
if( x.writableSchema ){
xCallback("PRAGMA writable_schema=OFF;\n", pArg);
}
xCallback(x.nErr ? "ROLLBACK; -- due to errors\n" : "COMMIT;\n", pArg);
sqlite3_exec(db, "COMMIT", 0, 0, 0);
return x.rc;
}
/* The generic subroutine is above. The code the follows implements
** the command-line interface.
*/
#ifdef DBDUMP_STANDALONE
#include <stdio.h>
/*
** Command-line interface
*/
int main(int argc, char **argv){
sqlite3 *db;
const char *zDb;
const char *zSchema;
const char *zTable = 0;
int rc;
if( argc<2 || argc>4 ){
fprintf(stderr, "Usage: %s DATABASE ?SCHEMA? ?TABLE?\n", argv[0]);
return 1;
}
zDb = argv[1];
zSchema = argc>=3 ? argv[2] : "main";
zTable = argc==4 ? argv[3] : 0;
rc = sqlite3_open(zDb, &db);
if( rc ){
fprintf(stderr, "Cannot open \"%s\": %s\n", zDb, sqlite3_errmsg(db));
sqlite3_close(db);
return 1;
}
rc = sqlite3_db_dump(db, zSchema, zTable,
(int(*)(const char*,void*))fputs, (void*)stdout);
if( rc ){
fprintf(stderr, "Error: sqlite3_db_dump() returns %d\n", rc);
}
sqlite3_close(db);
return rc!=SQLITE_OK;
}
#endif /* DBDUMP_STANDALONE */

4
ext/misc/fuzzer.c
View File

@@ -344,10 +344,10 @@ static int fuzzerLoadOneRule(
memset(pRule, 0, sizeof(*pRule));
pRule->zFrom = pRule->zTo;
pRule->zFrom += nTo + 1;
pRule->nFrom = nFrom;
pRule->nFrom = (fuzzer_len)nFrom;
memcpy(pRule->zFrom, zFrom, nFrom+1);
memcpy(pRule->zTo, zTo, nTo+1);
pRule->nTo = nTo;
pRule->nTo = (fuzzer_len)nTo;
pRule->rCost = nCost;
pRule->iRuleset = (int)iRuleset;
}

226
ext/misc/json1.c
View File

@@ -22,7 +22,7 @@
** how JSONB might improve on that.)
*/
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_JSON1)
#if !defined(_SQLITEINT_H_)
#if !defined(SQLITEINT_H)
#include "sqlite3ext.h"
#endif
SQLITE_EXTENSION_INIT1
@@ -49,13 +49,15 @@ SQLITE_EXTENSION_INIT1
#ifdef sqlite3Isdigit
/* Use the SQLite core versions if this routine is part of the
** SQLite amalgamation */
# define safe_isdigit(x) sqlite3Isdigit(x)
# define safe_isalnum(x) sqlite3Isalnum(x)
# define safe_isdigit(x) sqlite3Isdigit(x)
# define safe_isalnum(x) sqlite3Isalnum(x)
# define safe_isxdigit(x) sqlite3Isxdigit(x)
#else
/* Use the standard library for separate compilation */
#include <ctype.h> /* amalgamator: keep */
# define safe_isdigit(x) isdigit((unsigned char)(x))
# define safe_isalnum(x) isalnum((unsigned char)(x))
# define safe_isdigit(x) isdigit((unsigned char)(x))
# define safe_isalnum(x) isalnum((unsigned char)(x))
# define safe_isxdigit(x) isxdigit((unsigned char)(x))
#endif
/*
@@ -136,9 +138,10 @@ static const char * const jsonType[] = {
#define JNODE_RAW 0x01 /* Content is raw, not JSON encoded */
#define JNODE_ESCAPE 0x02 /* Content is text with \ escapes */
#define JNODE_REMOVE 0x04 /* Do not output */
#define JNODE_REPLACE 0x08 /* Replace with JsonNode.iVal */
#define JNODE_APPEND 0x10 /* More ARRAY/OBJECT entries at u.iAppend */
#define JNODE_LABEL 0x20 /* Is a label of an object */
#define JNODE_REPLACE 0x08 /* Replace with JsonNode.u.iReplace */
#define JNODE_PATCH 0x10 /* Patch with JsonNode.u.pPatch */
#define JNODE_APPEND 0x20 /* More ARRAY/OBJECT entries at u.iAppend */
#define JNODE_LABEL 0x40 /* Is a label of an object */
/* A single node of parsed JSON
@@ -146,12 +149,13 @@ static const char * const jsonType[] = {
struct JsonNode {
u8 eType; /* One of the JSON_ type values */
u8 jnFlags; /* JNODE flags */
u8 iVal; /* Replacement value when JNODE_REPLACE */
u32 n; /* Bytes of content, or number of sub-nodes */
union {
const char *zJContent; /* Content for INT, REAL, and STRING */
u32 iAppend; /* More terms for ARRAY and OBJECT */
u32 iKey; /* Key for ARRAY objects in json_tree() */
u32 iReplace; /* Replacement content for JNODE_REPLACE */
JsonNode *pPatch; /* Node chain of patch for JNODE_PATCH */
} u;
};
@@ -408,6 +412,13 @@ static void jsonRenderNode(
JsonString *pOut, /* Write JSON here */
sqlite3_value **aReplace /* Replacement values */
){
if( pNode->jnFlags & (JNODE_REPLACE|JNODE_PATCH) ){
if( pNode->jnFlags & JNODE_REPLACE ){
jsonAppendValue(pOut, aReplace[pNode->u.iReplace]);
return;
}
pNode = pNode->u.pPatch;
}
switch( pNode->eType ){
default: {
assert( pNode->eType==JSON_NULL );
@@ -439,12 +450,7 @@ static void jsonRenderNode(
jsonAppendChar(pOut, '[');
for(;;){
while( j<=pNode->n ){
if( pNode[j].jnFlags & (JNODE_REMOVE|JNODE_REPLACE) ){
if( pNode[j].jnFlags & JNODE_REPLACE ){
jsonAppendSeparator(pOut);
jsonAppendValue(pOut, aReplace[pNode[j].iVal]);
}
}else{
if( (pNode[j].jnFlags & JNODE_REMOVE)==0 ){
jsonAppendSeparator(pOut);
jsonRenderNode(&pNode[j], pOut, aReplace);
}
@@ -466,11 +472,7 @@ static void jsonRenderNode(
jsonAppendSeparator(pOut);
jsonRenderNode(&pNode[j], pOut, aReplace);
jsonAppendChar(pOut, ':');
if( pNode[j+1].jnFlags & JNODE_REPLACE ){
jsonAppendValue(pOut, aReplace[pNode[j+1].iVal]);
}else{
jsonRenderNode(&pNode[j+1], pOut, aReplace);
}
jsonRenderNode(&pNode[j+1], pOut, aReplace);
}
j += 1 + jsonNodeSize(&pNode[j+1]);
}
@@ -593,12 +595,13 @@ static void jsonReturn(
c = z[++i];
if( c=='u' ){
u32 v = 0, k;
for(k=0; k<4 && i<n-2; i++, k++){
for(k=0; k<4; i++, k++){
assert( i<n-2 );
c = z[i+1];
if( c>='0' && c<='9' ) v = v*16 + c - '0';
else if( c>='A' && c<='F' ) v = v*16 + c - 'A' + 10;
else if( c>='a' && c<='f' ) v = v*16 + c - 'a' + 10;
else break;
assert( safe_isxdigit(c) );
if( c<='9' ) v = v*16 + c - '0';
else if( c<='F' ) v = v*16 + c - 'A' + 10;
else v = v*16 + c - 'a' + 10;
}
if( v==0 ) break;
if( v<=0x7f ){
@@ -696,12 +699,20 @@ static int jsonParseAddNode(
p = &pParse->aNode[pParse->nNode];
p->eType = (u8)eType;
p->jnFlags = 0;
p->iVal = 0;
p->n = n;
p->u.zJContent = zContent;
return pParse->nNode++;
}
/*
** Return true if z[] begins with 4 (or more) hexadecimal digits
*/
static int jsonIs4Hex(const char *z){
int i;
for(i=0; i<4; i++) if( !safe_isxdigit(z[i]) ) return 0;
return 1;
}
/*
** Parse a single JSON value which begins at pParse->zJson[i]. Return the
** index of the first character past the end of the value parsed.
@@ -776,8 +787,13 @@ static int jsonParseValue(JsonParse *pParse, u32 i){
if( c==0 ) return -1;
if( c=='\\' ){
c = pParse->zJson[++j];
if( c==0 ) return -1;
jnFlags = JNODE_ESCAPE;
if( c=='"' || c=='\\' || c=='/' || c=='b' || c=='f'
|| c=='n' || c=='r' || c=='t'
|| (c=='u' && jsonIs4Hex(pParse->zJson+j+1)) ){
jnFlags = JNODE_ESCAPE;
}else{
return -1;
}
}else if( c=='"' ){
break;
}
@@ -1148,6 +1164,25 @@ static void jsonWrongNumArgs(
sqlite3_free(zMsg);
}
/*
** Mark all NULL entries in the Object passed in as JNODE_REMOVE.
*/
static void jsonRemoveAllNulls(JsonNode *pNode){
int i, n;
assert( pNode->eType==JSON_OBJECT );
n = pNode->n;
for(i=2; i<=n; i += jsonNodeSize(&pNode[i])+1){
switch( pNode[i].eType ){
case JSON_NULL:
pNode[i].jnFlags |= JNODE_REMOVE;
break;
case JSON_OBJECT:
jsonRemoveAllNulls(&pNode[i]);
break;
}
}
}
/****************************************************************************
** SQL functions used for testing and debugging
@@ -1211,6 +1246,26 @@ static void jsonTest1Func(
** Scalar SQL function implementations
****************************************************************************/
/*
** Implementation of the json_QUOTE(VALUE) function. Return a JSON value
** corresponding to the SQL value input. Mostly this means putting
** double-quotes around strings and returning the unquoted string "null"
** when given a NULL input.
*/
static void jsonQuoteFunc(
sqlite3_context *ctx,
int argc,
sqlite3_value **argv
){
JsonString jx;
UNUSED_PARAM(argc);
jsonInit(&jx, ctx);
jsonAppendValue(&jx, argv[0]);
jsonResult(&jx);
sqlite3_result_subtype(ctx, JSON_SUBTYPE);
}
/*
** Implementation of the json_array(VALUE,...) function. Return a JSON
** array that contains all values given in arguments. Or if any argument
@@ -1320,6 +1375,105 @@ static void jsonExtractFunc(
jsonParseReset(&x);
}
/* This is the RFC 7396 MergePatch algorithm.
*/
static JsonNode *jsonMergePatch(
JsonParse *pParse, /* The JSON parser that contains the TARGET */
int iTarget, /* Node of the TARGET in pParse */
JsonNode *pPatch /* The PATCH */
){
u32 i, j;
u32 iRoot;
JsonNode *pTarget;
if( pPatch->eType!=JSON_OBJECT ){
return pPatch;
}
assert( iTarget>=0 && iTarget<pParse->nNode );
pTarget = &pParse->aNode[iTarget];
assert( (pPatch->jnFlags & JNODE_APPEND)==0 );
if( pTarget->eType!=JSON_OBJECT ){
jsonRemoveAllNulls(pPatch);
return pPatch;
}
iRoot = iTarget;
for(i=1; i<pPatch->n; i += jsonNodeSize(&pPatch[i+1])+1){
u32 nKey;
const char *zKey;
assert( pPatch[i].eType==JSON_STRING );
assert( pPatch[i].jnFlags & JNODE_LABEL );
nKey = pPatch[i].n;
zKey = pPatch[i].u.zJContent;
assert( (pPatch[i].jnFlags & JNODE_RAW)==0 );
for(j=1; j<pTarget->n; j += jsonNodeSize(&pTarget[j+1])+1 ){
assert( pTarget[j].eType==JSON_STRING );
assert( pTarget[j].jnFlags & JNODE_LABEL );
assert( (pPatch[i].jnFlags & JNODE_RAW)==0 );
if( pTarget[j].n==nKey && strncmp(pTarget[j].u.zJContent,zKey,nKey)==0 ){
if( pTarget[j+1].jnFlags & (JNODE_REMOVE|JNODE_PATCH) ) break;
if( pPatch[i+1].eType==JSON_NULL ){
pTarget[j+1].jnFlags |= JNODE_REMOVE;
}else{
JsonNode *pNew = jsonMergePatch(pParse, iTarget+j+1, &pPatch[i+1]);
if( pNew==0 ) return 0;
pTarget = &pParse->aNode[iTarget];
if( pNew!=&pTarget[j+1] ){
pTarget[j+1].u.pPatch = pNew;
pTarget[j+1].jnFlags |= JNODE_PATCH;
}
}
break;
}
}
if( j>=pTarget->n && pPatch[i+1].eType!=JSON_NULL ){
int iStart, iPatch;
iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0);
jsonParseAddNode(pParse, JSON_STRING, nKey, zKey);
iPatch = jsonParseAddNode(pParse, JSON_TRUE, 0, 0);
if( pParse->oom ) return 0;
jsonRemoveAllNulls(pPatch);
pTarget = &pParse->aNode[iTarget];
pParse->aNode[iRoot].jnFlags |= JNODE_APPEND;
pParse->aNode[iRoot].u.iAppend = iStart - iRoot;
iRoot = iStart;
pParse->aNode[iPatch].jnFlags |= JNODE_PATCH;
pParse->aNode[iPatch].u.pPatch = &pPatch[i+1];
}
}
return pTarget;
}
/*
** Implementation of the json_mergepatch(JSON1,JSON2) function. Return a JSON
** object that is the result of running the RFC 7396 MergePatch() algorithm
** on the two arguments.
*/
static void jsonPatchFunc(
sqlite3_context *ctx,
int argc,
sqlite3_value **argv
){
JsonParse x; /* The JSON that is being patched */
JsonParse y; /* The patch */
JsonNode *pResult; /* The result of the merge */
UNUSED_PARAM(argc);
if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return;
if( jsonParse(&y, ctx, (const char*)sqlite3_value_text(argv[1])) ){
jsonParseReset(&x);
return;
}
pResult = jsonMergePatch(&x, 0, y.aNode);
assert( pResult!=0 || x.oom );
if( pResult ){
jsonReturnJson(pResult, ctx, 0);
}else{
sqlite3_result_error_nomem(ctx);
}
jsonParseReset(&x);
jsonParseReset(&y);
}
/*
** Implementation of the json_object(NAME,VALUE,...) function. Return a JSON
** object that contains all name/value given in arguments. Or if any name
@@ -1423,11 +1577,11 @@ static void jsonReplaceFunc(
if( x.nErr ) goto replace_err;
if( pNode ){
pNode->jnFlags |= (u8)JNODE_REPLACE;
pNode->iVal = (u8)(i+1);
pNode->u.iReplace = i + 1;
}
}
if( x.aNode[0].jnFlags & JNODE_REPLACE ){
sqlite3_result_value(ctx, argv[x.aNode[0].iVal]);
sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]);
}else{
jsonReturnJson(x.aNode, ctx, argv);
}
@@ -1477,11 +1631,11 @@ static void jsonSetFunc(
goto jsonSetDone;
}else if( pNode && (bApnd || bIsSet) ){
pNode->jnFlags |= (u8)JNODE_REPLACE;
pNode->iVal = (u8)(i+1);
pNode->u.iReplace = i + 1;
}
}
if( x.aNode[0].jnFlags & JNODE_REPLACE ){
sqlite3_result_value(ctx, argv[x.aNode[0].iVal]);
sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]);
}else{
jsonReturnJson(x.aNode, ctx, argv);
}
@@ -1625,7 +1779,7 @@ static void jsonObjectFinal(sqlite3_context *ctx){
if( pStr ){
jsonAppendChar(pStr, '}');
if( pStr->bErr ){
if( pStr->bErr==0 ) sqlite3_result_error_nomem(ctx);
if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx);
assert( pStr->bStatic );
}else{
sqlite3_result_text(ctx, pStr->zBuf, pStr->nUsed,
@@ -1903,9 +2057,9 @@ static int jsonEachColumn(
/* For json_each() path and root are the same so fall through
** into the root case */
}
case JEACH_ROOT: {
default: {
const char *zRoot = p->zRoot;
if( zRoot==0 ) zRoot = "$";
if( zRoot==0 ) zRoot = "$";
sqlite3_result_text(ctx, zRoot, -1, SQLITE_STATIC);
break;
}
@@ -2124,6 +2278,8 @@ int sqlite3Json1Init(sqlite3 *db){
{ "json_extract", -1, 0, jsonExtractFunc },
{ "json_insert", -1, 0, jsonSetFunc },
{ "json_object", -1, 0, jsonObjectFunc },
{ "json_patch", 2, 0, jsonPatchFunc },
{ "json_quote", 1, 0, jsonQuoteFunc },
{ "json_remove", -1, 0, jsonRemoveFunc },
{ "json_replace", -1, 0, jsonReplaceFunc },
{ "json_set", -1, 1, jsonSetFunc },

491
ext/misc/memvfs.c Normal file
View File

@@ -0,0 +1,491 @@
/*
** 2016-09-07
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** This is an in-memory read-only VFS implementation. The application
** supplies a block of memory which is the database file, and this VFS
** uses that block of memory.
**
** Because there is no place to store journals and no good way to lock
** the "file", this VFS is read-only.
**
** USAGE:
**
** sqlite3_open_v2("file:/whatever?ptr=0xf05538&sz=14336", &db,
** SQLITE_OPEN_READONLY | SQLITE_OPEN_URI,
** "memvfs");
**
** The ptr= and sz= query parameters are required or the open will fail.
** The ptr= parameter gives the memory address of the buffer holding the
** read-only database and sz= gives the size of the database. The parameter
** values may be in hexadecimal or decimal. The filename is ignored.
*/
#include <sqlite3ext.h>
SQLITE_EXTENSION_INIT1
#include <string.h>
#include <assert.h>
/*
** Forward declaration of objects used by this utility
*/
typedef struct sqlite3_vfs MemVfs;
typedef struct MemFile MemFile;
/* Access to a lower-level VFS that (might) implement dynamic loading,
** access to randomness, etc.
*/
#define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData))
/* An open file */
struct MemFile {
sqlite3_file base; /* IO methods */
sqlite3_int64 sz; /* Size of the file */
unsigned char *aData; /* content of the file */
};
/*
** Methods for MemFile
*/
static int memClose(sqlite3_file*);
static int memRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst);
static int memWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst);
static int memTruncate(sqlite3_file*, sqlite3_int64 size);
static int memSync(sqlite3_file*, int flags);
static int memFileSize(sqlite3_file*, sqlite3_int64 *pSize);
static int memLock(sqlite3_file*, int);
static int memUnlock(sqlite3_file*, int);
static int memCheckReservedLock(sqlite3_file*, int *pResOut);
static int memFileControl(sqlite3_file*, int op, void *pArg);
static int memSectorSize(sqlite3_file*);
static int memDeviceCharacteristics(sqlite3_file*);
static int memShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**);
static int memShmLock(sqlite3_file*, int offset, int n, int flags);
static void memShmBarrier(sqlite3_file*);
static int memShmUnmap(sqlite3_file*, int deleteFlag);
static int memFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp);
static int memUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p);
/*
** Methods for MemVfs
*/
static int memOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *);
static int memDelete(sqlite3_vfs*, const char *zName, int syncDir);
static int memAccess(sqlite3_vfs*, const char *zName, int flags, int *);
static int memFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut);
static void *memDlOpen(sqlite3_vfs*, const char *zFilename);
static void memDlError(sqlite3_vfs*, int nByte, char *zErrMsg);
static void (*memDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void);
static void memDlClose(sqlite3_vfs*, void*);
static int memRandomness(sqlite3_vfs*, int nByte, char *zOut);
static int memSleep(sqlite3_vfs*, int microseconds);
static int memCurrentTime(sqlite3_vfs*, double*);
static int memGetLastError(sqlite3_vfs*, int, char *);
static int memCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*);
static sqlite3_vfs mem_vfs = {
2, /* iVersion */
0, /* szOsFile (set when registered) */
1024, /* mxPathname */
0, /* pNext */
"memvfs", /* zName */
0, /* pAppData (set when registered) */
memOpen, /* xOpen */
memDelete, /* xDelete */
memAccess, /* xAccess */
memFullPathname, /* xFullPathname */
memDlOpen, /* xDlOpen */
memDlError, /* xDlError */
memDlSym, /* xDlSym */
memDlClose, /* xDlClose */
memRandomness, /* xRandomness */
memSleep, /* xSleep */
memCurrentTime, /* xCurrentTime */
memGetLastError, /* xGetLastError */
memCurrentTimeInt64 /* xCurrentTimeInt64 */
};
static const sqlite3_io_methods mem_io_methods = {
3, /* iVersion */
memClose, /* xClose */
memRead, /* xRead */
memWrite, /* xWrite */
memTruncate, /* xTruncate */
memSync, /* xSync */
memFileSize, /* xFileSize */
memLock, /* xLock */
memUnlock, /* xUnlock */
memCheckReservedLock, /* xCheckReservedLock */
memFileControl, /* xFileControl */
memSectorSize, /* xSectorSize */
memDeviceCharacteristics, /* xDeviceCharacteristics */
memShmMap, /* xShmMap */
memShmLock, /* xShmLock */
memShmBarrier, /* xShmBarrier */
memShmUnmap, /* xShmUnmap */
memFetch, /* xFetch */
memUnfetch /* xUnfetch */
};
/*
** Close an mem-file.
**
** The pData pointer is owned by the application, so there is nothing
** to free.
*/
static int memClose(sqlite3_file *pFile){
return SQLITE_OK;
}
/*
** Read data from an mem-file.
*/
static int memRead(
sqlite3_file *pFile,
void *zBuf,
int iAmt,
sqlite_int64 iOfst
){
MemFile *p = (MemFile *)pFile;
memcpy(zBuf, p->aData+iOfst, iAmt);
return SQLITE_OK;
}
/*
** Write data to an mem-file.
*/
static int memWrite(
sqlite3_file *pFile,
const void *z,
int iAmt,
sqlite_int64 iOfst
){
return SQLITE_READONLY;
}
/*
** Truncate an mem-file.
*/
static int memTruncate(sqlite3_file *pFile, sqlite_int64 size){
return SQLITE_READONLY;
}
/*
** Sync an mem-file.
*/
static int memSync(sqlite3_file *pFile, int flags){
return SQLITE_READONLY;
}
/*
** Return the current file-size of an mem-file.
*/
static int memFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){
MemFile *p = (MemFile *)pFile;
*pSize = p->sz;
return SQLITE_OK;
}
/*
** Lock an mem-file.
*/
static int memLock(sqlite3_file *pFile, int eLock){
return SQLITE_READONLY;
}
/*
** Unlock an mem-file.
*/
static int memUnlock(sqlite3_file *pFile, int eLock){
return SQLITE_OK;
}
/*
** Check if another file-handle holds a RESERVED lock on an mem-file.
*/
static int memCheckReservedLock(sqlite3_file *pFile, int *pResOut){
*pResOut = 0;
return SQLITE_OK;
}
/*
** File control method. For custom operations on an mem-file.
*/
static int memFileControl(sqlite3_file *pFile, int op, void *pArg){
MemFile *p = (MemFile *)pFile;
int rc = SQLITE_NOTFOUND;
if( op==SQLITE_FCNTL_VFSNAME ){
*(char**)pArg = sqlite3_mprintf("mem(%p,%lld)", p->aData, p->sz);
rc = SQLITE_OK;
}
return rc;
}
/*
** Return the sector-size in bytes for an mem-file.
*/
static int memSectorSize(sqlite3_file *pFile){
return 1024;
}
/*
** Return the device characteristic flags supported by an mem-file.
*/
static int memDeviceCharacteristics(sqlite3_file *pFile){
return SQLITE_IOCAP_IMMUTABLE;
}
/* Create a shared memory file mapping */
static int memShmMap(
sqlite3_file *pFile,
int iPg,
int pgsz,
int bExtend,
void volatile **pp
){
return SQLITE_READONLY;
}
/* Perform locking on a shared-memory segment */
static int memShmLock(sqlite3_file *pFile, int offset, int n, int flags){
return SQLITE_READONLY;
}
/* Memory barrier operation on shared memory */
static void memShmBarrier(sqlite3_file *pFile){
return;
}
/* Unmap a shared memory segment */
static int memShmUnmap(sqlite3_file *pFile, int deleteFlag){
return SQLITE_OK;
}
/* Fetch a page of a memory-mapped file */
static int memFetch(
sqlite3_file *pFile,
sqlite3_int64 iOfst,
int iAmt,
void **pp
){
MemFile *p = (MemFile *)pFile;
*pp = (void*)(p->aData + iOfst);
return SQLITE_OK;
}
/* Release a memory-mapped page */
static int memUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){
return SQLITE_OK;
}
/*
** Open an mem file handle.
*/
static int memOpen(
sqlite3_vfs *pVfs,
const char *zName,
sqlite3_file *pFile,
int flags,
int *pOutFlags
){
MemFile *p = (MemFile*)pFile;
memset(p, 0, sizeof(*p));
if( (flags & SQLITE_OPEN_MAIN_DB)==0 ) return SQLITE_CANTOPEN;
p->aData = (unsigned char*)sqlite3_uri_int64(zName,"ptr",0);
if( p->aData==0 ) return SQLITE_CANTOPEN;
p->sz = sqlite3_uri_int64(zName,"sz",0);
if( p->sz<0 ) return SQLITE_CANTOPEN;
pFile->pMethods = &mem_io_methods;
return SQLITE_OK;
}
/*
** Delete the file located at zPath. If the dirSync argument is true,
** ensure the file-system modifications are synced to disk before
** returning.
*/
static int memDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){
return SQLITE_READONLY;
}
/*
** Test for access permissions. Return true if the requested permission
** is available, or false otherwise.
*/
static int memAccess(
sqlite3_vfs *pVfs,
const char *zPath,
int flags,
int *pResOut
){
/* The spec says there are three possible values for flags. But only
** two of them are actually used */
assert( flags==SQLITE_ACCESS_EXISTS || flags==SQLITE_ACCESS_READWRITE );
if( flags==SQLITE_ACCESS_READWRITE ){
*pResOut = 0;
}else{
*pResOut = 1;
}
return SQLITE_OK;
}
/*
** Populate buffer zOut with the full canonical pathname corresponding
** to the pathname in zPath. zOut is guaranteed to point to a buffer
** of at least (INST_MAX_PATHNAME+1) bytes.
*/
static int memFullPathname(
sqlite3_vfs *pVfs,
const char *zPath,
int nOut,
char *zOut
){
sqlite3_snprintf(nOut, zOut, "%s", zPath);
return SQLITE_OK;
}
/*
** Open the dynamic library located at zPath and return a handle.
*/
static void *memDlOpen(sqlite3_vfs *pVfs, const char *zPath){
return ORIGVFS(pVfs)->xDlOpen(ORIGVFS(pVfs), zPath);
}
/*
** Populate the buffer zErrMsg (size nByte bytes) with a human readable
** utf-8 string describing the most recent error encountered associated
** with dynamic libraries.
*/
static void memDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){
ORIGVFS(pVfs)->xDlError(ORIGVFS(pVfs), nByte, zErrMsg);
}
/*
** Return a pointer to the symbol zSymbol in the dynamic library pHandle.
*/
static void (*memDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){
return ORIGVFS(pVfs)->xDlSym(ORIGVFS(pVfs), p, zSym);
}
/*
** Close the dynamic library handle pHandle.
*/
static void memDlClose(sqlite3_vfs *pVfs, void *pHandle){
ORIGVFS(pVfs)->xDlClose(ORIGVFS(pVfs), pHandle);
}
/*
** Populate the buffer pointed to by zBufOut with nByte bytes of
** random data.
*/
static int memRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){
return ORIGVFS(pVfs)->xRandomness(ORIGVFS(pVfs), nByte, zBufOut);
}
/*
** Sleep for nMicro microseconds. Return the number of microseconds
** actually slept.
*/
static int memSleep(sqlite3_vfs *pVfs, int nMicro){
return ORIGVFS(pVfs)->xSleep(ORIGVFS(pVfs), nMicro);
}
/*
** Return the current time as a Julian Day number in *pTimeOut.
*/
static int memCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){
return ORIGVFS(pVfs)->xCurrentTime(ORIGVFS(pVfs), pTimeOut);
}
static int memGetLastError(sqlite3_vfs *pVfs, int a, char *b){
return ORIGVFS(pVfs)->xGetLastError(ORIGVFS(pVfs), a, b);
}
static int memCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){
return ORIGVFS(pVfs)->xCurrentTimeInt64(ORIGVFS(pVfs), p);
}
#ifdef MEMVFS_TEST
/*
** memload(FILENAME)
**
** This an SQL function used to help in testing the memvfs VFS. The
** function reads the content of a file into memory and then returns
** a string that gives the locate and size of the in-memory buffer.
*/
#include <stdio.h>
static void memvfsMemloadFunc(
sqlite3_context *context,
int argc,
sqlite3_value **argv
){
unsigned char *p;
sqlite3_int64 sz;
FILE *in;
const char *zFilename = (const char*)sqlite3_value_text(argv[0]);
char zReturn[100];
if( zFilename==0 ) return;
in = fopen(zFilename, "rb");
if( in==0 ) return;
fseek(in, 0, SEEK_END);
sz = ftell(in);
rewind(in);
p = sqlite3_malloc( sz );
if( p==0 ){
fclose(in);
sqlite3_result_error_nomem(context);
return;
}
fread(p, sz, 1, in);
fclose(in);
sqlite3_snprintf(sizeof(zReturn),zReturn,"ptr=%lld&sz=%lld",
(sqlite3_int64)p, sz);
sqlite3_result_text(context, zReturn, -1, SQLITE_TRANSIENT);
}
/* Called for each new database connection */
static int memvfsRegister(
sqlite3 *db,
const char **pzErrMsg,
const struct sqlite3_api_routines *pThunk
){
return sqlite3_create_function(db, "memload", 1, SQLITE_UTF8, 0,
memvfsMemloadFunc, 0, 0);
}
#endif /* MEMVFS_TEST */
#ifdef _WIN32
__declspec(dllexport)
#endif
/*
** This routine is called when the extension is loaded.
** Register the new VFS.
*/
int sqlite3_memvfs_init(
sqlite3 *db,
char **pzErrMsg,
const sqlite3_api_routines *pApi
){
int rc = SQLITE_OK;
SQLITE_EXTENSION_INIT2(pApi);
mem_vfs.pAppData = sqlite3_vfs_find(0);
mem_vfs.szOsFile = sizeof(MemFile);
rc = sqlite3_vfs_register(&mem_vfs, 1);
#ifdef MEMVFS_TEST
if( rc==SQLITE_OK ){
rc = sqlite3_auto_extension((void(*)(void))memvfsRegister);
}
#endif
if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY;
return rc;
}

2
ext/misc/percentile.c
View File

@@ -167,7 +167,7 @@ static void percentStep(sqlite3_context *pCtx, int argc, sqlite3_value **argv){
/*
** Compare to doubles for sorting using qsort()
*/
static int doubleCmp(const void *pA, const void *pB){
static int SQLITE_CDECL doubleCmp(const void *pA, const void *pB){
double a = *(double*)pA;
double b = *(double*)pB;
if( a==b ) return 0;

10
ext/misc/regexp.c
View File

@@ -136,7 +136,7 @@ struct ReCompiled {
static void re_add_state(ReStateSet *pSet, int newState){
unsigned i;
for(i=0; i<pSet->nState; i++) if( pSet->aState[i]==newState ) return;
pSet->aState[pSet->nState++] = newState;
pSet->aState[pSet->nState++] = (ReStateNumber)newState;
}
/* Extract the next unicode character from *pzIn and return it. Advance
@@ -358,7 +358,7 @@ static int re_insert(ReCompiled *p, int iBefore, int op, int arg){
p->aArg[i] = p->aArg[i-1];
}
p->nState++;
p->aOp[iBefore] = op;
p->aOp[iBefore] = (char)op;
p->aArg[iBefore] = arg;
return iBefore;
}
@@ -677,12 +677,12 @@ const char *re_compile(ReCompiled **ppRe, const char *zIn, int noCase){
for(j=0, i=1; j<sizeof(pRe->zInit)-2 && pRe->aOp[i]==RE_OP_MATCH; i++){
unsigned x = pRe->aArg[i];
if( x<=127 ){
pRe->zInit[j++] = x;
pRe->zInit[j++] = (unsigned char)x;
}else if( x<=0xfff ){
pRe->zInit[j++] = 0xc0 | (x>>6);
pRe->zInit[j++] = (unsigned char)(0xc0 | (x>>6));
pRe->zInit[j++] = 0x80 | (x&0x3f);
}else if( x<=0xffff ){
pRe->zInit[j++] = 0xd0 | (x>>12);
pRe->zInit[j++] = (unsigned char)(0xd0 | (x>>12));
pRe->zInit[j++] = 0x80 | ((x>>6)&0x3f);
pRe->zInit[j++] = 0x80 | (x&0x3f);
}else{

68
ext/misc/remember.c Normal file
View File

@@ -0,0 +1,68 @@
/*
** 2016-08-09
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
**
** This file demonstrates how to create an SQL function that is a pass-through
** for integer values (it returns a copy of its argument) but also saves the
** value that is passed through into a C-language variable. The address of
** the C-language variable is supplied as the second argument.
**
** This allows, for example, a counter to incremented and the original
** value retrieved, atomically, using a single statement:
**
** UPDATE counterTab SET cnt=remember(cnt,$PTR)+1 WHERE id=$ID
**
** Prepare the above statement once. Then to use it, bind the address
** of the output variable to $PTR and the id of the counter to $ID and
** run the prepared statement.
**
** One can imagine doing similar things with floating-point values and
** strings, but this demonstration extension will stick to using just
** integers.
*/
#include "sqlite3ext.h"
SQLITE_EXTENSION_INIT1
#include <assert.h>
/*
** remember(V,PTR)
**
** Return the integer value V. Also save the value of V in a
** C-language variable whose address is PTR.
*/
static void rememberFunc(
sqlite3_context *pCtx,
int argc,
sqlite3_value **argv
){
sqlite3_int64 v;
sqlite3_int64 ptr;
assert( argc==2 );
v = sqlite3_value_int64(argv[0]);
ptr = sqlite3_value_int64(argv[1]);
*((sqlite3_int64*)ptr) = v;
sqlite3_result_int64(pCtx, v);
}
#ifdef _WIN32
__declspec(dllexport)
#endif
int sqlite3_remember_init(
sqlite3 *db,
char **pzErrMsg,
const sqlite3_api_routines *pApi
){
int rc = SQLITE_OK;
SQLITE_EXTENSION_INIT2(pApi);
rc = sqlite3_create_function(db, "remember", 2, SQLITE_UTF8, 0,
rememberFunc, 0, 0);
return rc;
}

610
ext/misc/scrub.c Normal file
View File

@@ -0,0 +1,610 @@
/*
** 2016-05-05
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** This file implements a utility function (and a utility program) that
** makes a copy of an SQLite database while simultaneously zeroing out all
** deleted content.
**
** Normally (when PRAGMA secure_delete=OFF, which is the default) when SQLite
** deletes content, it does not overwrite the deleted content but rather marks
** the region of the file that held that content as being reusable. This can
** cause deleted content to recoverable from the database file. This stale
** content is removed by the VACUUM command, but VACUUM can be expensive for
** large databases. When in PRAGMA secure_delete=ON mode, the deleted content
** is zeroed, but secure_delete=ON has overhead as well.
**
** This utility attempts to make a copy of a complete SQLite database where
** all of the deleted content is zeroed out in the copy, and it attempts to
** do so while being faster than running VACUUM.
**
** Usage:
**
** int sqlite3_scrub_backup(
** const char *zSourceFile, // Source database filename
** const char *zDestFile, // Destination database filename
** char **pzErrMsg // Write error message here
** );
**
** Simply call the API above specifying the filename of the source database
** and the name of the backup copy. The source database must already exist
** and can be in active use. (A read lock is held during the backup.) The
** destination file should not previously exist. If the pzErrMsg parameter
** is non-NULL and if an error occurs, then an error message might be written
** into memory obtained from sqlite3_malloc() and *pzErrMsg made to point to
** that error message. But if the error is an OOM, the error might not be
** reported. The routine always returns non-zero if there is an error.
**
** If compiled with -DSCRUB_STANDALONE then a main() procedure is added and
** this file becomes a standalone program that can be run as follows:
**
** ./sqlite3scrub SOURCE DEST
*/
#include "sqlite3.h"
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <stdarg.h>
#include <string.h>
typedef struct ScrubState ScrubState;
typedef unsigned char u8;
typedef unsigned short u16;
typedef unsigned int u32;
/* State information for a scrub-and-backup operation */
struct ScrubState {
const char *zSrcFile; /* Name of the source file */
const char *zDestFile; /* Name of the destination file */
int rcErr; /* Error code */
char *zErr; /* Error message text */
sqlite3 *dbSrc; /* Source database connection */
sqlite3_file *pSrc; /* Source file handle */
sqlite3 *dbDest; /* Destination database connection */
sqlite3_file *pDest; /* Destination file handle */
u32 szPage; /* Page size */
u32 szUsable; /* Usable bytes on each page */
u32 nPage; /* Number of pages */
u32 iLastPage; /* Page number of last page written so far*/
u8 *page1; /* Content of page 1 */
};
/* Store an error message */
static void scrubBackupErr(ScrubState *p, const char *zFormat, ...){
va_list ap;
sqlite3_free(p->zErr);
va_start(ap, zFormat);
p->zErr = sqlite3_vmprintf(zFormat, ap);
va_end(ap);
if( p->rcErr==0 ) p->rcErr = SQLITE_ERROR;
}
/* Allocate memory to hold a single page of content */
static u8 *scrubBackupAllocPage(ScrubState *p){
u8 *pPage;
if( p->rcErr ) return 0;
pPage = sqlite3_malloc( p->szPage );
if( pPage==0 ) p->rcErr = SQLITE_NOMEM;
return pPage;
}
/* Read a page from the source database into memory. Use the memory
** provided by pBuf if not NULL or allocate a new page if pBuf==NULL.
*/
static u8 *scrubBackupRead(ScrubState *p, int pgno, u8 *pBuf){
int rc;
sqlite3_int64 iOff;
u8 *pOut = pBuf;
if( p->rcErr ) return 0;
if( pOut==0 ){
pOut = scrubBackupAllocPage(p);
if( pOut==0 ) return 0;
}
iOff = (pgno-1)*(sqlite3_int64)p->szPage;
rc = p->pSrc->pMethods->xRead(p->pSrc, pOut, p->szPage, iOff);
if( rc!=SQLITE_OK ){
if( pBuf==0 ) sqlite3_free(pOut);
pOut = 0;
scrubBackupErr(p, "read failed for page %d", pgno);
p->rcErr = SQLITE_IOERR;
}
return pOut;
}
/* Write a page to the destination database */
static void scrubBackupWrite(ScrubState *p, int pgno, const u8 *pData){
int rc;
sqlite3_int64 iOff;
if( p->rcErr ) return;
iOff = (pgno-1)*(sqlite3_int64)p->szPage;
rc = p->pDest->pMethods->xWrite(p->pDest, pData, p->szPage, iOff);
if( rc!=SQLITE_OK ){
scrubBackupErr(p, "write failed for page %d", pgno);
p->rcErr = SQLITE_IOERR;
}
if( pgno>p->iLastPage ) p->iLastPage = pgno;
}
/* Prepare a statement against the "db" database. */
static sqlite3_stmt *scrubBackupPrepare(
ScrubState *p, /* Backup context */
sqlite3 *db, /* Database to prepare against */
const char *zSql /* SQL statement */
){
sqlite3_stmt *pStmt;
if( p->rcErr ) return 0;
p->rcErr = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
if( p->rcErr ){
scrubBackupErr(p, "SQL error \"%s\" on \"%s\"",
sqlite3_errmsg(db), zSql);
sqlite3_finalize(pStmt);
return 0;
}
return pStmt;
}
/* Open the source database file */
static void scrubBackupOpenSrc(ScrubState *p){
sqlite3_stmt *pStmt;
int rc;
/* Open the source database file */
p->rcErr = sqlite3_open_v2(p->zSrcFile, &p->dbSrc,
SQLITE_OPEN_READWRITE |
SQLITE_OPEN_URI | SQLITE_OPEN_PRIVATECACHE, 0);
if( p->rcErr ){
scrubBackupErr(p, "cannot open source database: %s",
sqlite3_errmsg(p->dbSrc));
return;
}
p->rcErr = sqlite3_exec(p->dbSrc, "SELECT 1 FROM sqlite_master; BEGIN;",
0, 0, 0);
if( p->rcErr ){
scrubBackupErr(p,
"cannot start a read transaction on the source database: %s",
sqlite3_errmsg(p->dbSrc));
return;
}
rc = sqlite3_wal_checkpoint_v2(p->dbSrc, "main", SQLITE_CHECKPOINT_FULL,
0, 0);
if( rc ){
scrubBackupErr(p, "cannot checkpoint the source database");
return;
}
pStmt = scrubBackupPrepare(p, p->dbSrc, "PRAGMA page_size");
if( pStmt==0 ) return;
rc = sqlite3_step(pStmt);
if( rc==SQLITE_ROW ){
p->szPage = sqlite3_column_int(pStmt, 0);
}else{
scrubBackupErr(p, "unable to determine the page size");
}
sqlite3_finalize(pStmt);
if( p->rcErr ) return;
pStmt = scrubBackupPrepare(p, p->dbSrc, "PRAGMA page_count");
if( pStmt==0 ) return;
rc = sqlite3_step(pStmt);
if( rc==SQLITE_ROW ){
p->nPage = sqlite3_column_int(pStmt, 0);
}else{
scrubBackupErr(p, "unable to determine the size of the source database");
}
sqlite3_finalize(pStmt);
sqlite3_file_control(p->dbSrc, "main", SQLITE_FCNTL_FILE_POINTER, &p->pSrc);
if( p->pSrc==0 || p->pSrc->pMethods==0 ){
scrubBackupErr(p, "cannot get the source file handle");
p->rcErr = SQLITE_ERROR;
}
}
/* Create and open the destination file */
static void scrubBackupOpenDest(ScrubState *p){
sqlite3_stmt *pStmt;
int rc;
char *zSql;
if( p->rcErr ) return;
p->rcErr = sqlite3_open_v2(p->zDestFile, &p->dbDest,
SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE |
SQLITE_OPEN_URI | SQLITE_OPEN_PRIVATECACHE, 0);
if( p->rcErr ){
scrubBackupErr(p, "cannot open destination database: %s",
sqlite3_errmsg(p->dbDest));
return;
}
zSql = sqlite3_mprintf("PRAGMA page_size(%u);", p->szPage);
if( zSql==0 ){
p->rcErr = SQLITE_NOMEM;
return;
}
p->rcErr = sqlite3_exec(p->dbDest, zSql, 0, 0, 0);
sqlite3_free(zSql);
if( p->rcErr ){
scrubBackupErr(p,
"cannot set the page size on the destination database: %s",
sqlite3_errmsg(p->dbDest));
return;
}
sqlite3_exec(p->dbDest, "PRAGMA journal_mode=OFF;", 0, 0, 0);
p->rcErr = sqlite3_exec(p->dbDest, "BEGIN EXCLUSIVE;", 0, 0, 0);
if( p->rcErr ){
scrubBackupErr(p,
"cannot start a write transaction on the destination database: %s",
sqlite3_errmsg(p->dbDest));
return;
}
pStmt = scrubBackupPrepare(p, p->dbDest, "PRAGMA page_count;");
if( pStmt==0 ) return;
rc = sqlite3_step(pStmt);
if( rc!=SQLITE_ROW ){
scrubBackupErr(p, "cannot measure the size of the destination");
}else if( sqlite3_column_int(pStmt, 0)>1 ){
scrubBackupErr(p, "destination database is not empty - holds %d pages",
sqlite3_column_int(pStmt, 0));
}
sqlite3_finalize(pStmt);
sqlite3_file_control(p->dbDest, "main", SQLITE_FCNTL_FILE_POINTER, &p->pDest);
if( p->pDest==0 || p->pDest->pMethods==0 ){
scrubBackupErr(p, "cannot get the destination file handle");
p->rcErr = SQLITE_ERROR;
}
}
/* Read a 32-bit big-endian integer */
static u32 scrubBackupInt32(const u8 *a){
u32 v = a[3];
v += ((u32)a[2])<<8;
v += ((u32)a[1])<<16;
v += ((u32)a[0])<<24;
return v;
}
/* Read a 16-bit big-endian integer */
static u32 scrubBackupInt16(const u8 *a){
return (a[0]<<8) + a[1];
}
/*
** Read a varint. Put the value in *pVal and return the number of bytes.
*/
static int scrubBackupVarint(const u8 *z, sqlite3_int64 *pVal){
sqlite3_int64 v = 0;
int i;
for(i=0; i<8; i++){
v = (v<<7) + (z[i]&0x7f);
if( (z[i]&0x80)==0 ){ *pVal = v; return i+1; }
}
v = (v<<8) + (z[i]&0xff);
*pVal = v;
return 9;
}
/*
** Return the number of bytes in a varint.
*/
static int scrubBackupVarintSize(const u8 *z){
int i;
for(i=0; i<8; i++){
if( (z[i]&0x80)==0 ){ return i+1; }
}
return 9;
}
/*
** Copy the freelist trunk page given, and all its descendents,
** zeroing out as much as possible in the process.
*/
static void scrubBackupFreelist(ScrubState *p, int pgno, u32 nFree){
u8 *a, *aBuf;
u32 n, mx;
if( p->rcErr ) return;
aBuf = scrubBackupAllocPage(p);
if( aBuf==0 ) return;
while( pgno && nFree){
a = scrubBackupRead(p, pgno, aBuf);
if( a==0 ) break;
n = scrubBackupInt32(&a[4]);
mx = p->szUsable/4 - 2;
if( n<mx ){
memset(&a[n*4+8], 0, 4*(mx-n));
}
scrubBackupWrite(p, pgno, a);
pgno = scrubBackupInt32(a);
#if 0
/* There is really no point in copying the freelist leaf pages.
** Simply leave them uninitialized in the destination database. The
** OS filesystem should zero those pages for us automatically.
*/
for(i=0; i<n && nFree; i++){
u32 iLeaf = scrubBackupInt32(&a[i*4+8]);
if( aZero==0 ){
aZero = scrubBackupAllocPage(p);
if( aZero==0 ){ pgno = 0; break; }
memset(aZero, 0, p->szPage);
}
scrubBackupWrite(p, iLeaf, aZero);
nFree--;
}
#endif
}
sqlite3_free(aBuf);
}
/*
** Copy an overflow chain from source to destination. Zero out any
** unused tail at the end of the overflow chain.
*/
static void scrubBackupOverflow(ScrubState *p, int pgno, u32 nByte){
u8 *a, *aBuf;
aBuf = scrubBackupAllocPage(p);
if( aBuf==0 ) return;
while( nByte>0 && pgno!=0 ){
a = scrubBackupRead(p, pgno, aBuf);
if( a==0 ) break;
if( nByte >= (p->szUsable)-4 ){
nByte -= (p->szUsable) - 4;
}else{
u32 x = (p->szUsable - 4) - nByte;
u32 i = p->szUsable - x;
memset(&a[i], 0, x);
nByte = 0;
}
scrubBackupWrite(p, pgno, a);
pgno = scrubBackupInt32(a);
}
sqlite3_free(aBuf);
}
/*
** Copy B-Tree page pgno, and all of its children, from source to destination.
** Zero out deleted content during the copy.
*/
static void scrubBackupBtree(ScrubState *p, int pgno, int iDepth){
u8 *a;
u32 i, n, pc;
u32 nCell;
u32 nPrefix;
u32 szHdr;
u32 iChild;
u8 *aTop;
u8 *aCell;
u32 x, y;
int ln = 0;
if( p->rcErr ) return;
if( iDepth>50 ){
scrubBackupErr(p, "corrupt: b-tree too deep at page %d", pgno);
return;
}
if( pgno==1 ){
a = p->page1;
}else{
a = scrubBackupRead(p, pgno, 0);
if( a==0 ) return;
}
nPrefix = pgno==1 ? 100 : 0;
aTop = &a[nPrefix];
szHdr = 8 + 4*(aTop[0]==0x02 || aTop[0]==0x05);
aCell = aTop + szHdr;
nCell = scrubBackupInt16(&aTop[3]);
/* Zero out the gap between the cell index and the start of the
** cell content area */
x = scrubBackupInt16(&aTop[5]); /* First byte of cell content area */
if( x>p->szUsable ){ ln=__LINE__; goto btree_corrupt; }
y = szHdr + nPrefix + nCell*2;
if( y>x ){ ln=__LINE__; goto btree_corrupt; }
if( y<x ) memset(a+y, 0, x-y); /* Zero the gap */
/* Zero out all the free blocks */
pc = scrubBackupInt16(&aTop[1]);
if( pc>0 && pc<x ){ ln=__LINE__; goto btree_corrupt; }
while( pc ){
if( pc>(p->szUsable)-4 ){ ln=__LINE__; goto btree_corrupt; }
n = scrubBackupInt16(&a[pc+2]);
if( pc+n>(p->szUsable) ){ ln=__LINE__; goto btree_corrupt; }
if( n>4 ) memset(&a[pc+4], 0, n-4);
x = scrubBackupInt16(&a[pc]);
if( x<pc+4 && x>0 ){ ln=__LINE__; goto btree_corrupt; }
pc = x;
}
/* Write this one page */
scrubBackupWrite(p, pgno, a);
/* Walk the tree and process child pages */
for(i=0; i<nCell; i++){
u32 X, M, K, nLocal;
sqlite3_int64 P;
pc = scrubBackupInt16(&aCell[i*2]);
if( pc <= szHdr ){ ln=__LINE__; goto btree_corrupt; }
if( pc > p->szUsable-3 ){ ln=__LINE__; goto btree_corrupt; }
if( aTop[0]==0x05 || aTop[0]==0x02 ){
if( pc+4 > p->szUsable ){ ln=__LINE__; goto btree_corrupt; }
iChild = scrubBackupInt32(&a[pc]);
pc += 4;
scrubBackupBtree(p, iChild, iDepth+1);
if( aTop[0]==0x05 ) continue;
}
pc += scrubBackupVarint(&a[pc], &P);
if( pc >= p->szUsable ){ ln=__LINE__; goto btree_corrupt; }
if( aTop[0]==0x0d ){
X = p->szUsable - 35;
}else{
X = ((p->szUsable - 12)*64/255) - 23;
}
if( P<=X ){
/* All content is local. No overflow */
continue;
}
M = ((p->szUsable - 12)*32/255)-23;
K = M + ((P-M)%(p->szUsable-4));
if( aTop[0]==0x0d ){
pc += scrubBackupVarintSize(&a[pc]);
if( pc > (p->szUsable-4) ){ ln=__LINE__; goto btree_corrupt; }
}
nLocal = K<=X ? K : M;
if( pc+nLocal > p->szUsable-4 ){ ln=__LINE__; goto btree_corrupt; }
iChild = scrubBackupInt32(&a[pc+nLocal]);
scrubBackupOverflow(p, iChild, P-nLocal);
}
/* Walk the right-most tree */
if( aTop[0]==0x05 || aTop[0]==0x02 ){
iChild = scrubBackupInt32(&aTop[8]);
scrubBackupBtree(p, iChild, iDepth+1);
}
/* All done */
if( pgno>1 ) sqlite3_free(a);
return;
btree_corrupt:
scrubBackupErr(p, "corruption on page %d of source database (errid=%d)",
pgno, ln);
if( pgno>1 ) sqlite3_free(a);
}
/*
** Copy all ptrmap pages from source to destination.
** This routine is only called if the source database is in autovacuum
** or incremental vacuum mode.
*/
static void scrubBackupPtrmap(ScrubState *p){
u32 pgno = 2;
u32 J = p->szUsable/5;
u32 iLock = (1073742335/p->szPage)+1;
u8 *a, *pBuf;
if( p->rcErr ) return;
pBuf = scrubBackupAllocPage(p);
if( pBuf==0 ) return;
while( pgno<=p->nPage ){
a = scrubBackupRead(p, pgno, pBuf);
if( a==0 ) break;
scrubBackupWrite(p, pgno, a);
pgno += J+1;
if( pgno==iLock ) pgno++;
}
sqlite3_free(pBuf);
}
int sqlite3_scrub_backup(
const char *zSrcFile, /* Source file */
const char *zDestFile, /* Destination file */
char **pzErr /* Write error here if non-NULL */
){
ScrubState s;
u32 n, i;
sqlite3_stmt *pStmt;
memset(&s, 0, sizeof(s));
s.zSrcFile = zSrcFile;
s.zDestFile = zDestFile;
/* Open both source and destination databases */
scrubBackupOpenSrc(&s);
scrubBackupOpenDest(&s);
/* Read in page 1 */
s.page1 = scrubBackupRead(&s, 1, 0);
if( s.page1==0 ) goto scrub_abort;
s.szUsable = s.szPage - s.page1[20];
/* Copy the freelist */
n = scrubBackupInt32(&s.page1[36]);
i = scrubBackupInt32(&s.page1[32]);
if( n ) scrubBackupFreelist(&s, i, n);
/* Copy ptrmap pages */
n = scrubBackupInt32(&s.page1[52]);
if( n ) scrubBackupPtrmap(&s);
/* Copy all of the btrees */
scrubBackupBtree(&s, 1, 0);
pStmt = scrubBackupPrepare(&s, s.dbSrc,
"SELECT rootpage FROM sqlite_master WHERE coalesce(rootpage,0)>0");
if( pStmt==0 ) goto scrub_abort;
while( sqlite3_step(pStmt)==SQLITE_ROW ){
i = (u32)sqlite3_column_int(pStmt, 0);
scrubBackupBtree(&s, i, 0);
}
sqlite3_finalize(pStmt);
/* If the last page of the input db file is a free-list leaf, then the
** backup file on disk is still smaller than the size indicated within
** the database header. In this case, write a page of zeroes to the
** last page of the backup database so that SQLite does not mistakenly
** think the db is corrupt. */
if( s.iLastPage<s.nPage ){
u8 *aZero = scrubBackupAllocPage(&s);
if( aZero ){
memset(aZero, 0, s.szPage);
scrubBackupWrite(&s, s.nPage, aZero);
sqlite3_free(aZero);
}
}
scrub_abort:
/* Close the destination database without closing the transaction. If we
** commit, page zero will be overwritten. */
sqlite3_close(s.dbDest);
/* But do close out the read-transaction on the source database */
sqlite3_exec(s.dbSrc, "COMMIT;", 0, 0, 0);
sqlite3_close(s.dbSrc);
sqlite3_free(s.page1);
if( pzErr ){
*pzErr = s.zErr;
}else{
sqlite3_free(s.zErr);
}
return s.rcErr;
}
#ifdef SCRUB_STANDALONE
/* Error and warning log */
static void errorLogCallback(void *pNotUsed, int iErr, const char *zMsg){
const char *zType;
switch( iErr&0xff ){
case SQLITE_WARNING: zType = "WARNING"; break;
case SQLITE_NOTICE: zType = "NOTICE"; break;
default: zType = "ERROR"; break;
}
fprintf(stderr, "%s: %s\n", zType, zMsg);
}
/* The main() routine when this utility is run as a stand-alone program */
int main(int argc, char **argv){
char *zErr = 0;
int rc;
if( argc!=3 ){
fprintf(stderr,"Usage: %s SOURCE DESTINATION\n", argv[0]);
exit(1);
}
sqlite3_config(SQLITE_CONFIG_LOG, errorLogCallback, 0);
rc = sqlite3_scrub_backup(argv[1], argv[2], &zErr);
if( rc==SQLITE_NOMEM ){
fprintf(stderr, "%s: out of memory\n", argv[0]);
exit(1);
}
if( zErr ){
fprintf(stderr, "%s: %s\n", argv[0], zErr);
sqlite3_free(zErr);
exit(1);
}
return 0;
}
#endif

407
ext/misc/sha1.c Normal file
View File

@@ -0,0 +1,407 @@
/*
** 2017-01-27
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** This SQLite extension implements a functions that compute SHA1 hashes.
** Two SQL functions are implemented:
**
** sha1(X)
** sha1_query(Y)
**
** The sha1(X) function computes the SHA1 hash of the input X, or NULL if
** X is NULL.
**
** The sha1_query(Y) function evalutes all queries in the SQL statements of Y
** and returns a hash of their results.
*/
#include "sqlite3ext.h"
SQLITE_EXTENSION_INIT1
#include <assert.h>
#include <string.h>
#include <stdarg.h>
/******************************************************************************
** The Hash Engine
*/
/* Context for the SHA1 hash */
typedef struct SHA1Context SHA1Context;
struct SHA1Context {
unsigned int state[5];
unsigned int count[2];
unsigned char buffer[64];
};
#if __GNUC__ && (defined(__i386__) || defined(__x86_64__))
/*
* GCC by itself only generates left rotates. Use right rotates if
* possible to be kinder to dinky implementations with iterative rotate
* instructions.
*/
#define SHA_ROT(op, x, k) \
({ unsigned int y; asm(op " %1,%0" : "=r" (y) : "I" (k), "0" (x)); y; })
#define rol(x,k) SHA_ROT("roll", x, k)
#define ror(x,k) SHA_ROT("rorl", x, k)
#else
/* Generic C equivalent */
#define SHA_ROT(x,l,r) ((x) << (l) | (x) >> (r))
#define rol(x,k) SHA_ROT(x,k,32-(k))
#define ror(x,k) SHA_ROT(x,32-(k),k)
#endif
#define blk0le(i) (block[i] = (ror(block[i],8)&0xFF00FF00) \
|(rol(block[i],8)&0x00FF00FF))
#define blk0be(i) block[i]
#define blk(i) (block[i&15] = rol(block[(i+13)&15]^block[(i+8)&15] \
^block[(i+2)&15]^block[i&15],1))
/*
* (R0+R1), R2, R3, R4 are the different operations (rounds) used in SHA1
*
* Rl0() for little-endian and Rb0() for big-endian. Endianness is
* determined at run-time.
*/
#define Rl0(v,w,x,y,z,i) \
z+=((w&(x^y))^y)+blk0le(i)+0x5A827999+rol(v,5);w=ror(w,2);
#define Rb0(v,w,x,y,z,i) \
z+=((w&(x^y))^y)+blk0be(i)+0x5A827999+rol(v,5);w=ror(w,2);
#define R1(v,w,x,y,z,i) \
z+=((w&(x^y))^y)+blk(i)+0x5A827999+rol(v,5);w=ror(w,2);
#define R2(v,w,x,y,z,i) \
z+=(w^x^y)+blk(i)+0x6ED9EBA1+rol(v,5);w=ror(w,2);
#define R3(v,w,x,y,z,i) \
z+=(((w|x)&y)|(w&x))+blk(i)+0x8F1BBCDC+rol(v,5);w=ror(w,2);
#define R4(v,w,x,y,z,i) \
z+=(w^x^y)+blk(i)+0xCA62C1D6+rol(v,5);w=ror(w,2);
/*
* Hash a single 512-bit block. This is the core of the algorithm.
*/
void SHA1Transform(unsigned int state[5], const unsigned char buffer[64]){
unsigned int qq[5]; /* a, b, c, d, e; */
static int one = 1;
unsigned int block[16];
memcpy(block, buffer, 64);
memcpy(qq,state,5*sizeof(unsigned int));
#define a qq[0]
#define b qq[1]
#define c qq[2]
#define d qq[3]
#define e qq[4]
/* Copy p->state[] to working vars */
/*
a = state[0];
b = state[1];
c = state[2];
d = state[3];
e = state[4];
*/
/* 4 rounds of 20 operations each. Loop unrolled. */
if( 1 == *(unsigned char*)&one ){
Rl0(a,b,c,d,e, 0); Rl0(e,a,b,c,d, 1); Rl0(d,e,a,b,c, 2); Rl0(c,d,e,a,b, 3);
Rl0(b,c,d,e,a, 4); Rl0(a,b,c,d,e, 5); Rl0(e,a,b,c,d, 6); Rl0(d,e,a,b,c, 7);
Rl0(c,d,e,a,b, 8); Rl0(b,c,d,e,a, 9); Rl0(a,b,c,d,e,10); Rl0(e,a,b,c,d,11);
Rl0(d,e,a,b,c,12); Rl0(c,d,e,a,b,13); Rl0(b,c,d,e,a,14); Rl0(a,b,c,d,e,15);
}else{
Rb0(a,b,c,d,e, 0); Rb0(e,a,b,c,d, 1); Rb0(d,e,a,b,c, 2); Rb0(c,d,e,a,b, 3);
Rb0(b,c,d,e,a, 4); Rb0(a,b,c,d,e, 5); Rb0(e,a,b,c,d, 6); Rb0(d,e,a,b,c, 7);
Rb0(c,d,e,a,b, 8); Rb0(b,c,d,e,a, 9); Rb0(a,b,c,d,e,10); Rb0(e,a,b,c,d,11);
Rb0(d,e,a,b,c,12); Rb0(c,d,e,a,b,13); Rb0(b,c,d,e,a,14); Rb0(a,b,c,d,e,15);
}
R1(e,a,b,c,d,16); R1(d,e,a,b,c,17); R1(c,d,e,a,b,18); R1(b,c,d,e,a,19);
R2(a,b,c,d,e,20); R2(e,a,b,c,d,21); R2(d,e,a,b,c,22); R2(c,d,e,a,b,23);
R2(b,c,d,e,a,24); R2(a,b,c,d,e,25); R2(e,a,b,c,d,26); R2(d,e,a,b,c,27);
R2(c,d,e,a,b,28); R2(b,c,d,e,a,29); R2(a,b,c,d,e,30); R2(e,a,b,c,d,31);
R2(d,e,a,b,c,32); R2(c,d,e,a,b,33); R2(b,c,d,e,a,34); R2(a,b,c,d,e,35);
R2(e,a,b,c,d,36); R2(d,e,a,b,c,37); R2(c,d,e,a,b,38); R2(b,c,d,e,a,39);
R3(a,b,c,d,e,40); R3(e,a,b,c,d,41); R3(d,e,a,b,c,42); R3(c,d,e,a,b,43);
R3(b,c,d,e,a,44); R3(a,b,c,d,e,45); R3(e,a,b,c,d,46); R3(d,e,a,b,c,47);
R3(c,d,e,a,b,48); R3(b,c,d,e,a,49); R3(a,b,c,d,e,50); R3(e,a,b,c,d,51);
R3(d,e,a,b,c,52); R3(c,d,e,a,b,53); R3(b,c,d,e,a,54); R3(a,b,c,d,e,55);
R3(e,a,b,c,d,56); R3(d,e,a,b,c,57); R3(c,d,e,a,b,58); R3(b,c,d,e,a,59);
R4(a,b,c,d,e,60); R4(e,a,b,c,d,61); R4(d,e,a,b,c,62); R4(c,d,e,a,b,63);
R4(b,c,d,e,a,64); R4(a,b,c,d,e,65); R4(e,a,b,c,d,66); R4(d,e,a,b,c,67);
R4(c,d,e,a,b,68); R4(b,c,d,e,a,69); R4(a,b,c,d,e,70); R4(e,a,b,c,d,71);
R4(d,e,a,b,c,72); R4(c,d,e,a,b,73); R4(b,c,d,e,a,74); R4(a,b,c,d,e,75);
R4(e,a,b,c,d,76); R4(d,e,a,b,c,77); R4(c,d,e,a,b,78); R4(b,c,d,e,a,79);
/* Add the working vars back into context.state[] */
state[0] += a;
state[1] += b;
state[2] += c;
state[3] += d;
state[4] += e;
#undef a
#undef b
#undef c
#undef d
#undef e
}
/* Initialize a SHA1 context */
static void hash_init(SHA1Context *p){
/* SHA1 initialization constants */
p->state[0] = 0x67452301;
p->state[1] = 0xEFCDAB89;
p->state[2] = 0x98BADCFE;
p->state[3] = 0x10325476;
p->state[4] = 0xC3D2E1F0;
p->count[0] = p->count[1] = 0;
}
/* Add new content to the SHA1 hash */
static void hash_step(
SHA1Context *p, /* Add content to this context */
const unsigned char *data, /* Data to be added */
unsigned int len /* Number of bytes in data */
){
unsigned int i, j;
j = p->count[0];
if( (p->count[0] += len << 3) < j ){
p->count[1] += (len>>29)+1;
}
j = (j >> 3) & 63;
if( (j + len) > 63 ){
(void)memcpy(&p->buffer[j], data, (i = 64-j));
SHA1Transform(p->state, p->buffer);
for(; i + 63 < len; i += 64){
SHA1Transform(p->state, &data[i]);
}
j = 0;
}else{
i = 0;
}
(void)memcpy(&p->buffer[j], &data[i], len - i);
}
/* Compute a string using sqlite3_vsnprintf() and hash it */
static void hash_step_vformat(
SHA1Context *p, /* Add content to this context */
const char *zFormat,
...
){
va_list ap;
int n;
char zBuf[50];
va_start(ap, zFormat);
sqlite3_vsnprintf(sizeof(zBuf),zBuf,zFormat,ap);
va_end(ap);
n = (int)strlen(zBuf);
hash_step(p, (unsigned char*)zBuf, n);
}
/* Add padding and compute the message digest. Render the
** message digest as lower-case hexadecimal and put it into
** zOut[]. zOut[] must be at least 41 bytes long. */
static void hash_finish(
SHA1Context *p, /* The SHA1 context to finish and render */
char *zOut /* Store hexadecimal hash here */
){
unsigned int i;
unsigned char finalcount[8];
unsigned char digest[20];
static const char zEncode[] = "0123456789abcdef";
for (i = 0; i < 8; i++){
finalcount[i] = (unsigned char)((p->count[(i >= 4 ? 0 : 1)]
>> ((3-(i & 3)) * 8) ) & 255); /* Endian independent */
}
hash_step(p, (const unsigned char *)"\200", 1);
while ((p->count[0] & 504) != 448){
hash_step(p, (const unsigned char *)"\0", 1);
}
hash_step(p, finalcount, 8); /* Should cause a SHA1Transform() */
for (i = 0; i < 20; i++){
digest[i] = (unsigned char)((p->state[i>>2] >> ((3-(i & 3)) * 8) ) & 255);
}
for(i=0; i<20; i++){
zOut[i*2] = zEncode[(digest[i]>>4)&0xf];
zOut[i*2+1] = zEncode[digest[i] & 0xf];
}
zOut[i*2]= 0;
}
/* End of the hashing logic
*****************************************************************************/
/*
** Implementation of the sha1(X) function.
**
** Return a lower-case hexadecimal rendering of the SHA1 hash of the
** argument X. If X is a BLOB, it is hashed as is. For all other
** types of input, X is converted into a UTF-8 string and the string
** is hash without the trailing 0x00 terminator. The hash of a NULL
** value is NULL.
*/
static void sha1Func(
sqlite3_context *context,
int argc,
sqlite3_value **argv
){
SHA1Context cx;
int eType = sqlite3_value_type(argv[0]);
int nByte = sqlite3_value_bytes(argv[0]);
char zOut[44];
assert( argc==1 );
if( eType==SQLITE_NULL ) return;
hash_init(&cx);
if( eType==SQLITE_BLOB ){
hash_step(&cx, sqlite3_value_blob(argv[0]), nByte);
}else{
hash_step(&cx, sqlite3_value_text(argv[0]), nByte);
}
hash_finish(&cx, zOut);
sqlite3_result_text(context, zOut, 40, SQLITE_TRANSIENT);
}
/*
** Implementation of the sha1_query(SQL) function.
**
** This function compiles and runs the SQL statement(s) given in the
** argument. The results are hashed using SHA1 and that hash is returned.
**
** The original SQL text is included as part of the hash.
**
** The hash is not just a concatenation of the outputs. Each query
** is delimited and each row and value within the query is delimited,
** with all values being marked with their datatypes.
*/
static void sha1QueryFunc(
sqlite3_context *context,
int argc,
sqlite3_value **argv
){
sqlite3 *db = sqlite3_context_db_handle(context);
const char *zSql = (const char*)sqlite3_value_text(argv[0]);
sqlite3_stmt *pStmt = 0;
int nCol; /* Number of columns in the result set */
int i; /* Loop counter */
int rc;
int n;
const char *z;
SHA1Context cx;
char zOut[44];
assert( argc==1 );
if( zSql==0 ) return;
hash_init(&cx);
while( zSql[0] ){
rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zSql);
if( rc ){
char *zMsg = sqlite3_mprintf("error SQL statement [%s]: %s",
zSql, sqlite3_errmsg(db));
sqlite3_finalize(pStmt);
sqlite3_result_error(context, zMsg, -1);
sqlite3_free(zMsg);
return;
}
if( !sqlite3_stmt_readonly(pStmt) ){
char *zMsg = sqlite3_mprintf("non-query: [%s]", sqlite3_sql(pStmt));
sqlite3_finalize(pStmt);
sqlite3_result_error(context, zMsg, -1);
sqlite3_free(zMsg);
return;
}
nCol = sqlite3_column_count(pStmt);
z = sqlite3_sql(pStmt);
n = (int)strlen(z);
hash_step_vformat(&cx,"S%d:",n);
hash_step(&cx,(unsigned char*)z,n);
/* Compute a hash over the result of the query */
while( SQLITE_ROW==sqlite3_step(pStmt) ){
hash_step(&cx,(const unsigned char*)"R",1);
for(i=0; i<nCol; i++){
switch( sqlite3_column_type(pStmt,i) ){
case SQLITE_NULL: {
hash_step(&cx, (const unsigned char*)"N",1);
break;
}
case SQLITE_INTEGER: {
sqlite3_uint64 u;
int j;
unsigned char x[9];
sqlite3_int64 v = sqlite3_column_int64(pStmt,i);
memcpy(&u, &v, 8);
for(j=8; j>=1; j--){
x[j] = u & 0xff;
u >>= 8;
}
x[0] = 'I';
hash_step(&cx, x, 9);
break;
}
case SQLITE_FLOAT: {
sqlite3_uint64 u;
int j;
unsigned char x[9];
double r = sqlite3_column_double(pStmt,i);
memcpy(&u, &r, 8);
for(j=8; j>=1; j--){
x[j] = u & 0xff;
u >>= 8;
}
x[0] = 'F';
hash_step(&cx,x,9);
break;
}
case SQLITE_TEXT: {
int n2 = sqlite3_column_bytes(pStmt, i);
const unsigned char *z2 = sqlite3_column_text(pStmt, i);
hash_step_vformat(&cx,"T%d:",n2);
hash_step(&cx, z2, n2);
break;
}
case SQLITE_BLOB: {
int n2 = sqlite3_column_bytes(pStmt, i);
const unsigned char *z2 = sqlite3_column_blob(pStmt, i);
hash_step_vformat(&cx,"B%d:",n2);
hash_step(&cx, z2, n2);
break;
}
}
}
}
sqlite3_finalize(pStmt);
}
hash_finish(&cx, zOut);
sqlite3_result_text(context, zOut, 40, SQLITE_TRANSIENT);
}
#ifdef _WIN32
__declspec(dllexport)
#endif
int sqlite3_sha_init(
sqlite3 *db,
char **pzErrMsg,
const sqlite3_api_routines *pApi
){
int rc = SQLITE_OK;
SQLITE_EXTENSION_INIT2(pApi);
(void)pzErrMsg; /* Unused parameter */
rc = sqlite3_create_function(db, "sha1", 1, SQLITE_UTF8, 0,
sha1Func, 0, 0);
if( rc==SQLITE_OK ){
rc = sqlite3_create_function(db, "sha1_query", 1, SQLITE_UTF8, 0,
sha1QueryFunc, 0, 0);
}
return rc;
}

714
ext/misc/shathree.c Normal file
View File

@@ -0,0 +1,714 @@
/*
** 2017-03-08
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** This SQLite extension implements a functions that compute SHA1 hashes.
** Two SQL functions are implemented:
**
** sha3(X,SIZE)
** sha3_query(Y,SIZE)
**
** The sha3(X) function computes the SHA3 hash of the input X, or NULL if
** X is NULL.
**
** The sha3_query(Y) function evalutes all queries in the SQL statements of Y
** and returns a hash of their results.
**
** The SIZE argument is optional. If omitted, the SHA3-256 hash algorithm
** is used. If SIZE is included it must be one of the integers 224, 256,
** 384, or 512, to determine SHA3 hash variant that is computed.
*/
#include "sqlite3ext.h"
SQLITE_EXTENSION_INIT1
#include <assert.h>
#include <string.h>
#include <stdarg.h>
typedef sqlite3_uint64 u64;
/******************************************************************************
** The Hash Engine
*/
/*
** Macros to determine whether the machine is big or little endian,
** and whether or not that determination is run-time or compile-time.
**
** For best performance, an attempt is made to guess at the byte-order
** using C-preprocessor macros. If that is unsuccessful, or if
** -DSHA3_BYTEORDER=0 is set, then byte-order is determined
** at run-time.
*/
#ifndef SHA3_BYTEORDER
# if defined(i386) || defined(__i386__) || defined(_M_IX86) || \
defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \
defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \
defined(__arm__)
# define SHA3_BYTEORDER 1234
# elif defined(sparc) || defined(__ppc__)
# define SHA3_BYTEORDER 4321
# else
# define SHA3_BYTEORDER 0
# endif
#endif
/*
** State structure for a SHA3 hash in progress
*/
typedef struct SHA3Context SHA3Context;
struct SHA3Context {
union {
u64 s[25]; /* Keccak state. 5x5 lines of 64 bits each */
unsigned char x[1600]; /* ... or 1600 bytes */
} u;
unsigned nRate; /* Bytes of input accepted per Keccak iteration */
unsigned nLoaded; /* Input bytes loaded into u.x[] so far this cycle */
unsigned ixMask; /* Insert next input into u.x[nLoaded^ixMask]. */
};
/*
** A single step of the Keccak mixing function for a 1600-bit state
*/
static void KeccakF1600Step(SHA3Context *p){
int i;
u64 B0, B1, B2, B3, B4;
u64 C0, C1, C2, C3, C4;
u64 D0, D1, D2, D3, D4;
static const u64 RC[] = {
0x0000000000000001ULL, 0x0000000000008082ULL,
0x800000000000808aULL, 0x8000000080008000ULL,
0x000000000000808bULL, 0x0000000080000001ULL,
0x8000000080008081ULL, 0x8000000000008009ULL,
0x000000000000008aULL, 0x0000000000000088ULL,
0x0000000080008009ULL, 0x000000008000000aULL,
0x000000008000808bULL, 0x800000000000008bULL,
0x8000000000008089ULL, 0x8000000000008003ULL,
0x8000000000008002ULL, 0x8000000000000080ULL,
0x000000000000800aULL, 0x800000008000000aULL,
0x8000000080008081ULL, 0x8000000000008080ULL,
0x0000000080000001ULL, 0x8000000080008008ULL
};
# define A00 (p->u.s[0])
# define A01 (p->u.s[1])
# define A02 (p->u.s[2])
# define A03 (p->u.s[3])
# define A04 (p->u.s[4])
# define A10 (p->u.s[5])
# define A11 (p->u.s[6])
# define A12 (p->u.s[7])
# define A13 (p->u.s[8])
# define A14 (p->u.s[9])
# define A20 (p->u.s[10])
# define A21 (p->u.s[11])
# define A22 (p->u.s[12])
# define A23 (p->u.s[13])
# define A24 (p->u.s[14])
# define A30 (p->u.s[15])
# define A31 (p->u.s[16])
# define A32 (p->u.s[17])
# define A33 (p->u.s[18])
# define A34 (p->u.s[19])
# define A40 (p->u.s[20])
# define A41 (p->u.s[21])
# define A42 (p->u.s[22])
# define A43 (p->u.s[23])
# define A44 (p->u.s[24])
# define ROL64(a,x) ((a<<x)|(a>>(64-x)))
for(i=0; i<24; i+=4){
C0 = A00^A10^A20^A30^A40;
C1 = A01^A11^A21^A31^A41;
C2 = A02^A12^A22^A32^A42;
C3 = A03^A13^A23^A33^A43;
C4 = A04^A14^A24^A34^A44;
D0 = C4^ROL64(C1, 1);
D1 = C0^ROL64(C2, 1);
D2 = C1^ROL64(C3, 1);
D3 = C2^ROL64(C4, 1);
D4 = C3^ROL64(C0, 1);
B0 = (A00^D0);
B1 = ROL64((A11^D1), 44);
B2 = ROL64((A22^D2), 43);
B3 = ROL64((A33^D3), 21);
B4 = ROL64((A44^D4), 14);
A00 = B0 ^((~B1)& B2 );
A00 ^= RC[i];
A11 = B1 ^((~B2)& B3 );
A22 = B2 ^((~B3)& B4 );
A33 = B3 ^((~B4)& B0 );
A44 = B4 ^((~B0)& B1 );
B2 = ROL64((A20^D0), 3);
B3 = ROL64((A31^D1), 45);
B4 = ROL64((A42^D2), 61);
B0 = ROL64((A03^D3), 28);
B1 = ROL64((A14^D4), 20);
A20 = B0 ^((~B1)& B2 );
A31 = B1 ^((~B2)& B3 );
A42 = B2 ^((~B3)& B4 );
A03 = B3 ^((~B4)& B0 );
A14 = B4 ^((~B0)& B1 );
B4 = ROL64((A40^D0), 18);
B0 = ROL64((A01^D1), 1);
B1 = ROL64((A12^D2), 6);
B2 = ROL64((A23^D3), 25);
B3 = ROL64((A34^D4), 8);
A40 = B0 ^((~B1)& B2 );
A01 = B1 ^((~B2)& B3 );
A12 = B2 ^((~B3)& B4 );
A23 = B3 ^((~B4)& B0 );
A34 = B4 ^((~B0)& B1 );
B1 = ROL64((A10^D0), 36);
B2 = ROL64((A21^D1), 10);
B3 = ROL64((A32^D2), 15);
B4 = ROL64((A43^D3), 56);
B0 = ROL64((A04^D4), 27);
A10 = B0 ^((~B1)& B2 );
A21 = B1 ^((~B2)& B3 );
A32 = B2 ^((~B3)& B4 );
A43 = B3 ^((~B4)& B0 );
A04 = B4 ^((~B0)& B1 );
B3 = ROL64((A30^D0), 41);
B4 = ROL64((A41^D1), 2);
B0 = ROL64((A02^D2), 62);
B1 = ROL64((A13^D3), 55);
B2 = ROL64((A24^D4), 39);
A30 = B0 ^((~B1)& B2 );
A41 = B1 ^((~B2)& B3 );
A02 = B2 ^((~B3)& B4 );
A13 = B3 ^((~B4)& B0 );
A24 = B4 ^((~B0)& B1 );
C0 = A00^A20^A40^A10^A30;
C1 = A11^A31^A01^A21^A41;
C2 = A22^A42^A12^A32^A02;
C3 = A33^A03^A23^A43^A13;
C4 = A44^A14^A34^A04^A24;
D0 = C4^ROL64(C1, 1);
D1 = C0^ROL64(C2, 1);
D2 = C1^ROL64(C3, 1);
D3 = C2^ROL64(C4, 1);
D4 = C3^ROL64(C0, 1);
B0 = (A00^D0);
B1 = ROL64((A31^D1), 44);
B2 = ROL64((A12^D2), 43);
B3 = ROL64((A43^D3), 21);
B4 = ROL64((A24^D4), 14);
A00 = B0 ^((~B1)& B2 );
A00 ^= RC[i+1];
A31 = B1 ^((~B2)& B3 );
A12 = B2 ^((~B3)& B4 );
A43 = B3 ^((~B4)& B0 );
A24 = B4 ^((~B0)& B1 );
B2 = ROL64((A40^D0), 3);
B3 = ROL64((A21^D1), 45);
B4 = ROL64((A02^D2), 61);
B0 = ROL64((A33^D3), 28);
B1 = ROL64((A14^D4), 20);
A40 = B0 ^((~B1)& B2 );
A21 = B1 ^((~B2)& B3 );
A02 = B2 ^((~B3)& B4 );
A33 = B3 ^((~B4)& B0 );
A14 = B4 ^((~B0)& B1 );
B4 = ROL64((A30^D0), 18);
B0 = ROL64((A11^D1), 1);
B1 = ROL64((A42^D2), 6);
B2 = ROL64((A23^D3), 25);
B3 = ROL64((A04^D4), 8);
A30 = B0 ^((~B1)& B2 );
A11 = B1 ^((~B2)& B3 );
A42 = B2 ^((~B3)& B4 );
A23 = B3 ^((~B4)& B0 );
A04 = B4 ^((~B0)& B1 );
B1 = ROL64((A20^D0), 36);
B2 = ROL64((A01^D1), 10);
B3 = ROL64((A32^D2), 15);
B4 = ROL64((A13^D3), 56);
B0 = ROL64((A44^D4), 27);
A20 = B0 ^((~B1)& B2 );
A01 = B1 ^((~B2)& B3 );
A32 = B2 ^((~B3)& B4 );
A13 = B3 ^((~B4)& B0 );
A44 = B4 ^((~B0)& B1 );
B3 = ROL64((A10^D0), 41);
B4 = ROL64((A41^D1), 2);
B0 = ROL64((A22^D2), 62);
B1 = ROL64((A03^D3), 55);
B2 = ROL64((A34^D4), 39);
A10 = B0 ^((~B1)& B2 );
A41 = B1 ^((~B2)& B3 );
A22 = B2 ^((~B3)& B4 );
A03 = B3 ^((~B4)& B0 );
A34 = B4 ^((~B0)& B1 );
C0 = A00^A40^A30^A20^A10;
C1 = A31^A21^A11^A01^A41;
C2 = A12^A02^A42^A32^A22;
C3 = A43^A33^A23^A13^A03;
C4 = A24^A14^A04^A44^A34;
D0 = C4^ROL64(C1, 1);
D1 = C0^ROL64(C2, 1);
D2 = C1^ROL64(C3, 1);
D3 = C2^ROL64(C4, 1);
D4 = C3^ROL64(C0, 1);
B0 = (A00^D0);
B1 = ROL64((A21^D1), 44);
B2 = ROL64((A42^D2), 43);
B3 = ROL64((A13^D3), 21);
B4 = ROL64((A34^D4), 14);
A00 = B0 ^((~B1)& B2 );
A00 ^= RC[i+2];
A21 = B1 ^((~B2)& B3 );
A42 = B2 ^((~B3)& B4 );
A13 = B3 ^((~B4)& B0 );
A34 = B4 ^((~B0)& B1 );
B2 = ROL64((A30^D0), 3);
B3 = ROL64((A01^D1), 45);
B4 = ROL64((A22^D2), 61);
B0 = ROL64((A43^D3), 28);
B1 = ROL64((A14^D4), 20);
A30 = B0 ^((~B1)& B2 );
A01 = B1 ^((~B2)& B3 );
A22 = B2 ^((~B3)& B4 );
A43 = B3 ^((~B4)& B0 );
A14 = B4 ^((~B0)& B1 );
B4 = ROL64((A10^D0), 18);
B0 = ROL64((A31^D1), 1);
B1 = ROL64((A02^D2), 6);
B2 = ROL64((A23^D3), 25);
B3 = ROL64((A44^D4), 8);
A10 = B0 ^((~B1)& B2 );
A31 = B1 ^((~B2)& B3 );
A02 = B2 ^((~B3)& B4 );
A23 = B3 ^((~B4)& B0 );
A44 = B4 ^((~B0)& B1 );
B1 = ROL64((A40^D0), 36);
B2 = ROL64((A11^D1), 10);
B3 = ROL64((A32^D2), 15);
B4 = ROL64((A03^D3), 56);
B0 = ROL64((A24^D4), 27);
A40 = B0 ^((~B1)& B2 );
A11 = B1 ^((~B2)& B3 );
A32 = B2 ^((~B3)& B4 );
A03 = B3 ^((~B4)& B0 );
A24 = B4 ^((~B0)& B1 );
B3 = ROL64((A20^D0), 41);
B4 = ROL64((A41^D1), 2);
B0 = ROL64((A12^D2), 62);
B1 = ROL64((A33^D3), 55);
B2 = ROL64((A04^D4), 39);
A20 = B0 ^((~B1)& B2 );
A41 = B1 ^((~B2)& B3 );
A12 = B2 ^((~B3)& B4 );
A33 = B3 ^((~B4)& B0 );
A04 = B4 ^((~B0)& B1 );
C0 = A00^A30^A10^A40^A20;
C1 = A21^A01^A31^A11^A41;
C2 = A42^A22^A02^A32^A12;
C3 = A13^A43^A23^A03^A33;
C4 = A34^A14^A44^A24^A04;
D0 = C4^ROL64(C1, 1);
D1 = C0^ROL64(C2, 1);
D2 = C1^ROL64(C3, 1);
D3 = C2^ROL64(C4, 1);
D4 = C3^ROL64(C0, 1);
B0 = (A00^D0);
B1 = ROL64((A01^D1), 44);
B2 = ROL64((A02^D2), 43);
B3 = ROL64((A03^D3), 21);
B4 = ROL64((A04^D4), 14);
A00 = B0 ^((~B1)& B2 );
A00 ^= RC[i+3];
A01 = B1 ^((~B2)& B3 );
A02 = B2 ^((~B3)& B4 );
A03 = B3 ^((~B4)& B0 );
A04 = B4 ^((~B0)& B1 );
B2 = ROL64((A10^D0), 3);
B3 = ROL64((A11^D1), 45);
B4 = ROL64((A12^D2), 61);
B0 = ROL64((A13^D3), 28);
B1 = ROL64((A14^D4), 20);
A10 = B0 ^((~B1)& B2 );
A11 = B1 ^((~B2)& B3 );
A12 = B2 ^((~B3)& B4 );
A13 = B3 ^((~B4)& B0 );
A14 = B4 ^((~B0)& B1 );
B4 = ROL64((A20^D0), 18);
B0 = ROL64((A21^D1), 1);
B1 = ROL64((A22^D2), 6);
B2 = ROL64((A23^D3), 25);
B3 = ROL64((A24^D4), 8);
A20 = B0 ^((~B1)& B2 );
A21 = B1 ^((~B2)& B3 );
A22 = B2 ^((~B3)& B4 );
A23 = B3 ^((~B4)& B0 );
A24 = B4 ^((~B0)& B1 );
B1 = ROL64((A30^D0), 36);
B2 = ROL64((A31^D1), 10);
B3 = ROL64((A32^D2), 15);
B4 = ROL64((A33^D3), 56);
B0 = ROL64((A34^D4), 27);
A30 = B0 ^((~B1)& B2 );
A31 = B1 ^((~B2)& B3 );
A32 = B2 ^((~B3)& B4 );
A33 = B3 ^((~B4)& B0 );
A34 = B4 ^((~B0)& B1 );
B3 = ROL64((A40^D0), 41);
B4 = ROL64((A41^D1), 2);
B0 = ROL64((A42^D2), 62);
B1 = ROL64((A43^D3), 55);
B2 = ROL64((A44^D4), 39);
A40 = B0 ^((~B1)& B2 );
A41 = B1 ^((~B2)& B3 );
A42 = B2 ^((~B3)& B4 );
A43 = B3 ^((~B4)& B0 );
A44 = B4 ^((~B0)& B1 );
}
}
/*
** Initialize a new hash. iSize determines the size of the hash
** in bits and should be one of 224, 256, 384, or 512. Or iSize
** can be zero to use the default hash size of 256 bits.
*/
static void SHA3Init(SHA3Context *p, int iSize){
memset(p, 0, sizeof(*p));
if( iSize>=128 && iSize<=512 ){
p->nRate = (1600 - ((iSize + 31)&~31)*2)/8;
}else{
p->nRate = (1600 - 2*256)/8;
}
#if SHA3_BYTEORDER==1234
/* Known to be little-endian at compile-time. No-op */
#elif SHA3_BYTEORDER==4321
p->ixMask = 7; /* Big-endian */
#else
{
static unsigned int one = 1;
if( 1==*(unsigned char*)&one ){
/* Little endian. No byte swapping. */
p->ixMask = 0;
}else{
/* Big endian. Byte swap. */
p->ixMask = 7;
}
}
#endif
}
/*
** Make consecutive calls to the SHA3Update function to add new content
** to the hash
*/
static void SHA3Update(
SHA3Context *p,
const unsigned char *aData,
unsigned int nData
){
unsigned int i = 0;
#if SHA3_BYTEORDER==1234
if( (p->nLoaded % 8)==0 && ((aData - (const unsigned char*)0)&7)==0 ){
for(; i+7<nData; i+=8){
p->u.s[p->nLoaded/8] ^= *(u64*)&aData[i];
p->nLoaded += 8;
if( p->nLoaded>=p->nRate ){
KeccakF1600Step(p);
p->nLoaded = 0;
}
}
}
#endif
for(; i<nData; i++){
#if SHA3_BYTEORDER==1234
p->u.x[p->nLoaded] ^= aData[i];
#elif SHA3_BYTEORDER==4321
p->u.x[p->nLoaded^0x07] ^= aData[i];
#else
p->u.x[p->nLoaded^p->ixMask] ^= aData[i];
#endif
p->nLoaded++;
if( p->nLoaded==p->nRate ){
KeccakF1600Step(p);
p->nLoaded = 0;
}
}
}
/*
** After all content has been added, invoke SHA3Final() to compute
** the final hash. The function returns a pointer to the binary
** hash value.
*/
static unsigned char *SHA3Final(SHA3Context *p){
unsigned int i;
if( p->nLoaded==p->nRate-1 ){
const unsigned char c1 = 0x86;
SHA3Update(p, &c1, 1);
}else{
const unsigned char c2 = 0x06;
const unsigned char c3 = 0x80;
SHA3Update(p, &c2, 1);
p->nLoaded = p->nRate - 1;
SHA3Update(p, &c3, 1);
}
for(i=0; i<p->nRate; i++){
p->u.x[i+p->nRate] = p->u.x[i^p->ixMask];
}
return &p->u.x[p->nRate];
}
/* End of the hashing logic
*****************************************************************************/
/*
** Implementation of the sha3(X,SIZE) function.
**
** Return a BLOB which is the SIZE-bit SHA3 hash of X. The default
** size is 256. If X is a BLOB, it is hashed as is.
** For all other non-NULL types of input, X is converted into a UTF-8 string
** and the string is hashed without the trailing 0x00 terminator. The hash
** of a NULL value is NULL.
*/
static void sha3Func(
sqlite3_context *context,
int argc,
sqlite3_value **argv
){
SHA3Context cx;
int eType = sqlite3_value_type(argv[0]);
int nByte = sqlite3_value_bytes(argv[0]);
int iSize;
if( argc==1 ){
iSize = 256;
}else{
iSize = sqlite3_value_int(argv[1]);
if( iSize!=224 && iSize!=256 && iSize!=384 && iSize!=512 ){
sqlite3_result_error(context, "SHA3 size should be one of: 224 256 "
"384 512", -1);
return;
}
}
if( eType==SQLITE_NULL ) return;
SHA3Init(&cx, iSize);
if( eType==SQLITE_BLOB ){
SHA3Update(&cx, sqlite3_value_blob(argv[0]), nByte);
}else{
SHA3Update(&cx, sqlite3_value_text(argv[0]), nByte);
}
sqlite3_result_blob(context, SHA3Final(&cx), iSize/8, SQLITE_TRANSIENT);
}
/* Compute a string using sqlite3_vsnprintf() with a maximum length
** of 50 bytes and add it to the hash.
*/
static void hash_step_vformat(
SHA3Context *p, /* Add content to this context */
const char *zFormat,
...
){
va_list ap;
int n;
char zBuf[50];
va_start(ap, zFormat);
sqlite3_vsnprintf(sizeof(zBuf),zBuf,zFormat,ap);
va_end(ap);
n = (int)strlen(zBuf);
SHA3Update(p, (unsigned char*)zBuf, n);
}
/*
** Implementation of the sha3_query(SQL,SIZE) function.
**
** This function compiles and runs the SQL statement(s) given in the
** argument. The results are hashed using a SIZE-bit SHA3. The default
** size is 256.
**
** The format of the byte stream that is hashed is summarized as follows:
**
** S<n>:<sql>
** R
** N
** I<int>
** F<ieee-float>
** B<size>:<bytes>
** T<size>:<text>
**
** <sql> is the original SQL text for each statement run and <n> is
** the size of that text. The SQL text is UTF-8. A single R character
** occurs before the start of each row. N means a NULL value.
** I mean an 8-byte little-endian integer <int>. F is a floating point
** number with an 8-byte little-endian IEEE floating point value <ieee-float>.
** B means blobs of <size> bytes. T means text rendered as <size>
** bytes of UTF-8. The <n> and <size> values are expressed as an ASCII
** text integers.
**
** For each SQL statement in the X input, there is one S segment. Each
** S segment is followed by zero or more R segments, one for each row in the
** result set. After each R, there are one or more N, I, F, B, or T segments,
** one for each column in the result set. Segments are concatentated directly
** with no delimiters of any kind.
*/
static void sha3QueryFunc(
sqlite3_context *context,
int argc,
sqlite3_value **argv
){
sqlite3 *db = sqlite3_context_db_handle(context);
const char *zSql = (const char*)sqlite3_value_text(argv[0]);
sqlite3_stmt *pStmt = 0;
int nCol; /* Number of columns in the result set */
int i; /* Loop counter */
int rc;
int n;
const char *z;
SHA3Context cx;
int iSize;
if( argc==1 ){
iSize = 256;
}else{
iSize = sqlite3_value_int(argv[1]);
if( iSize!=224 && iSize!=256 && iSize!=384 && iSize!=512 ){
sqlite3_result_error(context, "SHA3 size should be one of: 224 256 "
"384 512", -1);
return;
}
}
if( zSql==0 ) return;
SHA3Init(&cx, iSize);
while( zSql[0] ){
rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zSql);
if( rc ){
char *zMsg = sqlite3_mprintf("error SQL statement [%s]: %s",
zSql, sqlite3_errmsg(db));
sqlite3_finalize(pStmt);
sqlite3_result_error(context, zMsg, -1);
sqlite3_free(zMsg);
return;
}
if( !sqlite3_stmt_readonly(pStmt) ){
char *zMsg = sqlite3_mprintf("non-query: [%s]", sqlite3_sql(pStmt));
sqlite3_finalize(pStmt);
sqlite3_result_error(context, zMsg, -1);
sqlite3_free(zMsg);
return;
}
nCol = sqlite3_column_count(pStmt);
z = sqlite3_sql(pStmt);
n = (int)strlen(z);
hash_step_vformat(&cx,"S%d:",n);
SHA3Update(&cx,(unsigned char*)z,n);
/* Compute a hash over the result of the query */
while( SQLITE_ROW==sqlite3_step(pStmt) ){
SHA3Update(&cx,(const unsigned char*)"R",1);
for(i=0; i<nCol; i++){
switch( sqlite3_column_type(pStmt,i) ){
case SQLITE_NULL: {
SHA3Update(&cx, (const unsigned char*)"N",1);
break;
}
case SQLITE_INTEGER: {
sqlite3_uint64 u;
int j;
unsigned char x[9];
sqlite3_int64 v = sqlite3_column_int64(pStmt,i);
memcpy(&u, &v, 8);
for(j=8; j>=1; j--){
x[j] = u & 0xff;
u >>= 8;
}
x[0] = 'I';
SHA3Update(&cx, x, 9);
break;
}
case SQLITE_FLOAT: {
sqlite3_uint64 u;
int j;
unsigned char x[9];
double r = sqlite3_column_double(pStmt,i);
memcpy(&u, &r, 8);
for(j=8; j>=1; j--){
x[j] = u & 0xff;
u >>= 8;
}
x[0] = 'F';
SHA3Update(&cx,x,9);
break;
}
case SQLITE_TEXT: {
int n2 = sqlite3_column_bytes(pStmt, i);
const unsigned char *z2 = sqlite3_column_text(pStmt, i);
hash_step_vformat(&cx,"T%d:",n2);
SHA3Update(&cx, z2, n2);
break;
}
case SQLITE_BLOB: {
int n2 = sqlite3_column_bytes(pStmt, i);
const unsigned char *z2 = sqlite3_column_blob(pStmt, i);
hash_step_vformat(&cx,"B%d:",n2);
SHA3Update(&cx, z2, n2);
break;
}
}
}
}
sqlite3_finalize(pStmt);
}
sqlite3_result_blob(context, SHA3Final(&cx), iSize/8, SQLITE_TRANSIENT);
}
#ifdef _WIN32
__declspec(dllexport)
#endif
int sqlite3_shathree_init(
sqlite3 *db,
char **pzErrMsg,
const sqlite3_api_routines *pApi
){
int rc = SQLITE_OK;
SQLITE_EXTENSION_INIT2(pApi);
(void)pzErrMsg; /* Unused parameter */
rc = sqlite3_create_function(db, "sha3", 1, SQLITE_UTF8, 0,
sha3Func, 0, 0);
if( rc==SQLITE_OK ){
rc = sqlite3_create_function(db, "sha3", 2, SQLITE_UTF8, 0,
sha3Func, 0, 0);
}
if( rc==SQLITE_OK ){
rc = sqlite3_create_function(db, "sha3_query", 1, SQLITE_UTF8, 0,
sha3QueryFunc, 0, 0);
}
if( rc==SQLITE_OK ){
rc = sqlite3_create_function(db, "sha3_query", 2, SQLITE_UTF8, 0,
sha3QueryFunc, 0, 0);
}
return rc;
}

22
ext/misc/spellfix.c
View File

@@ -390,7 +390,7 @@ static int editdist1(const char *zA, const char *zB, int *pnMatch){
/* Special processing if either string is empty */
if( nA==0 ){
cBprev = dc;
cBprev = (char)dc;
for(xB=res=0; (cB = zB[xB])!=0; xB++){
res += insertOrDeleteCost(cBprev, cB, zB[xB+1])/FINAL_INS_COST_DIV;
cBprev = cB;
@@ -398,7 +398,7 @@ static int editdist1(const char *zA, const char *zB, int *pnMatch){
return res;
}
if( nB==0 ){
cAprev = dc;
cAprev = (char)dc;
for(xA=res=0; (cA = zA[xA])!=0; xA++){
res += insertOrDeleteCost(cAprev, cA, zA[xA+1]);
cAprev = cA;
@@ -420,8 +420,8 @@ static int editdist1(const char *zA, const char *zB, int *pnMatch){
/* Compute the Wagner edit distance */
m[0] = 0;
cx[0] = dc;
cBprev = dc;
cx[0] = (char)dc;
cBprev = (char)dc;
for(xB=1; xB<=nB; xB++){
cBnext = zB[xB];
cB = zB[xB-1];
@@ -429,7 +429,7 @@ static int editdist1(const char *zA, const char *zB, int *pnMatch){
m[xB] = m[xB-1] + insertOrDeleteCost(cBprev, cB, cBnext);
cBprev = cB;
}
cAprev = dc;
cAprev = (char)dc;
for(xA=1; xA<=nA; xA++){
int lastA = (xA==nA);
cA = zA[xA-1];
@@ -476,7 +476,7 @@ static int editdist1(const char *zA, const char *zB, int *pnMatch){
d = m[xB];
dc = cx[xB];
m[xB] = totalCost;
cx[xB] = ncx;
cx[xB] = (char)ncx;
cBprev = cB;
}
cAprev = cA;
@@ -711,9 +711,9 @@ static int editDist3ConfigLoad(
if( nExtra<0 ) nExtra = 0;
pCost = sqlite3_malloc64( sizeof(*pCost) + nExtra );
if( pCost==0 ){ rc = SQLITE_NOMEM; break; }
pCost->nFrom = nFrom;
pCost->nTo = nTo;
pCost->iCost = iCost;
pCost->nFrom = (u8)nFrom;
pCost->nTo = (u8)nTo;
pCost->iCost = (u16)iCost;
memcpy(pCost->a, zFrom, nFrom);
memcpy(pCost->a + nFrom, zTo, nTo);
pCost->pNext = pLang->pCost;
@@ -1616,7 +1616,7 @@ static unsigned char *transliterate(const unsigned char *zIn, int nIn){
zIn += sz;
nIn -= sz;
if( c<=127 ){
zOut[nOut++] = c;
zOut[nOut++] = (unsigned char)c;
}else{
int xTop, xBtm, x;
xTop = sizeof(translit)/sizeof(translit[0]) - 1;
@@ -2231,7 +2231,7 @@ static int spellfix1Score(int iDistance, int iRank){
** Compare two spellfix1_row objects for sorting purposes in qsort() such
** that they sort in order of increasing distance.
*/
static int spellfix1RowCompare(const void *A, const void *B){
static int SQLITE_CDECL spellfix1RowCompare(const void *A, const void *B){
const struct spellfix1_row *a = (const struct spellfix1_row*)A;
const struct spellfix1_row *b = (const struct spellfix1_row*)B;
return a->iScore - b->iScore;

816
ext/misc/vfsstat.c Normal file
View File

@@ -0,0 +1,816 @@
/*
** 2016-05-27
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** This file contains the implementation of an SQLite vfs shim that
** tracks I/O. Access to the accumulated status counts is provided using
** an eponymous virtual table.
*/
#include <sqlite3ext.h>
SQLITE_EXTENSION_INIT1
/*
** This module contains code for a wrapper VFS that cause stats for
** most VFS calls to be recorded.
**
** To use this module, first compile it as a loadable extension. See
** https://www.sqlite.org/loadext.html#build for compilations instructions.
**
** After compliing, load this extension, then open database connections to be
** measured. Query usages status using the vfsstat virtual table:
**
** SELECT * FROM vfsstat;
**
** Reset counters using UPDATE statements against vfsstat:
**
** UPDATE vfsstat SET count=0;
**
** EXAMPLE SCRIPT:
**
** .load ./vfsstat
** .open test.db
** DROP TABLE IF EXISTS t1;
** CREATE TABLE t1(x,y);
** INSERT INTO t1 VALUES(123, randomblob(5000));
** CREATE INDEX t1x ON t1(x);
** DROP TABLE t1;
** VACUUM;
** SELECT * FROM vfsstat WHERE count>0;
**
** LIMITATIONS:
**
** This module increments counters without using mutex protection. So if
** two or more threads try to use this module at the same time, race conditions
** may occur which mess up the counts. This is harmless, other than giving
** incorrect statistics.
*/
#include <string.h>
#include <stdlib.h>
#include <assert.h>
/*
** File types
*/
#define VFSSTAT_MAIN 0 /* Main database file */
#define VFSSTAT_JOURNAL 1 /* Rollback journal */
#define VFSSTAT_WAL 2 /* Write-ahead log file */
#define VFSSTAT_MASTERJRNL 3 /* Master journal */
#define VFSSTAT_SUBJRNL 4 /* Subjournal */
#define VFSSTAT_TEMPDB 5 /* TEMP database */
#define VFSSTAT_TEMPJRNL 6 /* Journal for TEMP database */
#define VFSSTAT_TRANSIENT 7 /* Transient database */
#define VFSSTAT_ANY 8 /* Unspecified file type */
#define VFSSTAT_nFile 9 /* This many file types */
/* Names of the file types. These are allowed values for the
** first column of the vfsstat virtual table.
*/
static const char *azFile[] = {
"database", "journal", "wal", "master-journal", "sub-journal",
"temp-database", "temp-journal", "transient-db", "*"
};
/*
** Stat types
*/
#define VFSSTAT_BYTESIN 0 /* Bytes read in */
#define VFSSTAT_BYTESOUT 1 /* Bytes written out */
#define VFSSTAT_READ 2 /* Read requests */
#define VFSSTAT_WRITE 3 /* Write requests */
#define VFSSTAT_SYNC 4 /* Syncs */
#define VFSSTAT_OPEN 5 /* File opens */
#define VFSSTAT_LOCK 6 /* Lock requests */
#define VFSSTAT_ACCESS 0 /* xAccess calls. filetype==ANY only */
#define VFSSTAT_DELETE 1 /* xDelete calls. filetype==ANY only */
#define VFSSTAT_FULLPATH 2 /* xFullPathname calls. ANY only */
#define VFSSTAT_RANDOM 3 /* xRandomness calls. ANY only */
#define VFSSTAT_SLEEP 4 /* xSleep calls. ANY only */
#define VFSSTAT_CURTIME 5 /* xCurrentTime calls. ANY only */
#define VFSSTAT_nStat 7 /* This many stat types */
/* Names for the second column of the vfsstat virtual table for all
** cases except when the first column is "*" or VFSSTAT_ANY. */
static const char *azStat[] = {
"bytes-in", "bytes-out", "read", "write", "sync", "open", "lock",
};
static const char *azStatAny[] = {
"access", "delete", "fullpathname", "randomness", "sleep", "currenttimestamp",
"not-used"
};
/* Total number of counters */
#define VFSSTAT_MXCNT (VFSSTAT_nStat*VFSSTAT_nFile)
/*
** Performance stats are collected in an instance of the following
** global array.
*/
static sqlite3_uint64 aVfsCnt[VFSSTAT_MXCNT];
/*
** Access to a specific counter
*/
#define STATCNT(filetype,stat) (aVfsCnt[(filetype)*VFSSTAT_nStat+(stat)])
/*
** Forward declaration of objects used by this utility
*/
typedef struct VStatVfs VStatVfs;
typedef struct VStatFile VStatFile;
/* An instance of the VFS */
struct VStatVfs {
sqlite3_vfs base; /* VFS methods */
sqlite3_vfs *pVfs; /* Parent VFS */
};
/* An open file */
struct VStatFile {
sqlite3_file base; /* IO methods */
sqlite3_file *pReal; /* Underlying file handle */
unsigned char eFiletype; /* What type of file is this */
};
#define REALVFS(p) (((VStatVfs*)(p))->pVfs)
/*
** Methods for VStatFile
*/
static int vstatClose(sqlite3_file*);
static int vstatRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst);
static int vstatWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst);
static int vstatTruncate(sqlite3_file*, sqlite3_int64 size);
static int vstatSync(sqlite3_file*, int flags);
static int vstatFileSize(sqlite3_file*, sqlite3_int64 *pSize);
static int vstatLock(sqlite3_file*, int);
static int vstatUnlock(sqlite3_file*, int);
static int vstatCheckReservedLock(sqlite3_file*, int *pResOut);
static int vstatFileControl(sqlite3_file*, int op, void *pArg);
static int vstatSectorSize(sqlite3_file*);
static int vstatDeviceCharacteristics(sqlite3_file*);
static int vstatShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**);
static int vstatShmLock(sqlite3_file*, int offset, int n, int flags);
static void vstatShmBarrier(sqlite3_file*);
static int vstatShmUnmap(sqlite3_file*, int deleteFlag);
static int vstatFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp);
static int vstatUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p);
/*
** Methods for VStatVfs
*/
static int vstatOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *);
static int vstatDelete(sqlite3_vfs*, const char *zName, int syncDir);
static int vstatAccess(sqlite3_vfs*, const char *zName, int flags, int *);
static int vstatFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut);
static void *vstatDlOpen(sqlite3_vfs*, const char *zFilename);
static void vstatDlError(sqlite3_vfs*, int nByte, char *zErrMsg);
static void (*vstatDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void);
static void vstatDlClose(sqlite3_vfs*, void*);
static int vstatRandomness(sqlite3_vfs*, int nByte, char *zOut);
static int vstatSleep(sqlite3_vfs*, int microseconds);
static int vstatCurrentTime(sqlite3_vfs*, double*);
static int vstatGetLastError(sqlite3_vfs*, int, char *);
static int vstatCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*);
static VStatVfs vstat_vfs = {
{
2, /* iVersion */
0, /* szOsFile (set by register_vstat()) */
1024, /* mxPathname */
0, /* pNext */
"vfslog", /* zName */
0, /* pAppData */
vstatOpen, /* xOpen */
vstatDelete, /* xDelete */
vstatAccess, /* xAccess */
vstatFullPathname, /* xFullPathname */
vstatDlOpen, /* xDlOpen */
vstatDlError, /* xDlError */
vstatDlSym, /* xDlSym */
vstatDlClose, /* xDlClose */
vstatRandomness, /* xRandomness */
vstatSleep, /* xSleep */
vstatCurrentTime, /* xCurrentTime */
vstatGetLastError, /* xGetLastError */
vstatCurrentTimeInt64 /* xCurrentTimeInt64 */
},
0
};
static const sqlite3_io_methods vstat_io_methods = {
3, /* iVersion */
vstatClose, /* xClose */
vstatRead, /* xRead */
vstatWrite, /* xWrite */
vstatTruncate, /* xTruncate */
vstatSync, /* xSync */
vstatFileSize, /* xFileSize */
vstatLock, /* xLock */
vstatUnlock, /* xUnlock */
vstatCheckReservedLock, /* xCheckReservedLock */
vstatFileControl, /* xFileControl */
vstatSectorSize, /* xSectorSize */
vstatDeviceCharacteristics, /* xDeviceCharacteristics */
vstatShmMap, /* xShmMap */
vstatShmLock, /* xShmLock */
vstatShmBarrier, /* xShmBarrier */
vstatShmUnmap, /* xShmUnmap */
vstatFetch, /* xFetch */
vstatUnfetch /* xUnfetch */
};
/*
** Close an vstat-file.
*/
static int vstatClose(sqlite3_file *pFile){
VStatFile *p = (VStatFile *)pFile;
int rc = SQLITE_OK;
if( p->pReal->pMethods ){
rc = p->pReal->pMethods->xClose(p->pReal);
}
return rc;
}
/*
** Read data from an vstat-file.
*/
static int vstatRead(
sqlite3_file *pFile,
void *zBuf,
int iAmt,
sqlite_int64 iOfst
){
int rc;
VStatFile *p = (VStatFile *)pFile;
rc = p->pReal->pMethods->xRead(p->pReal, zBuf, iAmt, iOfst);
STATCNT(p->eFiletype,VFSSTAT_READ)++;
if( rc==SQLITE_OK ){
STATCNT(p->eFiletype,VFSSTAT_BYTESIN) += iAmt;
}
return rc;
}
/*
** Write data to an vstat-file.
*/
static int vstatWrite(
sqlite3_file *pFile,
const void *z,
int iAmt,
sqlite_int64 iOfst
){
int rc;
VStatFile *p = (VStatFile *)pFile;
rc = p->pReal->pMethods->xWrite(p->pReal, z, iAmt, iOfst);
STATCNT(p->eFiletype,VFSSTAT_WRITE)++;
if( rc==SQLITE_OK ){
STATCNT(p->eFiletype,VFSSTAT_BYTESOUT) += iAmt;
}
return rc;
}
/*
** Truncate an vstat-file.
*/
static int vstatTruncate(sqlite3_file *pFile, sqlite_int64 size){
int rc;
VStatFile *p = (VStatFile *)pFile;
rc = p->pReal->pMethods->xTruncate(p->pReal, size);
return rc;
}
/*
** Sync an vstat-file.
*/
static int vstatSync(sqlite3_file *pFile, int flags){
int rc;
VStatFile *p = (VStatFile *)pFile;
rc = p->pReal->pMethods->xSync(p->pReal, flags);
STATCNT(p->eFiletype,VFSSTAT_SYNC)++;
return rc;
}
/*
** Return the current file-size of an vstat-file.
*/
static int vstatFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){
int rc;
VStatFile *p = (VStatFile *)pFile;
rc = p->pReal->pMethods->xFileSize(p->pReal, pSize);
return rc;
}
/*
** Lock an vstat-file.
*/
static int vstatLock(sqlite3_file *pFile, int eLock){
int rc;
VStatFile *p = (VStatFile *)pFile;
rc = p->pReal->pMethods->xLock(p->pReal, eLock);
STATCNT(p->eFiletype,VFSSTAT_LOCK)++;
return rc;
}
/*
** Unlock an vstat-file.
*/
static int vstatUnlock(sqlite3_file *pFile, int eLock){
int rc;
VStatFile *p = (VStatFile *)pFile;
rc = p->pReal->pMethods->xUnlock(p->pReal, eLock);
STATCNT(p->eFiletype,VFSSTAT_LOCK)++;
return rc;
}
/*
** Check if another file-handle holds a RESERVED lock on an vstat-file.
*/
static int vstatCheckReservedLock(sqlite3_file *pFile, int *pResOut){
int rc;
VStatFile *p = (VStatFile *)pFile;
rc = p->pReal->pMethods->xCheckReservedLock(p->pReal, pResOut);
STATCNT(p->eFiletype,VFSSTAT_LOCK)++;
return rc;
}
/*
** File control method. For custom operations on an vstat-file.
*/
static int vstatFileControl(sqlite3_file *pFile, int op, void *pArg){
VStatFile *p = (VStatFile *)pFile;
int rc;
rc = p->pReal->pMethods->xFileControl(p->pReal, op, pArg);
if( op==SQLITE_FCNTL_VFSNAME && rc==SQLITE_OK ){
*(char**)pArg = sqlite3_mprintf("vstat/%z", *(char**)pArg);
}
return rc;
}
/*
** Return the sector-size in bytes for an vstat-file.
*/
static int vstatSectorSize(sqlite3_file *pFile){
int rc;
VStatFile *p = (VStatFile *)pFile;
rc = p->pReal->pMethods->xSectorSize(p->pReal);
return rc;
}
/*
** Return the device characteristic flags supported by an vstat-file.
*/
static int vstatDeviceCharacteristics(sqlite3_file *pFile){
int rc;
VStatFile *p = (VStatFile *)pFile;
rc = p->pReal->pMethods->xDeviceCharacteristics(p->pReal);
return rc;
}
/* Create a shared memory file mapping */
static int vstatShmMap(
sqlite3_file *pFile,
int iPg,
int pgsz,
int bExtend,
void volatile **pp
){
VStatFile *p = (VStatFile *)pFile;
return p->pReal->pMethods->xShmMap(p->pReal, iPg, pgsz, bExtend, pp);
}
/* Perform locking on a shared-memory segment */
static int vstatShmLock(sqlite3_file *pFile, int offset, int n, int flags){
VStatFile *p = (VStatFile *)pFile;
return p->pReal->pMethods->xShmLock(p->pReal, offset, n, flags);
}
/* Memory barrier operation on shared memory */
static void vstatShmBarrier(sqlite3_file *pFile){
VStatFile *p = (VStatFile *)pFile;
p->pReal->pMethods->xShmBarrier(p->pReal);
}
/* Unmap a shared memory segment */
static int vstatShmUnmap(sqlite3_file *pFile, int deleteFlag){
VStatFile *p = (VStatFile *)pFile;
return p->pReal->pMethods->xShmUnmap(p->pReal, deleteFlag);
}
/* Fetch a page of a memory-mapped file */
static int vstatFetch(
sqlite3_file *pFile,
sqlite3_int64 iOfst,
int iAmt,
void **pp
){
VStatFile *p = (VStatFile *)pFile;
return p->pReal->pMethods->xFetch(p->pReal, iOfst, iAmt, pp);
}
/* Release a memory-mapped page */
static int vstatUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){
VStatFile *p = (VStatFile *)pFile;
return p->pReal->pMethods->xUnfetch(p->pReal, iOfst, pPage);
}
/*
** Open an vstat file handle.
*/
static int vstatOpen(
sqlite3_vfs *pVfs,
const char *zName,
sqlite3_file *pFile,
int flags,
int *pOutFlags
){
int rc;
VStatFile *p = (VStatFile*)pFile;
p->pReal = (sqlite3_file*)&p[1];
rc = REALVFS(pVfs)->xOpen(REALVFS(pVfs), zName, p->pReal, flags, pOutFlags);
if( flags & SQLITE_OPEN_MAIN_DB ){
p->eFiletype = VFSSTAT_MAIN;
}else if( flags & SQLITE_OPEN_MAIN_JOURNAL ){
p->eFiletype = VFSSTAT_JOURNAL;
}else if( flags & SQLITE_OPEN_WAL ){
p->eFiletype = VFSSTAT_WAL;
}else if( flags & SQLITE_OPEN_MASTER_JOURNAL ){
p->eFiletype = VFSSTAT_MASTERJRNL;
}else if( flags & SQLITE_OPEN_SUBJOURNAL ){
p->eFiletype = VFSSTAT_SUBJRNL;
}else if( flags & SQLITE_OPEN_TEMP_DB ){
p->eFiletype = VFSSTAT_TEMPDB;
}else if( flags & SQLITE_OPEN_TEMP_JOURNAL ){
p->eFiletype = VFSSTAT_TEMPJRNL;
}else{
p->eFiletype = VFSSTAT_TRANSIENT;
}
STATCNT(p->eFiletype,VFSSTAT_OPEN)++;
pFile->pMethods = rc ? 0 : &vstat_io_methods;
return rc;
}
/*
** Delete the file located at zPath. If the dirSync argument is true,
** ensure the file-system modifications are synced to disk before
** returning.
*/
static int vstatDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){
int rc;
rc = REALVFS(pVfs)->xDelete(REALVFS(pVfs), zPath, dirSync);
STATCNT(VFSSTAT_ANY,VFSSTAT_DELETE)++;
return rc;
}
/*
** Test for access permissions. Return true if the requested permission
** is available, or false otherwise.
*/
static int vstatAccess(
sqlite3_vfs *pVfs,
const char *zPath,
int flags,
int *pResOut
){
int rc;
rc = REALVFS(pVfs)->xAccess(REALVFS(pVfs), zPath, flags, pResOut);
STATCNT(VFSSTAT_ANY,VFSSTAT_ACCESS)++;
return rc;
}
/*
** Populate buffer zOut with the full canonical pathname corresponding
** to the pathname in zPath. zOut is guaranteed to point to a buffer
** of at least (INST_MAX_PATHNAME+1) bytes.
*/
static int vstatFullPathname(
sqlite3_vfs *pVfs,
const char *zPath,
int nOut,
char *zOut
){
STATCNT(VFSSTAT_ANY,VFSSTAT_FULLPATH)++;
return REALVFS(pVfs)->xFullPathname(REALVFS(pVfs), zPath, nOut, zOut);
}
/*
** Open the dynamic library located at zPath and return a handle.
*/
static void *vstatDlOpen(sqlite3_vfs *pVfs, const char *zPath){
return REALVFS(pVfs)->xDlOpen(REALVFS(pVfs), zPath);
}
/*
** Populate the buffer zErrMsg (size nByte bytes) with a human readable
** utf-8 string describing the most recent error encountered associated
** with dynamic libraries.
*/
static void vstatDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){
REALVFS(pVfs)->xDlError(REALVFS(pVfs), nByte, zErrMsg);
}
/*
** Return a pointer to the symbol zSymbol in the dynamic library pHandle.
*/
static void (*vstatDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){
return REALVFS(pVfs)->xDlSym(REALVFS(pVfs), p, zSym);
}
/*
** Close the dynamic library handle pHandle.
*/
static void vstatDlClose(sqlite3_vfs *pVfs, void *pHandle){
REALVFS(pVfs)->xDlClose(REALVFS(pVfs), pHandle);
}
/*
** Populate the buffer pointed to by zBufOut with nByte bytes of
** random data.
*/
static int vstatRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){
STATCNT(VFSSTAT_ANY,VFSSTAT_RANDOM)++;
return REALVFS(pVfs)->xRandomness(REALVFS(pVfs), nByte, zBufOut);
}
/*
** Sleep for nMicro microseconds. Return the number of microseconds
** actually slept.
*/
static int vstatSleep(sqlite3_vfs *pVfs, int nMicro){
STATCNT(VFSSTAT_ANY,VFSSTAT_SLEEP)++;
return REALVFS(pVfs)->xSleep(REALVFS(pVfs), nMicro);
}
/*
** Return the current time as a Julian Day number in *pTimeOut.
*/
static int vstatCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){
STATCNT(VFSSTAT_ANY,VFSSTAT_CURTIME)++;
return REALVFS(pVfs)->xCurrentTime(REALVFS(pVfs), pTimeOut);
}
static int vstatGetLastError(sqlite3_vfs *pVfs, int a, char *b){
return REALVFS(pVfs)->xGetLastError(REALVFS(pVfs), a, b);
}
static int vstatCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){
STATCNT(VFSSTAT_ANY,VFSSTAT_CURTIME)++;
return REALVFS(pVfs)->xCurrentTimeInt64(REALVFS(pVfs), p);
}
/*
** A virtual table for accessing the stats collected by this VFS shim
*/
static int vstattabConnect(sqlite3*, void*, int, const char*const*,
sqlite3_vtab**,char**);
static int vstattabBestIndex(sqlite3_vtab*,sqlite3_index_info*);
static int vstattabDisconnect(sqlite3_vtab*);
static int vstattabOpen(sqlite3_vtab*, sqlite3_vtab_cursor**);
static int vstattabClose(sqlite3_vtab_cursor*);
static int vstattabFilter(sqlite3_vtab_cursor*, int idxNum, const char *idxStr,
int argc, sqlite3_value **argv);
static int vstattabNext(sqlite3_vtab_cursor*);
static int vstattabEof(sqlite3_vtab_cursor*);
static int vstattabColumn(sqlite3_vtab_cursor*,sqlite3_context*,int);
static int vstattabRowid(sqlite3_vtab_cursor*,sqlite3_int64*);
static int vstattabUpdate(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*);
/* A cursor for the vfsstat virtual table */
typedef struct VfsStatCursor {
sqlite3_vtab_cursor base; /* Base class. Must be first */
int i; /* Pointing to this aVfsCnt[] value */
} VfsStatCursor;
static int vstattabConnect(
sqlite3 *db,
void *pAux,
int argc, const char *const*argv,
sqlite3_vtab **ppVtab,
char **pzErr
){
sqlite3_vtab *pNew;
int rc;
/* Column numbers */
#define VSTAT_COLUMN_FILE 0
#define VSTAT_COLUMN_STAT 1
#define VSTAT_COLUMN_COUNT 2
rc = sqlite3_declare_vtab(db,"CREATE TABLE x(file,stat,count)");
if( rc==SQLITE_OK ){
pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) );
if( pNew==0 ) return SQLITE_NOMEM;
memset(pNew, 0, sizeof(*pNew));
}
return rc;
}
/*
** This method is the destructor for vstat table object.
*/
static int vstattabDisconnect(sqlite3_vtab *pVtab){
sqlite3_free(pVtab);
return SQLITE_OK;
}
/*
** Constructor for a new vstat table cursor object.
*/
static int vstattabOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
VfsStatCursor *pCur;
pCur = sqlite3_malloc( sizeof(*pCur) );
if( pCur==0 ) return SQLITE_NOMEM;
memset(pCur, 0, sizeof(*pCur));
*ppCursor = &pCur->base;
return SQLITE_OK;
}
/*
** Destructor for a VfsStatCursor.
*/
static int vstattabClose(sqlite3_vtab_cursor *cur){
sqlite3_free(cur);
return SQLITE_OK;
}
/*
** Advance a VfsStatCursor to its next row of output.
*/
static int vstattabNext(sqlite3_vtab_cursor *cur){
((VfsStatCursor*)cur)->i++;
return SQLITE_OK;
}
/*
** Return values of columns for the row at which the VfsStatCursor
** is currently pointing.
*/
static int vstattabColumn(
sqlite3_vtab_cursor *cur, /* The cursor */
sqlite3_context *ctx, /* First argument to sqlite3_result_...() */
int i /* Which column to return */
){
VfsStatCursor *pCur = (VfsStatCursor*)cur;
switch( i ){
case VSTAT_COLUMN_FILE: {
sqlite3_result_text(ctx, azFile[pCur->i/VFSSTAT_nStat], -1, SQLITE_STATIC);
break;
}
case VSTAT_COLUMN_STAT: {
const char **az;
az = (pCur->i/VFSSTAT_nStat)==VFSSTAT_ANY ? azStatAny : azStat;
sqlite3_result_text(ctx, az[pCur->i%VFSSTAT_nStat], -1, SQLITE_STATIC);
break;
}
case VSTAT_COLUMN_COUNT: {
sqlite3_result_int64(ctx, aVfsCnt[pCur->i]);
break;
}
}
return SQLITE_OK;
}
/*
** Return the rowid for the current row.
*/
static int vstattabRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
VfsStatCursor *pCur = (VfsStatCursor*)cur;
*pRowid = pCur->i;
return SQLITE_OK;
}
/*
** Return TRUE if the cursor has been moved off of the last
** row of output.
*/
static int vstattabEof(sqlite3_vtab_cursor *cur){
VfsStatCursor *pCur = (VfsStatCursor*)cur;
return pCur->i >= VFSSTAT_MXCNT;
}
/*
** Only a full table scan is supported. So xFilter simply rewinds to
** the beginning.
*/
static int vstattabFilter(
sqlite3_vtab_cursor *pVtabCursor,
int idxNum, const char *idxStr,
int argc, sqlite3_value **argv
){
VfsStatCursor *pCur = (VfsStatCursor*)pVtabCursor;
pCur->i = 0;
return SQLITE_OK;
}
/*
** Only a forwards full table scan is supported. xBestIndex is a no-op.
*/
static int vstattabBestIndex(
sqlite3_vtab *tab,
sqlite3_index_info *pIdxInfo
){
return SQLITE_OK;
}
/*
** Any VSTAT_COLUMN_COUNT can be changed to a positive integer.
** No deletions or insertions are allowed. No changes to other
** columns are allowed.
*/
static int vstattabUpdate(
sqlite3_vtab *tab,
int argc, sqlite3_value **argv,
sqlite3_int64 *pRowid
){
sqlite3_int64 iRowid, x;
if( argc==1 ) return SQLITE_ERROR;
if( sqlite3_value_type(argv[0])!=SQLITE_INTEGER ) return SQLITE_ERROR;
iRowid = sqlite3_value_int64(argv[0]);
if( iRowid!=sqlite3_value_int64(argv[1]) ) return SQLITE_ERROR;
if( iRowid<0 || iRowid>=VFSSTAT_MXCNT ) return SQLITE_ERROR;
if( sqlite3_value_type(argv[VSTAT_COLUMN_COUNT+2])!=SQLITE_INTEGER ){
return SQLITE_ERROR;
}
x = sqlite3_value_int64(argv[VSTAT_COLUMN_COUNT+2]);
if( x<0 ) return SQLITE_ERROR;
aVfsCnt[iRowid] = x;
return SQLITE_OK;
}
static sqlite3_module VfsStatModule = {
0, /* iVersion */
0, /* xCreate */
vstattabConnect, /* xConnect */
vstattabBestIndex, /* xBestIndex */
vstattabDisconnect, /* xDisconnect */
0, /* xDestroy */
vstattabOpen, /* xOpen - open a cursor */
vstattabClose, /* xClose - close a cursor */
vstattabFilter, /* xFilter - configure scan constraints */
vstattabNext, /* xNext - advance a cursor */
vstattabEof, /* xEof - check for end of scan */
vstattabColumn, /* xColumn - read data */
vstattabRowid, /* xRowid - read data */
vstattabUpdate, /* xUpdate */
0, /* xBegin */
0, /* xSync */
0, /* xCommit */
0, /* xRollback */
0, /* xFindMethod */
0, /* xRename */
};
/*
** This routine is an sqlite3_auto_extension() callback, invoked to register
** the vfsstat virtual table for all new database connections.
*/
static int vstatRegister(
sqlite3 *db,
const char **pzErrMsg,
const struct sqlite3_api_routines *pThunk
){
return sqlite3_create_module(db, "vfsstat", &VfsStatModule, 0);
}
#ifdef _WIN32
__declspec(dllexport)
#endif
/*
** This routine is called when the extension is loaded.
**
** Register the new VFS. Make arrangement to register the virtual table
** for each new database connection.
*/
int sqlite3_vfsstat_init(
sqlite3 *db,
char **pzErrMsg,
const sqlite3_api_routines *pApi
){
int rc = SQLITE_OK;
SQLITE_EXTENSION_INIT2(pApi);
vstat_vfs.pVfs = sqlite3_vfs_find(0);
vstat_vfs.base.szOsFile = sizeof(VStatFile) + vstat_vfs.pVfs->szOsFile;
rc = sqlite3_vfs_register(&vstat_vfs.base, 1);
if( rc==SQLITE_OK ){
rc = sqlite3_auto_extension(vstatRegister);
}
if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY;
return rc;
}

2
ext/misc/vtshim.c
View File

@@ -95,6 +95,7 @@ static int vtshimCreate(
if( rc ){
sqlite3_free(pNew);
*ppVtab = 0;
return rc;
}
pNew->pAux = pAux;
pNew->ppPrev = &pAux->pAllVtab;
@@ -133,6 +134,7 @@ static int vtshimConnect(
if( rc ){
sqlite3_free(pNew);
*ppVtab = 0;
return rc;
}
pNew->pAux = pAux;
pNew->ppPrev = &pAux->pAllVtab;

56
ext/rbu/rbu.c
View File

@@ -24,12 +24,22 @@
*/
void usage(const char *zArgv0){
fprintf(stderr,
"Usage: %s [-step NSTEP] TARGET-DB RBU-DB\n"
"Usage: %s ?OPTIONS? TARGET-DB RBU-DB\n"
"\n"
" Argument RBU-DB must be an RBU database containing an update suitable for\n"
" target database TARGET-DB. If NSTEP is set to less than or equal to zero\n"
" (the default value), this program attempts to apply the entire update to\n"
" the target database.\n"
"Where options are:\n"
"\n"
" -step NSTEP\n"
" -vacuum\n"
"\n"
" If the -vacuum switch is not present, argument RBU-DB must be an RBU\n"
" database containing an update suitable for target database TARGET-DB.\n"
" Or, if -vacuum is specified, then TARGET-DB is a database to vacuum using\n"
" RBU, and RBU-DB is used as the state database for the vacuum (refer to\n"
" API documentation for details).\n"
"\n"
" If NSTEP is set to less than or equal to zero (the default value), this \n"
" program attempts to perform the entire update or vacuum operation before\n"
" exiting\n"
"\n"
" If NSTEP is greater than zero, then a maximum of NSTEP calls are made\n"
" to sqlite3rbu_step(). If the RBU update has not been completely applied\n"
@@ -69,29 +79,43 @@ int main(int argc, char **argv){
char *zErrmsg; /* Error message, if any */
sqlite3rbu *pRbu; /* RBU handle */
int nStep = 0; /* Maximum number of step() calls */
int bVacuum = 0;
int rc;
sqlite3_int64 nProgress = 0;
int nArg = argc-2;
/* Process command line arguments. Following this block local variables
** zTarget, zRbu and nStep are all set. */
if( argc==5 ){
size_t nArg1 = strlen(argv[1]);
if( nArg1>5 || nArg1<2 || memcmp("-step", argv[1], nArg1) ) usage(argv[0]);
nStep = atoi(argv[2]);
}else if( argc!=3 ){
usage(argv[0]);
if( argc<3 ) usage(argv[0]);
for(i=1; i<nArg; i++){
const char *zArg = argv[i];
int nArg = strlen(zArg);
if( nArg>1 && nArg<=8 && 0==memcmp(zArg, "-vacuum", nArg) ){
bVacuum = 1;
}else if( nArg>1 && nArg<=5 && 0==memcmp(zArg, "-step", nArg) && i<nArg-1 ){
i++;
nStep = atoi(argv[i]);
}else{
usage(argv[0]);
}
}
zTarget = argv[argc-2];
zRbu = argv[argc-1];
report_default_vfs();
/* Open an RBU handle. If nStep is less than or equal to zero, call
/* Open an RBU handle. A vacuum handle if -vacuum was specified, or a
** regular RBU update handle otherwise. */
if( bVacuum ){
pRbu = sqlite3rbu_vacuum(zTarget, zRbu);
}else{
pRbu = sqlite3rbu_open(zTarget, zRbu, 0);
}
report_rbu_vfs(pRbu);
/* If nStep is less than or equal to zero, call
** sqlite3rbu_step() until either the RBU has been completely applied
** or an error occurs. Or, if nStep is greater than zero, call
** sqlite3rbu_step() a maximum of nStep times. */
pRbu = sqlite3rbu_open(zTarget, zRbu, 0);
report_rbu_vfs(pRbu);
for(i=0; (nStep<=0 || i<nStep) && sqlite3rbu_step(pRbu)==SQLITE_OK; i++);
nProgress = sqlite3rbu_progress(pRbu);
rc = sqlite3rbu_close(pRbu, &zErrmsg);

44
ext/rbu/rbu1.test
View File

@@ -10,10 +10,7 @@
#***********************************************************************
#
if {![info exists testdir]} {
set testdir [file join [file dirname [info script]] .. .. test]
}
source $testdir/tester.tcl
source [file join [file dirname [info script]] rbu_common.tcl]
set ::testprefix rbu1
db close
@@ -96,26 +93,6 @@ proc create_rbu5 {filename} {
return $filename
}
# Run the RBU in file $rbu on target database $target until completion.
#
proc run_rbu {target rbu} {
sqlite3rbu rbu $target $rbu
while 1 {
set rc [rbu step]
if {$rc!="SQLITE_OK"} break
}
rbu close
}
proc step_rbu {target rbu} {
while 1 {
sqlite3rbu rbu $target $rbu
set rc [rbu step]
rbu close
if {$rc != "SQLITE_OK"} break
}
set rc
}
# Same as [step_rbu], except using a URI to open the target db.
#
@@ -611,7 +588,7 @@ foreach {tn3 create_vfs destroy_vfs} {
9 {
CREATE TABLE t1(a, b PRIMARY KEY) WITHOUT ROWID;
CREATE TABLE rbu.data_t1(a, b, rbu_control);
INSERT INTO rbu.data_t1 VALUES(1, 2, 2);
INSERT INTO rbu.data_t1 VALUES(1, 2, 3);
} {SQLITE_ERROR - invalid rbu_control value}
10 {
@@ -641,7 +618,22 @@ foreach {tn3 create_vfs destroy_vfs} {
# correctly.
reset_db
forcedelete rbu.db
do_test $tn3.8 {
do_test $tn3.8.1 {
list [catch { run_rbu test.db rbu.db } msg] $msg
} {0 SQLITE_DONE}
# Test that an RBU database containing only empty data_xxx tables is
# also handled correctly.
reset_db
forcedelete rbu.db
do_execsql_test $tn3.8.2.1 {
CREATE TABLE t1(a PRIMARY KEY, b);
INSERT INTO t1 VALUES(1, 2);
ATTACH 'rbu.db' AS rbu;
CREATE TABLE data_t1(a, b, rbu_control);
DETACH rbu;
}
do_test $tn3.8.2.1 {
list [catch { run_rbu test.db rbu.db } msg] $msg
} {0 SQLITE_DONE}

27
ext/rbu/rbu5.test
View File

@@ -12,35 +12,10 @@
# Test some properties of the pager_rbu_mode and rbu_mode pragmas.
#
if {![info exists testdir]} {
set testdir [file join [file dirname [info script]] .. .. test]
}
source $testdir/tester.tcl
source [file join [file dirname [info script]] rbu_common.tcl]
set ::testprefix rbu5
# Run the RBU in file $rbu on target database $target until completion.
#
proc run_rbu {target rbu} {
sqlite3rbu rbu $target $rbu
while { [rbu step]=="SQLITE_OK" } {}
rbu close
}
# Run the RBU in file $rbu on target database $target one step at a
# time until completion.
#
proc step_rbu {target rbu} {
while 1 {
sqlite3rbu rbu $target $rbu
set rc [rbu step]
rbu close
if {$rc != "SQLITE_OK"} break
}
set rc
}
# Return a list of the primary key columns for table $tbl in the database
# opened by database handle $db.
#

54
ext/rbu/rbu_common.tcl
View File

@@ -15,12 +15,43 @@ if {![info exists testdir]} {
}
source $testdir/tester.tcl
proc check_prestep_state {target state} {
set oal_exists [file exists $target-oal]
set wal_exists [file exists $target-wal]
set progress [rbu progress]
if {($progress==0 && $state!="oal" && $state!="done")
|| ($oal_exists && $wal_exists)
|| ($progress>0 && $state=="oal" && (!$oal_exists || $wal_exists))
|| ($state=="move" && (!$oal_exists || $wal_exists))
|| ($state=="checkpoint" && ($oal_exists || !$wal_exists))
|| ($state=="done" && ($oal_exists && $progress!=0))
} {
error "B: state=$state progress=$progress oal=$oal_exists wal=$wal_exists"
}
}
proc check_poststep_state {rc target state} {
if {$rc=="SQLITE_OK" || $rc=="SQLITE_DONE"} {
set oal_exists [file exists $target-oal]
set wal_exists [file exists $target-wal]
if {$state=="move" && ($oal_exists || !$wal_exists)} {
error "A: state=$state progress=$progress oal=$oal_exists wal=$wal_exists"
}
}
}
# Run the RBU in file $rbu on target database $target until completion.
#
proc run_rbu {target rbu} {
sqlite3rbu rbu $target $rbu
while 1 {
set state [rbu state]
check_prestep_state $target $state
set rc [rbu step]
check_poststep_state $rc $target $state
if {$rc!="SQLITE_OK"} break
}
rbu close
@@ -29,10 +60,33 @@ proc run_rbu {target rbu} {
proc step_rbu {target rbu} {
while 1 {
sqlite3rbu rbu $target $rbu
set state [rbu state]
check_prestep_state $target $state
set rc [rbu step]
check_poststep_state $rc $target $state
rbu close
if {$rc != "SQLITE_OK"} break
}
set rc
}
proc do_rbu_vacuum_test {tn step} {
uplevel [list do_test $tn.1 {
if {$step==0} { sqlite3rbu_vacuum rbu test.db state.db }
while 1 {
if {$step==1} { sqlite3rbu_vacuum rbu test.db state.db }
set state [rbu state]
check_prestep_state test.db $state
set rc [rbu step]
check_poststep_state $rc test.db $state
if {$rc!="SQLITE_OK"} break
if {$step==1} { rbu close }
}
rbu close
} {SQLITE_DONE}]
uplevel [list do_execsql_test $tn.2 {
PRAGMA integrity_check
} ok]
}

106
ext/rbu/rbucrash2.test Normal file
View File

@@ -0,0 +1,106 @@
# 2017 March 02
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
if {![info exists testdir]} {
set testdir [file join [file dirname [info script]] .. .. test]
}
source $testdir/tester.tcl
set ::testprefix rbucrash2
db close
forcedelete test.db-oal rbu.db
sqlite3_shutdown
sqlite3_config_uri 1
reset_db
# Set up a target database and an rbu update database. The target
# db is the usual "test.db", the rbu db is "test.db2".
#
forcedelete test.db2
do_execsql_test 1.0 {
CREATE TABLE t1(a, b, c, PRIMARY KEY(a), UNIQUE(b));
INSERT INTO t1 VALUES(1, 2, 3);
INSERT INTO t1 VALUES(4, 5, 6);
INSERT INTO t1 VALUES(7, 8, 9);
ATTACH 'test.db2' AS rbu;
CREATE TABLE rbu.data_t1(a, b, c, rbu_control);
INSERT INTO data_t1 VALUES('one', randomblob(3500), NULL, 0);
INSERT INTO data_t1 VALUES('two', randomblob(3500), NULL, 0);
INSERT INTO data_t1 VALUES('three', randomblob(3500), NULL, 0);
INSERT INTO data_t1 VALUES('four', randomblob(3500), NULL, 0);
INSERT INTO data_t1 VALUES('five', randomblob(3500), NULL, 0);
INSERT INTO data_t1 VALUES('six', randomblob(3500), NULL, 0);
}
db_save_and_close
proc do_rbu_crash_test2 {tn script} {
foreach {f blksz} {
test.db 512
test.db2 512
test.db 4096
test.db2 4096
} {
set bDone 0
for {set iDelay 1} {$bDone==0} {incr iDelay} {
forcedelete test.db2 test.db2-journal test.db test.db-oal test.db-wal
db_restore
set res [
crashsql -file $f -delay $iDelay -tclbody $script -dflt 1 -opendb {} \
-blocksize $blksz {}
]
set bDone 1
if {$res == "1 {child process exited abnormally}"} {
set bDone 0
} elseif {$res != "0 {}"} {
error "unexected catchsql result: $res"
}
sqlite3rbu rbu test.db test.db2
while {[rbu step]=="SQLITE_OK"} {}
rbu close
sqlite3 db test.db
do_execsql_test $tn.delay=$iDelay.f=$f.blksz=$blksz {
PRAGMA integrity_check;
} {ok}
db close
}
}
}
for {set x 1} {$x < 10} {incr x} {
do_rbu_crash_test2 1.$x {
sqlite3rbu rbu test.db test.db2
while {[rbu step]=="SQLITE_OK"} {
rbu savestate
}
rbu close
}
}
for {set x 1} {$x < 2} {incr x} {
do_rbu_crash_test2 2.$x {
sqlite3rbu rbu test.db test.db2
while {[rbu step]=="SQLITE_OK"} {
rbu close
sqlite3rbu rbu test.db test.db2
}
rbu close
}
}
finish_test

177
ext/rbu/rbudiff.test
View File

@@ -18,22 +18,17 @@ if {![info exists testdir]} {
source $testdir/tester.tcl
set testprefix rbudiff
if {$tcl_platform(platform)=="windows"} {
set PROG "sqldiff.exe"
} else {
set PROG "./sqldiff"
}
if {![file exe $PROG]} {
puts "rbudiff.test cannot run because $PROG is not available"
finish_test
return
}
set PROG [test_find_sqldiff]
db close
proc get_rbudiff_sql {db1 db2} {
exec $::PROG --rbu $db1 $db2
}
proc get_vtab_rbudiff_sql {db1 db2} {
exec $::PROG --vtab --rbu $db1 $db2
}
proc step_rbu {target rbu} {
while 1 {
sqlite3rbu rbu $target $rbu
@@ -45,6 +40,7 @@ proc step_rbu {target rbu} {
}
proc apply_rbudiff {sql target} {
test_rbucount $sql
forcedelete rbu.db
sqlite3 rbudb rbu.db
rbudb eval $sql
@@ -52,15 +48,43 @@ proc apply_rbudiff {sql target} {
step_rbu $target rbu.db
}
proc sqlesc {id} {
set ret "'[string map {' ''} $id]'"
set ret
}
# The only argument is the output of an [sqldiff -rbu] run. This command
# tests that the contents of the rbu_count table is correct. An exception
# is thrown if it is not.
#
proc test_rbucount {sql} {
sqlite3 tmpdb ""
tmpdb eval $sql
tmpdb eval {
SELECT name FROM sqlite_master WHERE name LIKE 'data%' AND type='table'
} {
set a [tmpdb eval "SELECT count(*) FROM [sqlesc $name]"]
set b [tmpdb eval {SELECT cnt FROM rbu_count WHERE tbl = $name}]
if {$a != $b} {
tmpdb close
error "rbu_count error - tbl = $name"
}
}
tmpdb close
return ""
}
proc rbudiff_cksum {db1} {
set txt ""
sqlite3 dbtmp $db1
foreach tbl [dbtmp eval {SELECT name FROM sqlite_master WHERE type='table'}] {
set cols [list]
dbtmp eval "PRAGMA table_info = $tbl" { lappend cols "quote( $name )" }
dbtmp eval "PRAGMA table_info = [sqlesc $tbl]" {
lappend cols "quote( $name )"
}
append txt [dbtmp eval \
"SELECT [join $cols {||'.'||}] FROM $tbl ORDER BY 1"
"SELECT [join $cols {||'.'||}] FROM [sqlesc $tbl] ORDER BY 1"
]
}
dbtmp close
@@ -116,6 +140,24 @@ foreach {tn init mod} {
);
}
4 {
CREATE TABLE x1(a, b, c, PRIMARY KEY(a, b, c));
INSERT INTO x1 VALUES('u', 'v', NULL);
INSERT INTO x1 VALUES('x', 'y', 'z');
INSERT INTO x1 VALUES('a', NULL, 'b');
} {
INSERT INTO x1 VALUES('a', 'b', 'c');
}
5 {
CREATE TABLE t1(a PRIMARY KEY, b);
INSERT INTO t1 VALUES(1, NULL);
INSERT INTO t1 VALUES(2, X'');
} {
UPDATE t1 SET b = X'' WHERE a=1;
UPDATE t1 SET b = NULL WHERE a=2;
}
} {
catch { db close }
@@ -146,5 +188,116 @@ foreach {tn init mod} {
do_test 1.$tn.5 { rbudiff_cksum test.db } [rbudiff_cksum test.db2]
}
#-------------------------------------------------------------------------
# Test that if the --vtab switch is present, [sqldiff] handles virtual
# table types fts[345] and rtree correctly.
#
ifcapable fts3&&fts5&&rtree {
foreach {tn init mod} {
1 {
CREATE VIRTUAL TABLE t1 USING fts5(c);
INSERT INTO t1 VALUES('a b c');
INSERT INTO t1 VALUES('a b c');
} {
DELETE FROM t1 WHERE rowid = 1;
INSERT INTO t1 VALUES('a b c');
}
2 {
CREATE VIRTUAL TABLE "x y" USING 'rtree'(id, x1, x2);
INSERT INTO "x y" VALUES(1, 2, 3);
INSERT INTO "x y" VALUES(2, 4, 6);
} {
DELETE FROM "x y" WHERE rowid = 1;
INSERT INTO "x y" VALUES(3, 6, 9);
}
3 {
CREATE VIRTUAL TABLE 'x''y' USING fts3;
INSERT INTO 'x''y' VALUES('one two three');
INSERT INTO 'x''y' VALUES('four five six');
} {
DELETE FROM 'x''y' WHERE rowid = 1;
INSERT INTO 'x''y' VALUES('one two three');
}
} {
forcedelete test.db test.db2
sqlite3 db test.db
db eval "$init"
sqlite3 db test.db2
db eval "$init ; $mod"
db close
do_test 2.$tn.1 {
set sql [get_vtab_rbudiff_sql test.db test.db2]
apply_rbudiff $sql test.db
} {SQLITE_DONE}
do_test 2.$tn.2 { rbudiff_cksum test.db } [rbudiff_cksum test.db2]
}
}
ifcapable fts5 {
foreach {tn init mod} {
1 {
CREATE VIRTUAL TABLE t1 USING fts5(c);
INSERT INTO t1 VALUES('a b c');
INSERT INTO t1 VALUES('a b c');
} {
DELETE FROM t1 WHERE rowid = 1;
INSERT INTO t1 VALUES('a b c');
}
2 {
CREATE VIRTUAL TABLE t1 USING FTs5(c);
INSERT INTO t1 VALUES('a b c');
INSERT INTO t1 VALUES('a b c');
} {
DELETE FROM t1 WHERE rowid = 1;
INSERT INTO t1 VALUES('a b c');
}
3 {
creAte virTUal
tablE t1 USING FTs5(c);
INSERT INTO t1 VALUES('a b c');
INSERT INTO t1 VALUES('a b c');
} {
DELETE FROM t1 WHERE rowid = 1;
INSERT INTO t1 VALUES('a b c');
}
} {
forcedelete test.db test.db2
sqlite3 db test.db
db eval "$init"
sqlite3 db test.db2
db eval "$init ; $mod"
db eval { INSERT INTO t1(t1) VALUES('optimize') }
db close
do_test 3.$tn.1 {
set sql [get_vtab_rbudiff_sql test.db test.db2]
apply_rbudiff $sql test.db
} {SQLITE_DONE}
sqlite3 db test.db
sqlite3 db2 test.db2
do_test 3.$tn.2 {
db2 eval { SELECT * FROM t1 ORDER BY rowid }
} [db eval { SELECT * FROM t1 ORDER BY rowid }]
do_test 3.$tn.3 {
db2 eval { INSERT INTO t1(t1) VALUES('integrity-check') }
} {}
db close
db2 close
}
}
finish_test

59
ext/rbu/rbudor.test Normal file
View File

@@ -0,0 +1,59 @@
# 2016 October 21
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This test file focuses on interactions between RBU and the feature
# enabled by SQLITE_DIRECT_OVERFLOW_READ - Direct Overflow Read.
#
if {![info exists testdir]} {
set testdir [file join [file dirname [info script]] .. .. test]
}
source $testdir/tester.tcl
set ::testprefix rbudor
set bigA [string repeat a 5000]
set bigB [string repeat b 5000]
do_execsql_test 1.0 {
PRAGMA page_size = 1024;
CREATE TABLE t1(a INTEGER PRIMARY KEY, b BLOB);
INSERT INTO t1 VALUES(1, $bigA);
} {}
do_test 1.1 {
forcedelete rbu.db
sqlite3 rbu rbu.db
rbu eval {
CREATE TABLE data_t1(a, b, rbu_control);
INSERT INTO data_t1 VALUES(2, $bigB, 0);
}
rbu close
} {}
do_test 1.2 {
sqlite3rbu rbu test.db rbu.db
while {[rbu state]!="checkpoint"} {
rbu step
}
rbu step
db eval { SELECT * FROM t1 }
} [list 1 $bigA 2 $bigB]
do_test 1.3 {
while {[rbu step]=="SQLITE_OK"} {}
rbu close
} {SQLITE_DONE}
do_execsql_test 1.4 {
SELECT * FROM t1
} [list 1 $bigA 2 $bigB]
finish_test

98
ext/rbu/rbufault3.test Normal file
View File

@@ -0,0 +1,98 @@
# 2016 April 20
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file contains fault injection tests for RBU vacuum operations.
#
source [file join [file dirname [info script]] rbu_common.tcl]
source $testdir/malloc_common.tcl
set ::testprefix rbufault3
foreach {fault errlist} {
oom-* {
{1 SQLITE_NOMEM}
{1 SQLITE_IOERR_NOMEM}
{1 {SQLITE_NOMEM - out of memory}}
}
ioerr-* {
{1 {SQLITE_IOERR - disk I/O error}}
{1 SQLITE_IOERR}
{1 SQLITE_IOERR_WRITE}
{1 SQLITE_IOERR_FSYNC}
{1 SQLITE_IOERR_READ}
{1 {SQLITE_IOERR - unable to open database: test.db2}}
{1 {SQLITE_ERROR - unable to open database: test.db2}}
{1 {SQLITE_ERROR - SQL logic error or missing database}}
}
cantopen* {
{1 {SQLITE_CANTOPEN - unable to open database: test.db2}}
{1 {SQLITE_CANTOPEN - unable to open database: test.db2}}
{1 {SQLITE_CANTOPEN - unable to open database file}}
{1 SQLITE_CANTOPEN}
}
} {
reset_db
do_execsql_test 0 {
CREATE TABLE target(x UNIQUE, y, z, PRIMARY KEY(y));
INSERT INTO target VALUES(1, 2, 3);
INSERT INTO target VALUES(4, 5, 6);
INSERT INTO target VALUES(7, 8, 9);
CREATE INDEX i1 ON target(z);
}
faultsim_save_and_close
do_faultsim_test 1 -faults $fault -prep {
faultsim_restore_and_reopen
forcedelete test.db2
} -body {
sqlite3rbu_vacuum rbu test.db test.db2
while {[rbu step]=="SQLITE_OK"} {}
rbu close
} -test {
eval [list faultsim_test_result {0 SQLITE_DONE} {*}$::errlist]
}
do_faultsim_test 2 -faults $fault -prep {
faultsim_restore_and_reopen
forcedelete test.db2
} -body {
sqlite3rbu_vacuum rbu test.db test.db2
rbu step
rbu close
} -test {
eval [list faultsim_test_result {0 SQLITE_OK} {*}$::errlist]
}
forcedelete test.db2
sqlite3rbu_vacuum rbu test.db test.db2
rbu step
rbu close
faultsim_save_and_close
do_faultsim_test 3 -faults $fault -prep {
faultsim_restore_and_reopen
forcedelete test.db2
} -body {
sqlite3rbu_vacuum rbu test.db test.db2
rbu step
rbu close
} -test {
eval [list faultsim_test_result {0 SQLITE_OK} {*}$::errlist]
}
}
finish_test

66
ext/rbu/rbufault4.test Normal file
View File

@@ -0,0 +1,66 @@
# 2014 October 22
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
if {![info exists testdir]} {
set testdir [file join [file dirname [info script]] .. .. test]
}
source $testdir/tester.tcl
source $testdir/malloc_common.tcl
set ::testprefix rbufault4
for {set tn 1} {1} {incr tn} {
reset_db
do_execsql_test 1.0 {
CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
CREATE INDEX i1b ON t1(b);
CREATE INDEX i1c ON t1(c);
INSERT INTO t1 VALUES(1, 2, 3);
INSERT INTO t1 VALUES(4, 5, 6);
}
forcedelete test.db2
sqlite3rbu_vacuum rbu test.db test.db2
for {set i 0} {$i < $tn} {incr i} { rbu step }
set rc [rbu close]
if {$rc!="SQLITE_OK"} {
if {$rc!="SQLITE_DONE"} {error $rc}
break
}
faultsim_save
do_faultsim_test $tn -faults oom-t* -prep {
faultsim_restore
} -body {
sqlite3rbu_vacuum rbu test.db test.db2
while 1 {
set rc [rbu step]
if {$rc=="SQLITE_DONE"} break
if {$rc!="SQLITE_OK"} { error $rc }
}
} -test {
catch {rbu close}
faultsim_test_result {0 {}} {1 SQLITE_NOMEM} {1 SQLITE_IOERR_NOMEM}
sqlite3rbu_vacuum rbu test.db test.db2
while {[rbu step]=="SQLITE_OK"} {}
set trc [rbu close]
if {$trc!="SQLITE_DONE"} { error "Got $trc instead of SQLITE_DONE!" }
set rc [db one {PRAGMA integrity_check}]
if {$rc!="ok"} { error "Got $rc instead of ok!" }
}
}
finish_test

419
ext/rbu/rbuprogress.test Normal file
View File

@@ -0,0 +1,419 @@
# 2016 March 18
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
source [file join [file dirname [info script]] rbu_common.tcl]
set ::testprefix rbuprogress
proc create_db_file {filename sql} {
forcedelete $filename
sqlite3 tmpdb $filename
tmpdb eval $sql
tmpdb close
}
# Create a simple RBU database. That expects to write to a table:
#
# CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
#
proc create_rbu1 {filename} {
create_db_file $filename {
CREATE TABLE data_t1(a, b, c, rbu_control);
INSERT INTO data_t1 VALUES(1, 2, 3, 0);
INSERT INTO data_t1 VALUES(2, 'two', 'three', 0);
INSERT INTO data_t1 VALUES(3, NULL, 8.2, 0);
CREATE TABLE rbu_count(tbl, cnt);
INSERT INTO rbu_count VALUES('data_t1', 3);
}
return $filename
}
do_execsql_test 1.0 {
PRAGMA page_size = 4096;
CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
}
do_test 1.1 {
create_rbu1 rbu.db
sqlite3rbu rbu test.db rbu.db
rbu bp_progress
} {0 0}
do_test 1.2 { rbu step ; rbu bp_progress } {3333 0}
do_test 1.3 { rbu step ; rbu bp_progress } {6666 0}
do_test 1.4 { rbu step ; rbu bp_progress } {10000 0}
do_test 1.5 { rbu step ; rbu bp_progress } {10000 0}
do_test 1.6 { rbu step ; rbu bp_progress } {10000 0}
do_test 1.7 { rbu step ; rbu bp_progress } {10000 5000}
do_test 1.8 { rbu step ; rbu bp_progress } {10000 10000}
do_test 1.9 { rbu step ; rbu bp_progress } {10000 10000}
do_test 1.10 {
rbu close
} {SQLITE_DONE}
#-------------------------------------------------------------------------
#
proc do_sp_test {tn bReopen target rbu reslist} {
uplevel [list do_test $tn [subst -nocommands {
if {$bReopen==0} { sqlite3rbu rbu $target $rbu }
set res [list]
while 1 {
if {$bReopen} { sqlite3rbu rbu $target $rbu }
set rc [rbu step]
if {[set rc] != "SQLITE_OK"} { rbu close ; error "error 1" }
lappend res [lindex [rbu bp_progress] 0]
if {[lindex [set res] end]==10000} break
if {$bReopen} { rbu close }
}
if {[set res] != [list $reslist]} {
rbu close
error "1. reslist incorrect (expect=$reslist got=[set res])"
}
# One step to clean up the temporary tables used to update the only
# target table in the rbu database. And one more to move the *-oal
# file to *-wal. After each of these steps, the progress remains
# at "10000 0".
#
if {[lindex [list $reslist] 0]!=-1} {
rbu step
set res [rbu bp_progress]
if {[set res] != [list 10000 0]} {
rbu close
error "2. reslist incorrect (expect=10000 0 got=[set res])"
}
}
rbu step
set res [rbu bp_progress]
if {[set res] != [list 10000 0]} {
rbu close
error "3. reslist incorrect (expect=10000 0 got=[set res])"
}
# Do the checkpoint.
while {[rbu step]=="SQLITE_OK"} {
foreach {a b} [rbu bp_progress] {}
if {[set a]!=10000 || [set b]<=0 || [set b]>10000} {
rbu close
error "4. reslist incorrect (expect=10000 1..10000 got=[set a] [set b])"
}
}
set res [rbu bp_progress]
if {[set res] != [list 10000 10000]} {
rbu close
error "5. reslist is incorrect (expect=10000 10000 got=[set res])"
}
rbu close
}] {SQLITE_DONE}]
}
foreach {bReopen} { 0 1 } {
reset_db
do_test 2.$bReopen.1.0 {
execsql {
CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
}
create_db_file rbu.db {
CREATE TABLE data_t1(a, b, c, rbu_control);
INSERT INTO data_t1 VALUES(4, 4, 4, 0);
INSERT INTO data_t1 VALUES(5, 5, 5, 0);
CREATE TABLE rbu_count(tbl, cnt);
INSERT INTO rbu_count VALUES('data_t1', 2);
}
} {}
do_sp_test 2.$bReopen.1.1 $bReopen test.db rbu.db {5000 10000}
reset_db
do_test 2.$bReopen.2.0 {
execsql { CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c) }
create_rbu1 rbu.db
} {rbu.db}
do_sp_test 2.$bReopen.2.1 $bReopen test.db rbu.db {3333 6666 10000}
reset_db
do_test 2.$bReopen.3.0 {
execsql {
CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
CREATE INDEX i1 ON t1(b);
INSERT INTO t1 VALUES(1, 1, 1);
INSERT INTO t1 VALUES(2, 2, 2);
INSERT INTO t1 VALUES(3, 3, 3);
}
create_db_file rbu.db {
CREATE TABLE data_t1(a, b, c, rbu_control);
INSERT INTO data_t1 VALUES(4, 4, 4, 0);
INSERT INTO data_t1 VALUES(2, NULL, NULL, 1);
INSERT INTO data_t1 VALUES(5, NULL, NULL, 1);
CREATE TABLE rbu_count(tbl, cnt);
INSERT INTO rbu_count VALUES('data_t1', 3);
}
} {}
do_sp_test 2.$bReopen.3.1 $bReopen test.db rbu.db {1666 3333 6000 8000 10000}
reset_db
do_test 2.$bReopen.4.0 {
execsql {
CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
CREATE INDEX i1 ON t1(b);
INSERT INTO t1 VALUES(1, 1, 1);
INSERT INTO t1 VALUES(2, 2, 2);
INSERT INTO t1 VALUES(3, 3, 3);
}
create_db_file rbu.db {
CREATE TABLE data_t1(a, b, c, rbu_control);
INSERT INTO data_t1 VALUES(2, 4, 4, '.xx');
CREATE TABLE rbu_count(tbl, cnt);
INSERT INTO rbu_count VALUES('data_t1', 1);
}
} {}
do_sp_test 2.$bReopen.4.1 $bReopen test.db rbu.db {3333 6666 10000}
reset_db
do_test 2.$bReopen.5.0 {
execsql {
CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
CREATE INDEX i1 ON t1(b);
INSERT INTO t1 VALUES(1, 1, 1);
INSERT INTO t1 VALUES(2, 2, 2);
INSERT INTO t1 VALUES(3, 3, 3);
}
create_db_file rbu.db {
CREATE TABLE data_t1(a, b, c, rbu_control);
INSERT INTO data_t1 VALUES(4, NULL, 4, '.xx');
CREATE TABLE rbu_count(tbl, cnt);
INSERT INTO rbu_count VALUES('data_t1', 1);
}
} {}
do_sp_test 2.$bReopen.5.1 $bReopen test.db rbu.db {10000}
reset_db
do_test 2.$bReopen.6.0 {
execsql {
CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
CREATE INDEX i1 ON t1(b);
INSERT INTO t1 VALUES(1, 1, 1);
INSERT INTO t1 VALUES(2, 2, 2);
INSERT INTO t1 VALUES(3, 3, 3);
}
create_db_file rbu.db {
CREATE TABLE data_t1(a, b, c, rbu_control);
INSERT INTO data_t1 VALUES(4, 4, 4, 0);
INSERT INTO data_t1 VALUES(2, NULL, NULL, 1);
INSERT INTO data_t1 VALUES(5, NULL, NULL, 1);
}
} {}
do_sp_test 2.$bReopen.6.1 $bReopen test.db rbu.db {-1 -1 -1 -1 -1 10000}
}
#-------------------------------------------------------------------------
# The following tests verify that the API works when resuming an update
# during the incremental checkpoint stage.
#
proc do_phase2_test {tn bReopen target rbu nStep} {
uplevel [list do_test $tn [subst -nocommands {
# Build the OAL/WAL file:
sqlite3rbu rbu $target $rbu
while {[lindex [rbu bp_progress] 0]<10000} {
set rc [rbu step]
if {"SQLITE_OK" != [set rc]} { rbu close }
}
# Clean up the temp tables and move the *-oal file to *-wal.
rbu step
rbu step
for {set i 0} {[set i] < $nStep} {incr i} {
if {$bReopen} {
rbu close
sqlite3rbu rbu $target $rbu
}
rbu step
set res [rbu bp_progress]
set expect [expr (1 + [set i]) * 10000 / $nStep]
if {[lindex [set res] 1] != [set expect]} {
error "Have [set res], expected 10000 [set expect]"
}
}
set rc [rbu step]
if {[set rc] != "SQLITE_DONE"} {
error "Have [set rc], expected SQLITE_DONE"
}
rbu close
}] {SQLITE_DONE}]
}
foreach bReopen {0 1} {
do_test 3.$bReopen.1.0 {
reset_db
execsql {
PRAGMA page_size = 4096;
CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
CREATE TABLE t2(a INTEGER PRIMARY KEY, b);
CREATE TABLE t3(a INTEGER PRIMARY KEY, b);
CREATE TABLE t4(a INTEGER PRIMARY KEY, b);
}
create_db_file rbu.db {
CREATE TABLE data_t1(a, b, rbu_control);
CREATE TABLE data_t2(a, b, rbu_control);
CREATE TABLE data_t3(a, b, rbu_control);
CREATE TABLE data_t4(a, b, rbu_control);
INSERT INTO data_t1 VALUES(1, 2, 0);
INSERT INTO data_t2 VALUES(1, 2, 0);
INSERT INTO data_t3 VALUES(1, 2, 0);
INSERT INTO data_t4 VALUES(1, 2, 0);
CREATE TABLE rbu_count(tbl, cnt);
INSERT INTO rbu_count VALUES('data_t1', 1);
INSERT INTO rbu_count VALUES('data_t2', 1);
INSERT INTO rbu_count VALUES('data_t3', 1);
INSERT INTO rbu_count VALUES('data_t4', 1);
}
} {}
do_phase2_test 3.$bReopen.1.1 $bReopen test.db rbu.db 5
}
foreach {bReopen} { 0 1 } {
foreach {tn tbl} {
ipk { CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c) }
wr { CREATE TABLE t1(a INT PRIMARY KEY, b, c) WITHOUT ROWID }
pk { CREATE TABLE t1(a INT PRIMARY KEY, b, c) }
} {
foreach {tn2 rbusql r1 r3} {
1 {
CREATE TABLE data0_t1(a, b, c, rbu_control);
INSERT INTO data0_t1 VALUES(15, 15, 15, 0);
INSERT INTO data0_t1 VALUES(20, 20, 20, 0);
CREATE TABLE rbu_count(tbl, cnt);
INSERT INTO rbu_count VALUES('data0_t1', 2);
}
{2500 5000 7500 10000}
{1666 3333 5000 6666 8333 10000}
2 {
CREATE TABLE data0_t1(a, b, c, rbu_control);
INSERT INTO data0_t1 VALUES(10, 10, 10, 2);
CREATE TABLE rbu_count(tbl, cnt);
INSERT INTO rbu_count VALUES('data0_t1', 1);
}
{3333 6666 10000}
{2000 4000 6000 8000 10000}
3 {
CREATE TABLE data0_t1(a, b, c, rbu_control);
INSERT INTO data0_t1 VALUES(7, 7, 7, 2);
INSERT INTO data0_t1 VALUES(10, 10, 10, 2);
CREATE TABLE rbu_count(tbl, cnt);
INSERT INTO rbu_count VALUES('data0_t1', 2);
}
{2500 4000 6000 8000 10000}
{1666 2500 3750 5000 6250 7500 8750 10000}
} {
reset_db ; execsql $tbl
do_test 4.$tn.$bReopen.$tn2.0 {
execsql {
CREATE INDEX t1c ON t1(c);
INSERT INTO t1 VALUES(1, 1, 1);
INSERT INTO t1 VALUES(5, 5, 5);
INSERT INTO t1 VALUES(10, 10, 10);
}
create_db_file rbu.db $rbusql
} {}
set R(ipk) $r1
set R(wr) $r1
set R(pk) $r3
do_sp_test 4.$tn.$bReopen.$tn2.1 $bReopen test.db rbu.db $R($tn)
}
}
}
foreach {bReopen} { 0 1 } {
foreach {tn tbl} {
nopk {
CREATE TABLE t1(a, b, c);
CREATE INDEX t1c ON t1(c);
}
vtab {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, c);
}
} {
if {$tn=="vtab"} { ifcapable !fts5 break }
foreach {tn2 rbusql r1 r2} {
1 {
CREATE TABLE data0_t1(a, b, c, rbu_rowid, rbu_control);
INSERT INTO data0_t1 VALUES(15, 15, 15, 4, 0);
INSERT INTO data0_t1 VALUES(20, 20, 20, 5, 0);
CREATE TABLE rbu_count(tbl, cnt);
INSERT INTO rbu_count VALUES('data0_t1', 2);
}
{2500 5000 7500 10000}
{5000 10000}
2 {
CREATE TABLE data0_t1(rbu_rowid, a, b, c, rbu_control);
INSERT INTO data0_t1 VALUES(0, 7, 7, 7, 2);
INSERT INTO data0_t1 VALUES(2, 10, 10, 10, 2);
CREATE TABLE rbu_count(tbl, cnt);
INSERT INTO rbu_count VALUES('data0_t1', 2);
}
{2500 4000 6000 8000 10000}
{5000 10000}
3 {
CREATE TABLE data0_t1(rbu_rowid, a, b, c, rbu_control);
INSERT INTO data0_t1 VALUES(1, NULL, NULL, NULL, 1);
INSERT INTO data0_t1 VALUES(2, NULL, NULL, 7, '..x');
CREATE TABLE rbu_count(tbl, cnt);
INSERT INTO rbu_count VALUES('data0_t1', 2);
}
{2500 4000 6000 8000 10000}
{5000 10000}
} {
reset_db ; execsql $tbl
do_test 5.$tn.$bReopen.$tn2.0 {
execsql {
INSERT INTO t1 VALUES(1, 1, 1);
INSERT INTO t1 VALUES(5, 5, 5);
INSERT INTO t1 VALUES(10, 10, 10);
}
create_db_file rbu.db $rbusql
} {}
set R(nopk) $r1
set R(vtab) $r2
do_sp_test 5.$tn.$bReopen.$tn2.1 $bReopen test.db rbu.db $R($tn)
}
}
}
finish_test

254
ext/rbu/rburesume.test Normal file
View File

@@ -0,0 +1,254 @@
# 2017 January 13
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file contains tests for resumption of RBU operations in the
# case where the previous RBU process crashed.
#
source [file join [file dirname [info script]] rbu_common.tcl]
set ::testprefix rburesume
forcedelete test.db-shm test.db-oal
do_execsql_test 1.0 {
CREATE TABLE t1(a PRIMARY KEY, b, c);
CREATE INDEX t1a ON t1(a);
CREATE INDEX t1b ON t1(b);
CREATE INDEX t1c ON t1(c);
WITH s(i) AS (
VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<50
)
INSERT INTO t1 SELECT randomblob(50), randomblob(75), randomblob(100) FROM s;
}
db_save_and_close
do_test 1.1 {
list [file exists test.db] \
[file exists test.db-wal] \
[file exists test.db-shm] \
[file exists test.db-oal]
} {1 0 0 0}
# Each iteration of the following loop:
#
# 1. Restores the db to the state it was in following test case 1.0
# 2. Opens an RBU vacuum and steps it $n times.
# 3. Closes the RBU vacuum handled opened in (2).
# 4. Opens a second RBU vacuum handle, resumes and completes the vacuum op.
#
# The loop runs until $n is large enough that step (2) vacuums the entire
# database.
#
for {set n 1} {$n < 5000} {incr n} {
db_restore
forcedelete state.db
sqlite3rbu_vacuum rbu test.db state.db
for {set i 0} {$i<$n} {incr i} {
set rc [rbu step]
if {$rc == "SQLITE_DONE"} break
}
rbu close
if {$rc == "SQLITE_DONE"} break
do_test 1.2.$n.1 {
sqlite3rbu_vacuum rbu test.db state.db
while {[rbu step]=="SQLITE_OK"} {}
rbu close
} {SQLITE_DONE}
do_test 1.2.$n.2 {
sqlite3 db2 test.db
db2 eval {
SELECT count(*) FROM t1;
PRAGMA integrity_check;
}
} {50 ok}
db2 close
}
# Each iteration of this loop:
#
# 1. Restores the db to the state it was in following test case 1.0
# 2. Opens an RBU vacuum and steps it $n times.
# 3. Takes a copy of all database files and the state db.
# 4. Opens a second RBU vacuum handle on the copy, resumes and completes the
# vacuum op.
#
# The loop runs until $n is large enough that step (2) vacuums the entire
# database.
#
for {set n 1} {$n < 5000} {incr n} {
db_restore
forcedelete state.db state.db-shm state.db-oal state.db-wal
sqlite3rbu_vacuum rbu test.db state.db
for {set i 0} {$i<$n} {incr i} {
set rc [rbu step]
if {$rc == "SQLITE_DONE"} break
}
if {$rc == "SQLITE_DONE"} {
rbu close
break
}
foreach f {test.db test.db-oal test.db-wal test.db-shm test.db-vacuum} {
set f2 [string map [list test.db test.db2] $f]
if {[file exists $f]} {
forcecopy $f $f2
} else {
forcedelete $f2
}
}
forcecopy state.db state.db2
rbu close
do_test 1.3.$n.1 {
sqlite3rbu_vacuum rbu test.db2 state.db2
while {[rbu step]=="SQLITE_OK"} {}
rbu close
} {SQLITE_DONE}
do_test 1.3.$n.2 {
sqlite3 db2 test.db2
db2 eval {
SELECT count(*) FROM t1;
PRAGMA integrity_check;
}
} {50 ok}
db2 close
}
# Each iteration of this loop:
#
# 1. Restores the db to the state it was in following test case 1.0
# 2. Opens an RBU vacuum and steps it 10 times. Then closes it.
# 2. Opens an RBU vacuum and steps it $n times.
# 3. Takes a copy of all database files and the state db.
# 4. Opens a second RBU vacuum handle on the copy, resumes and completes the
# vacuum op.
#
# The loop runs until $n is large enough that step (3) vacuums the entire
# database.
#
for {set n 1} {$n < 5000} {incr n} {
db_restore
forcedelete state.db state.db-shm state.db-oal state.db-wal
sqlite3rbu_vacuum rbu test.db state.db
for {set i 0} {$i<10} {incr i} {
rbu step
}
rbu close
sqlite3rbu_vacuum rbu test.db state.db
for {set i 0} {$i<$n} {incr i} {
set rc [rbu step]
if {$rc == "SQLITE_DONE"} break
}
if {$rc == "SQLITE_DONE"} {
rbu close
break
}
foreach f {test.db test.db-oal test.db-wal test.db-shm test.db-vacuum} {
set f2 [string map [list test.db test.db2] $f]
if {[file exists $f]} {
forcecopy $f $f2
} else {
forcedelete $f2
}
}
forcecopy state.db state.db2
rbu close
do_test 1.4.$n.1 {
sqlite3rbu_vacuum rbu test.db2 state.db2
while {[rbu step]=="SQLITE_OK"} {}
rbu close
} {SQLITE_DONE}
do_test 1.4.$n.2 {
sqlite3 db2 test.db2
db2 eval {
SELECT count(*) FROM t1;
PRAGMA integrity_check;
}
} {50 ok}
db2 close
}
forcedelete rbu.db
do_test 2.0 {
sqlite3 db2 rbu.db
db2 eval {
CREATE TABLE data_t1(a, b, c, rbu_control);
WITH s(i) AS (
VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<10
)
INSERT INTO data_t1
SELECT randomblob(50), randomblob(75), randomblob(100), 0 FROM s;
}
db2 close
} {}
# Each iteration of this loop:
#
# 1. Restores the db to the state it was in following test case 1.0
# 2. Opens an RBU handle to apply the RBU update created in test case 2.0.
# 3. Steps the RBU handle $n times.
# 4. Takes a copy of all database files and the state db.
# 5. Opens a second RBU handle on the copy, resumes and completes the
# RBU op. Checks it worked as expected.
#
# The loop runs until $n is large enough that step (3) applies the entire
# update.
#
for {set n 1} {$n < 5000} {incr n} {
db_restore
forcedelete state.db state.db-shm state.db-oal state.db-wal
sqlite3rbu rbu test.db rbu.db state.db
for {set i 0} {$i<$n} {incr i} {
set rc [rbu step]
if {$rc == "SQLITE_DONE"} break
}
if {$rc == "SQLITE_DONE"} {
rbu close
break
}
foreach f {test.db test.db-oal test.db-wal test.db-shm test.db-vacuum} {
set f2 [string map [list test.db test.db2] $f]
if {[file exists $f]} {
forcecopy $f $f2
} else {
forcedelete $f2
}
}
forcecopy state.db state.db2
rbu close
do_test 2.$n.1 {
sqlite3rbu rbu test.db2 rbu.db state.db2
while {[rbu step]=="SQLITE_OK"} {}
rbu close
} {SQLITE_DONE}
do_test 2.$n.2 {
sqlite3 db2 test.db2
db2 eval {
SELECT count(*) FROM t1;
PRAGMA integrity_check;
}
} {60 ok}
db2 close
}
finish_test

392
ext/rbu/rbuvacuum.test Normal file
View File

@@ -0,0 +1,392 @@
# 2016 April 15
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file contains tests for the RBU module. More specifically, it
# contains tests to ensure that the sqlite3rbu_vacuum() API works as
# expected.
#
source [file join [file dirname [info script]] rbu_common.tcl]
set ::testprefix rbuvacuum
foreach step {0 1} {
set ::testprefix rbuvacuum-step=$step
reset_db
# Simplest possible vacuum.
do_execsql_test 1.0 {
PRAGMA page_size = 1024;
CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
INSERT INTO t1 VALUES(1, 2, 3);
INSERT INTO t1 VALUES(4, 5, 6);
INSERT INTO t1 VALUES(7, 8, 9);
PRAGMA integrity_check;
} {ok}
do_rbu_vacuum_test 1.1 $step
# A vacuum that actually reclaims space.
do_execsql_test 1.2.1 {
INSERT INTO t1 VALUES(8, randomblob(900), randomblob(900));
INSERT INTO t1 VALUES(9, randomblob(900), randomblob(900));
INSERT INTO t1 VALUES(10, randomblob(900), randomblob(900));
INSERT INTO t1 VALUES(11, randomblob(900), randomblob(900));
INSERT INTO t1 VALUES(12, randomblob(900), randomblob(900));
PRAGMA page_count;
} {12}
do_execsql_test 1.2.2 {
DELETE FROM t1 WHERE rowid BETWEEN 8 AND 11;
PRAGMA page_count;
} {12}
do_rbu_vacuum_test 1.2.3 $step
do_execsql_test 1.2.4 {
PRAGMA page_count;
} {3}
# Add an index to the table.
do_execsql_test 1.3.1 {
CREATE INDEX t1b ON t1(b);
INSERT INTO t1 VALUES(13, randomblob(900), randomblob(900));
INSERT INTO t1 VALUES(14, randomblob(900), randomblob(900));
INSERT INTO t1 VALUES(15, randomblob(900), randomblob(900));
INSERT INTO t1 VALUES(16, randomblob(900), randomblob(900));
PRAGMA page_count;
} {18}
do_execsql_test 1.3.2 {
DELETE FROM t1 WHERE rowid BETWEEN 12 AND 15;
PRAGMA page_count;
} {18}
do_rbu_vacuum_test 1.3.3 $step
do_execsql_test 1.3.4 {
PRAGMA page_count;
} {5}
# WITHOUT ROWID table.
do_execsql_test 1.4.1 {
CREATE TABLE t2(a, b, c, PRIMARY KEY(a, b)) WITHOUT ROWID;
INSERT INTO t2 VALUES(randomblob(900), 1, randomblob(900));
INSERT INTO t2 VALUES(randomblob(900), 2, randomblob(900));
INSERT INTO t2 VALUES(randomblob(900), 3, randomblob(900));
INSERT INTO t2 VALUES(randomblob(900), 4, randomblob(900));
INSERT INTO t2 VALUES(randomblob(900), 6, randomblob(900));
INSERT INTO t2 VALUES(randomblob(900), 7, randomblob(900));
INSERT INTO t2 VALUES(randomblob(900), 8, randomblob(900));
DELETE FROM t2 WHERE b BETWEEN 2 AND 7;
PRAGMA page_count;
} {20}
do_rbu_vacuum_test 1.4.2 $step
do_execsql_test 1.4.3 {
PRAGMA page_count;
} {10}
# WITHOUT ROWID table with an index.
do_execsql_test 1.4.1 {
CREATE INDEX t2c ON t2(c);
INSERT INTO t2 VALUES(randomblob(900), 9, randomblob(900));
INSERT INTO t2 VALUES(randomblob(900), 10, randomblob(900));
INSERT INTO t2 VALUES(randomblob(900), 11, randomblob(900));
INSERT INTO t2 VALUES(randomblob(900), 12, randomblob(900));
INSERT INTO t2 VALUES(randomblob(900), 13, randomblob(900));
DELETE FROM t2 WHERE b BETWEEN 8 AND 12;
PRAGMA page_count;
} {35}
do_rbu_vacuum_test 1.4.2 $step
do_execsql_test 1.4.3 {
PRAGMA page_count;
} {15}
do_execsql_test 1.4.4 {
VACUUM;
PRAGMA page_count;
} {15}
do_execsql_test 1.5.1 {
CREATE TABLE t3(a, b, c);
INSERT INTO t3 VALUES('a', 'b', 'c');
INSERT INTO t3 VALUES('d', 'e', 'f');
INSERT INTO t3 VALUES('g', 'h', 'i');
}
do_rbu_vacuum_test 1.5.2 $step
do_execsql_test 1.5.3 {
SELECT * FROM t3
} {a b c d e f g h i}
do_execsql_test 1.5.4 {
CREATE INDEX t3a ON t3(a);
CREATE INDEX t3b ON t3(b);
CREATE INDEX t3c ON t3(c);
INSERT INTO t3 VALUES('j', 'k', 'l');
DELETE FROM t3 WHERE a = 'g';
}
do_rbu_vacuum_test 1.5.5 $step
do_execsql_test 1.5.6 {
SELECT rowid, * FROM t3 ORDER BY b
} {1 a b c 2 d e f 4 j k l}
do_execsql_test 1.6.1 {
CREATE TABLE t4(a PRIMARY KEY, b, c);
INSERT INTO t4 VALUES('a', 'b', 'c');
INSERT INTO t4 VALUES('d', 'e', 'f');
INSERT INTO t4 VALUES('g', 'h', 'i');
}
do_rbu_vacuum_test 1.6.2 $step
do_execsql_test 1.6.3 {
SELECT * FROM t4
} {a b c d e f g h i}
do_execsql_test 1.6.4 {
CREATE INDEX t4a ON t4(a);
CREATE INDEX t4b ON t4(b);
CREATE INDEX t4c ON t4(c);
INSERT INTO t4 VALUES('j', 'k', 'l');
DELETE FROM t4 WHERE a='g';
}
do_rbu_vacuum_test 1.6.5 $step
do_execsql_test 1.6.6 {
SELECT * FROM t4 ORDER BY b
} {a b c d e f j k l}
reset_db
do_execsql_test 1.7.0 {
CREATE TABLE t1(a INTEGER PRIMARY KEY AUTOINCREMENT, b);
INSERT INTO t1 VALUES(NULL, 'one');
INSERT INTO t1 VALUES(NULL, 'two');
DELETE FROM t1 WHERE a=2;
INSERT INTO t1 VALUES(NULL, 'three');
INSERT INTO t1 VALUES(NULL, 'four');
DELETE FROM t1 WHERE a=4;
INSERT INTO t1 VALUES(NULL, 'five');
INSERT INTO t1 VALUES(NULL, 'six');
DELETE FROM t1 WHERE a=6;
SELECT * FROM t1;
} {1 one 3 three 5 five}
do_rbu_vacuum_test 1.7.1 $step
do_execsql_test 1.7.2 {
INSERT INTO t1 VALUES(NULL, 'seven');
SELECT * FROM t1;
} {1 one 3 three 5 five 7 seven}
reset_db
do_execsql_test 1.8.0 {
CREATE TABLE t1(a INTEGER PRIMARY KEY AUTOINCREMENT, b);
CREATE INDEX i1 ON t1(b);
INSERT INTO t1 VALUES(NULL, 'one');
INSERT INTO t1 VALUES(NULL, 'two');
INSERT INTO t1 VALUES(NULL, 'three');
INSERT INTO t1 VALUES(NULL, 'four');
INSERT INTO t1 VALUES(NULL, 'five');
INSERT INTO t1 VALUES(NULL, 'six');
ANALYZE;
SELECT * FROM sqlite_stat1;
} {t1 i1 {6 1}}
do_rbu_vacuum_test 1.8.1 $step
do_execsql_test 1.7.2 {
SELECT * FROM sqlite_stat1;
} {t1 i1 {6 1}}
reset_db
do_execsql_test 1.9.0 {
PRAGMA page_size = 8192;
PRAGMA auto_vacuum = 2;
PRAGMA user_version = 412;
PRAGMA application_id = 413;
CREATE TABLE t1(a INTEGER PRIMARY KEY AUTOINCREMENT, b);
CREATE INDEX i1 ON t1(b);
INSERT INTO t1 VALUES(NULL, 'one');
INSERT INTO t1 VALUES(NULL, 'two');
INSERT INTO t1 VALUES(NULL, 'three');
INSERT INTO t1 VALUES(NULL, 'four');
INSERT INTO t1 VALUES(NULL, 'five');
INSERT INTO t1 VALUES(NULL, 'six');
PRAGMA main.page_size;
PRAGMA main.auto_vacuum;
PRAGMA main.user_version;
PRAGMA main.application_id;
} {8192 2 412 413}
do_rbu_vacuum_test 1.9.1 $step
do_execsql_test 1.9.2 {
PRAGMA main.page_size;
PRAGMA main.auto_vacuum;
PRAGMA main.user_version;
PRAGMA main.application_id;
} {8192 2 412 413}
# Vacuum a database with a large sqlite_master table.
#
reset_db
do_test 1.10.1 {
for {set i 1} {$i < 50} {incr i} {
execsql "PRAGMA page_size = 1024"
execsql "CREATE TABLE t$i (a, b, c, PRIMARY KEY(a, b));"
execsql "
INSERT INTO t$i VALUES(1, 2, 3);
INSERT INTO t$i VALUES(4, 5, 6);
"
}
} {}
do_rbu_vacuum_test 1.10.2 $step
# Database with empty tables.
#
reset_db
do_execsql_test 1.11.1 {
CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
CREATE TABLE t2(a INTEGER PRIMARY KEY, b);
CREATE TABLE t3(a INTEGER PRIMARY KEY, b);
CREATE TABLE t4(a INTEGER PRIMARY KEY, b);
INSERT INTO t4 VALUES(1, 2);
}
do_rbu_vacuum_test 1.11.2 $step
do_execsql_test 1.11.3 {
SELECT * FROM t1;
SELECT * FROM t2;
SELECT * FROM t3;
SELECT * FROM t4;
} {1 2}
reset_db
do_execsql_test 1.12.1 {
CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
CREATE TABLE t2(a INTEGER PRIMARY KEY, b);
CREATE TABLE t3(a INTEGER PRIMARY KEY, b);
CREATE TABLE t4(a INTEGER PRIMARY KEY, b);
INSERT INTO t1 VALUES(1, 2);
}
do_rbu_vacuum_test 1.12.2 $step
do_execsql_test 1.12.3 {
SELECT * FROM t1;
SELECT * FROM t2;
SELECT * FROM t3;
SELECT * FROM t4;
} {1 2}
}
set ::testprefix rbuvacuum
#-------------------------------------------------------------------------
# Test some error cases:
#
# 2.1.* the db being vacuumed being in wal mode already.
# 2.2.* database modified mid vacuum.
#
reset_db
do_execsql_test 2.1.0 {
CREATE TABLE t1(a, b);
INSERT INTO t1 VALUES(1, 2);
INSERT INTO t1 VALUES(3, 4);
INSERT INTO t1 VALUES(5, 6);
INSERT INTO t1 VALUES(7, 8);
PRAGMA journal_mode = wal;
INSERT INTO t1 VALUES(9, 10);
} wal
do_test 2.1.1 {
sqlite3rbu_vacuum rbu test.db state.db
rbu step
} {SQLITE_ERROR}
do_test 2.1.2 {
list [catch { rbu close } msg] $msg
} {1 {SQLITE_ERROR - cannot vacuum wal mode database}}
reset_db
do_execsql_test 2.2.0 {
CREATE TABLE tx(a PRIMARY KEY, b BLOB);
INSERT INTO tx VALUES(1, randomblob(900));
INSERT INTO tx SELECT a+1, randomblob(900) FROM tx;
INSERT INTO tx SELECT a+2, randomblob(900) FROM tx;
INSERT INTO tx SELECT a+4, randomblob(900) FROM tx;
INSERT INTO tx SELECT a+8, randomblob(900) FROM tx;
}
db_save_and_close
for {set i 1} 1 {incr i} {
db_restore_and_reopen
sqlite3rbu_vacuum rbu test.db state.db
for {set step 0} {$step<$i} {incr step} { rbu step }
rbu close
if {[file exists test.db-wal]} break
execsql { INSERT INTO tx VALUES(20, 20) }
do_test 2.2.$i.1 {
sqlite3rbu_vacuum rbu test.db state.db
rbu step
} {SQLITE_BUSY}
do_test 2.2.$i.2 {
list [catch { rbu close } msg] $msg
} {1 {SQLITE_BUSY - database modified during rbu vacuum}}
}
#-------------------------------------------------------------------------
# Test that a database that uses custom collation sequences can be RBU
# vacuumed.
#
reset_db
forcedelete state.db
proc noop {args} {}
proc length_cmp {x y} {
set n1 [string length $x]
set n2 [string length $y]
return [expr $n1 - $n2]
}
sqlite3_create_collation_v2 db length length_cmp noop
do_execsql_test 3.0 {
CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
INSERT INTO t1 VALUES(1, 'i');
INSERT INTO t1 VALUES(2, 'iiii');
INSERT INTO t1 VALUES(3, 'ii');
INSERT INTO t1 VALUES(4, 'iii');
SELECT a FROM t1 ORDER BY b COLLATE length;
} {1 3 4 2}
do_execsql_test 3.1 {
CREATE INDEX i1 ON t1(b COLLATE length);
}
do_test 3.2 {
sqlite3rbu_vacuum rbu test.db state.db
while {[rbu step]=="SQLITE_OK"} {}
list [catch { rbu close } msg] $msg
} {1 {SQLITE_ERROR - no such collation sequence: length}}
do_test 3.3 {
sqlite3rbu_vacuum rbu test.db state.db
set db1 [rbu db 0]
sqlite3_create_collation_v2 $db1 length length_cmp noop
while {[rbu step]=="SQLITE_OK"} {}
list [catch { rbu close } msg] $msg
} {1 {SQLITE_ERROR - no such collation sequence: length}}
do_test 3.4 {
sqlite3rbu_vacuum rbu test.db state.db
set db1 [rbu db 1]
sqlite3_create_collation_v2 $db1 length length_cmp noop
while {[rbu step]=="SQLITE_OK"} {}
list [catch { rbu close } msg] $msg
} {1 {SQLITE_ERROR - no such collation sequence: length}}
do_test 3.5 {
sqlite3rbu_vacuum rbu test.db state.db
set db1 [rbu db 0]
set db2 [rbu db 1]
sqlite3_create_collation_v2 $db1 length length_cmp noop
sqlite3_create_collation_v2 $db2 length length_cmp noop
while {[rbu step]=="SQLITE_OK"} {}
list [catch { rbu close } msg] $msg
} {0 SQLITE_DONE}
catch { db close }
finish_test

235
ext/rbu/rbuvacuum2.test Normal file
View File

@@ -0,0 +1,235 @@
# 2016 June 1
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file contains tests for the RBU module. More specifically, it
# contains tests to ensure that the sqlite3rbu_vacuum() API works as
# expected.
#
source [file join [file dirname [info script]] rbu_common.tcl]
foreach step {0 1} {
set ::testprefix rbuvacuum2-$step
#-------------------------------------------------------------------------
# Test that a database that contains fts3 tables can be vacuumed.
#
ifcapable fts3 {
reset_db
do_execsql_test 1.1 {
CREATE VIRTUAL TABLE t1 USING fts3(z, y);
INSERT INTO t1 VALUES('fix this issue', 'at some point');
}
do_rbu_vacuum_test 1.2 $step
do_execsql_test 1.3 {
SELECT * FROM t1;
} {{fix this issue} {at some point}}
do_execsql_test 1.4 {
SELECT rowid FROM t1 WHERE t1 MATCH 'fix';
} {1}
do_execsql_test 1.5 {
INSERT INTO t1 VALUES('a b c', 'd e f');
INSERT INTO t1 VALUES('l h i', 'd e f');
DELETE FROM t1 WHERE docid = 2;
INSERT INTO t1 VALUES('a b c', 'x y z');
}
do_rbu_vacuum_test 1.6 $step
do_execsql_test 1.7 {
INSERT INTO t1(t1) VALUES('integrity-check');
SELECT * FROM t1;
} {
{fix this issue} {at some point}
{l h i} {d e f}
{a b c} {x y z}
}
}
#-------------------------------------------------------------------------
# Test that a database that contains fts5 tables can be vacuumed.
#
ifcapable fts5 {
reset_db
do_execsql_test 2.1 {
CREATE VIRTUAL TABLE t1 USING fts5(z, y);
INSERT INTO t1 VALUES('fix this issue', 'at some point');
}
do_rbu_vacuum_test 2.2 $step
do_execsql_test 2.3 {
SELECT * FROM t1;
} {{fix this issue} {at some point}}
do_execsql_test 2.4 {
SELECT rowid FROM t1 ('fix');
} {1}
do_execsql_test 2.5 {
INSERT INTO t1 VALUES('a b c', 'd e f');
INSERT INTO t1 VALUES('l h i', 'd e f');
DELETE FROM t1 WHERE rowid = 2;
INSERT INTO t1 VALUES('a b c', 'x y z');
}
do_rbu_vacuum_test 2.6 $step
do_execsql_test 2.7 {
INSERT INTO t1(t1) VALUES('integrity-check');
SELECT * FROM t1;
} {
{fix this issue} {at some point}
{l h i} {d e f}
{a b c} {x y z}
}
}
#-------------------------------------------------------------------------
# Test that a database that contains an rtree table can be vacuumed.
#
ifcapable rtree {
reset_db
do_execsql_test 3.1 {
CREATE VIRTUAL TABLE rt USING rtree(id, x1, x2);
INSERT INTO rt VALUES(1, 45, 55);
INSERT INTO rt VALUES(2, 50, 60);
INSERT INTO rt VALUES(3, 55, 65);
}
do_rbu_vacuum_test 3.2 $step
do_execsql_test 3.3 {
SELECT * FROM rt;
} {1 45.0 55.0 2 50.0 60.0 3 55.0 65.0}
do_execsql_test 3.4.1 {
SELECT rowid FROM rt WHERE x2>51 AND x1 < 51
} {1 2}
do_execsql_test 3.4.2 {
SELECT rowid FROM rt WHERE x2>59 AND x1 < 59
} {2 3}
do_rbu_vacuum_test 3.5 $step
do_execsql_test 3.6.1 {
SELECT rowid FROM rt WHERE x2>51 AND x1 < 51
} {1 2}
do_execsql_test 3.6.2 {
SELECT rowid FROM rt WHERE x2>59 AND x1 < 59
} {2 3}
}
ifcapable trigger {
reset_db
do_execsql_test 4.1 {
CREATE TABLE t1(a, b, c);
INSERT INTO t1 VALUES(1, 2, 3);
CREATE VIEW v1 AS SELECT * FROM t1;
CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN SELECT 1; END;
}
do_execsql_test 4.2 {
SELECT * FROM sqlite_master;
} {
table t1 t1 2 {CREATE TABLE t1(a, b, c)}
view v1 v1 0 {CREATE VIEW v1 AS SELECT * FROM t1}
trigger tr1 t1 0 {CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN SELECT 1; END}
}
do_rbu_vacuum_test 4.3 $step
do_execsql_test 4.4 {
SELECT * FROM sqlite_master;
} {
table t1 t1 2 {CREATE TABLE t1(a, b, c)}
view v1 v1 0 {CREATE VIEW v1 AS SELECT * FROM t1}
trigger tr1 t1 0 {CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN SELECT 1; END}
}
}
}
#-------------------------------------------------------------------------
# Test that passing a NULL value as the second argument to
# sqlite3rbu_vacuum() causes it to:
#
# * Use <database>-vacuum as the state db, and
# * Set the state db permissions to the same as those on the db file.
#
db close
if {$::tcl_platform(platform)=="unix"} {
forcedelete test.db
sqlite3 db test.db
do_execsql_test 5.0 {
CREATE TABLE t1(a, b);
INSERT INTO t1 VALUES(1, 2);
INSERT INTO t1 VALUES(3, 4);
INSERT INTO t1 VALUES(5, 6);
INSERT INTO t1 VALUES(7, 8);
}
db close
foreach {tn perm} {
1 00755
2 00666
3 00644
4 00444
} {
forcedelete test.db-vacuum
do_test 5.$tn.1 {
file attributes test.db -permissions $perm
sqlite3rbu_vacuum rbu test.db
rbu step
} {SQLITE_OK}
do_test 5.$tn.2 { file exists test.db-vacuum } 1
do_test 5.$tn.3 { file attributes test.db-vacuum -permissions} $perm
rbu close
}
}
#-------------------------------------------------------------------------
# Test the outcome of some other connection running a checkpoint while
# the incremental checkpoint is suspended.
#
reset_db
do_execsql_test 6.0 {
CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
CREATE INDEX i1b ON t1(b);
CREATE INDEX i1c ON t1(c);
INSERT INTO t1 VALUES(1, 2, 3);
INSERT INTO t1 VALUES(4, 5, 6);
}
forcedelete test.db2
do_test 6.1 {
sqlite3rbu_vacuum rbu test.db test.db2
while {[rbu state]!="checkpoint"} { rbu step }
rbu close
} {SQLITE_OK}
do_execsql_test 6.2 {
SELECT 1 FROM sqlite_master LIMIT 1;
PRAGMA wal_checkpoint;
} {1 0 4 4}
do_test 6.3 {
sqlite3rbu_vacuum rbu test.db test.db2
while {[rbu step]!="SQLITE_DONE"} { rbu step }
rbu close
execsql { PRAGMA integrity_check }
} {ok}
finish_test

1175
ext/rbu/sqlite3rbu.c
View File

File diff suppressed because it is too large Load Diff

120
ext/rbu/sqlite3rbu.h
View File

@@ -104,7 +104,7 @@
** may also be named data<integer>_<target>, where <integer> is any sequence
** of zero or more numeric characters (0-9). This can be significant because
** tables within the RBU database are always processed in order sorted by
** name. By judicious selection of the the <integer> portion of the names
** name. By judicious selection of the <integer> portion of the names
** of the RBU tables the user can therefore control the order in which they
** are processed. This can be useful, for example, to ensure that "external
** content" FTS4 tables are updated before their underlying content tables.
@@ -314,6 +314,44 @@ sqlite3rbu *sqlite3rbu_open(
const char *zState
);
/*
** Open an RBU handle to perform an RBU vacuum on database file zTarget.
** An RBU vacuum is similar to SQLite's built-in VACUUM command, except
** that it can be suspended and resumed like an RBU update.
**
** The second argument to this function identifies a database in which
** to store the state of the RBU vacuum operation if it is suspended. The
** first time sqlite3rbu_vacuum() is called, to start an RBU vacuum
** operation, the state database should either not exist or be empty
** (contain no tables). If an RBU vacuum is suspended by calling
** sqlite3rbu_close() on the RBU handle before sqlite3rbu_step() has
** returned SQLITE_DONE, the vacuum state is stored in the state database.
** The vacuum can be resumed by calling this function to open a new RBU
** handle specifying the same target and state databases.
**
** If the second argument passed to this function is NULL, then the
** name of the state database is "<database>-vacuum", where <database>
** is the name of the target database file. In this case, on UNIX, if the
** state database is not already present in the file-system, it is created
** with the same permissions as the target db is made.
**
** This function does not delete the state database after an RBU vacuum
** is completed, even if it created it. However, if the call to
** sqlite3rbu_close() returns any value other than SQLITE_OK, the contents
** of the state tables within the state database are zeroed. This way,
** the next call to sqlite3rbu_vacuum() opens a handle that starts a
** new RBU vacuum operation.
**
** As with sqlite3rbu_open(), Zipvfs users should rever to the comment
** describing the sqlite3rbu_create_vfs() API function below for
** a description of the complications associated with using RBU with
** zipvfs databases.
*/
sqlite3rbu *sqlite3rbu_vacuum(
const char *zTarget,
const char *zState
);
/*
** Internally, each RBU connection uses a separate SQLite database
** connection to access the target and rbu update databases. This
@@ -400,6 +438,86 @@ int sqlite3rbu_close(sqlite3rbu *pRbu, char **pzErrmsg);
*/
sqlite3_int64 sqlite3rbu_progress(sqlite3rbu *pRbu);
/*
** Obtain permyriadage (permyriadage is to 10000 as percentage is to 100)
** progress indications for the two stages of an RBU update. This API may
** be useful for driving GUI progress indicators and similar.
**
** An RBU update is divided into two stages:
**
** * Stage 1, in which changes are accumulated in an oal/wal file, and
** * Stage 2, in which the contents of the wal file are copied into the
** main database.
**
** The update is visible to non-RBU clients during stage 2. During stage 1
** non-RBU reader clients may see the original database.
**
** If this API is called during stage 2 of the update, output variable
** (*pnOne) is set to 10000 to indicate that stage 1 has finished and (*pnTwo)
** to a value between 0 and 10000 to indicate the permyriadage progress of
** stage 2. A value of 5000 indicates that stage 2 is half finished,
** 9000 indicates that it is 90% finished, and so on.
**
** If this API is called during stage 1 of the update, output variable
** (*pnTwo) is set to 0 to indicate that stage 2 has not yet started. The
** value to which (*pnOne) is set depends on whether or not the RBU
** database contains an "rbu_count" table. The rbu_count table, if it
** exists, must contain the same columns as the following:
**
** CREATE TABLE rbu_count(tbl TEXT PRIMARY KEY, cnt INTEGER) WITHOUT ROWID;
**
** There must be one row in the table for each source (data_xxx) table within
** the RBU database. The 'tbl' column should contain the name of the source
** table. The 'cnt' column should contain the number of rows within the
** source table.
**
** If the rbu_count table is present and populated correctly and this
** API is called during stage 1, the *pnOne output variable is set to the
** permyriadage progress of the same stage. If the rbu_count table does
** not exist, then (*pnOne) is set to -1 during stage 1. If the rbu_count
** table exists but is not correctly populated, the value of the *pnOne
** output variable during stage 1 is undefined.
*/
void sqlite3rbu_bp_progress(sqlite3rbu *pRbu, int *pnOne, int *pnTwo);
/*
** Obtain an indication as to the current stage of an RBU update or vacuum.
** This function always returns one of the SQLITE_RBU_STATE_XXX constants
** defined in this file. Return values should be interpreted as follows:
**
** SQLITE_RBU_STATE_OAL:
** RBU is currently building a *-oal file. The next call to sqlite3rbu_step()
** may either add further data to the *-oal file, or compute data that will
** be added by a subsequent call.
**
** SQLITE_RBU_STATE_MOVE:
** RBU has finished building the *-oal file. The next call to sqlite3rbu_step()
** will move the *-oal file to the equivalent *-wal path. If the current
** operation is an RBU update, then the updated version of the database
** file will become visible to ordinary SQLite clients following the next
** call to sqlite3rbu_step().
**
** SQLITE_RBU_STATE_CHECKPOINT:
** RBU is currently performing an incremental checkpoint. The next call to
** sqlite3rbu_step() will copy a page of data from the *-wal file into
** the target database file.
**
** SQLITE_RBU_STATE_DONE:
** The RBU operation has finished. Any subsequent calls to sqlite3rbu_step()
** will immediately return SQLITE_DONE.
**
** SQLITE_RBU_STATE_ERROR:
** An error has occurred. Any subsequent calls to sqlite3rbu_step() will
** immediately return the SQLite error code associated with the error.
*/
#define SQLITE_RBU_STATE_OAL 1
#define SQLITE_RBU_STATE_MOVE 2
#define SQLITE_RBU_STATE_CHECKPOINT 3
#define SQLITE_RBU_STATE_DONE 4
#define SQLITE_RBU_STATE_ERROR 5
int sqlite3rbu_state(sqlite3rbu *pRbu);
/*
** Create an RBU VFS named zName that accesses the underlying file-system
** via existing VFS zParent. Or, if the zParent parameter is passed NULL,

101
ext/rbu/test_rbu.c
View File

@@ -17,11 +17,19 @@
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RBU)
#include "sqlite3rbu.h"
#include <tcl.h>
#if defined(INCLUDE_SQLITE_TCL_H)
# include "sqlite_tcl.h"
#else
# include "tcl.h"
# ifndef SQLITE_TCLAPI
# define SQLITE_TCLAPI
# endif
#endif
#include <assert.h>
/* From main.c (apparently...) */
/* From main.c */
extern const char *sqlite3ErrName(int);
extern int sqlite3TestMakePointerStr(Tcl_Interp*, char*, void*);
void test_rbu_delta(sqlite3_context *pCtx, int nArg, sqlite3_value **apVal){
Tcl_Interp *interp = (Tcl_Interp*)sqlite3_user_data(pCtx);
@@ -48,7 +56,7 @@ void test_rbu_delta(sqlite3_context *pCtx, int nArg, sqlite3_value **apVal){
}
static int test_sqlite3rbu_cmd(
static int SQLITE_TCLAPI test_sqlite3rbu_cmd(
ClientData clientData,
Tcl_Interp *interp,
int objc,
@@ -66,6 +74,10 @@ static int test_sqlite3rbu_cmd(
{"create_rbu_delta", 2, ""}, /* 2 */
{"savestate", 2, ""}, /* 3 */
{"dbMain_eval", 3, "SQL"}, /* 4 */
{"bp_progress", 2, ""}, /* 5 */
{"db", 3, "RBU"}, /* 6 */
{"state", 2, ""}, /* 7 */
{"progress", 2, ""}, /* 8 */
{0,0,0}
};
int iCmd;
@@ -136,6 +148,46 @@ static int test_sqlite3rbu_cmd(
break;
}
case 5: /* bp_progress */ {
int one, two;
Tcl_Obj *pObj;
sqlite3rbu_bp_progress(pRbu, &one, &two);
pObj = Tcl_NewObj();
Tcl_ListObjAppendElement(interp, pObj, Tcl_NewIntObj(one));
Tcl_ListObjAppendElement(interp, pObj, Tcl_NewIntObj(two));
Tcl_SetObjResult(interp, pObj);
break;
}
case 6: /* db */ {
int bArg;
if( Tcl_GetBooleanFromObj(interp, objv[2], &bArg) ){
ret = TCL_ERROR;
}else{
char zBuf[50];
sqlite3 *db = sqlite3rbu_db(pRbu, bArg);
if( sqlite3TestMakePointerStr(interp, zBuf, (void*)db) ){
ret = TCL_ERROR;
}else{
Tcl_SetResult(interp, zBuf, TCL_VOLATILE);
}
}
break;
}
case 7: /* state */ {
const char *aRes[] = { 0, "oal", "move", "checkpoint", "done", "error" };
int eState = sqlite3rbu_state(pRbu);
assert( eState>0 && eState<=5 );
Tcl_SetResult(interp, (char*)aRes[eState], TCL_STATIC);
break;
}
case 8: /* progress */ {
sqlite3_int64 nStep = sqlite3rbu_progress(pRbu);
Tcl_SetObjResult(interp, Tcl_NewWideIntObj(nStep));
break;
}
default: /* seems unlikely */
assert( !"cannot happen" );
break;
@@ -147,7 +199,7 @@ static int test_sqlite3rbu_cmd(
/*
** Tclcmd: sqlite3rbu CMD <target-db> <rbu-db> ?<state-db>?
*/
static int test_sqlite3rbu(
static int SQLITE_TCLAPI test_sqlite3rbu(
ClientData clientData,
Tcl_Interp *interp,
int objc,
@@ -174,10 +226,38 @@ static int test_sqlite3rbu(
return TCL_OK;
}
/*
** Tclcmd: sqlite3rbu_vacuum CMD <target-db> <state-db>
*/
static int SQLITE_TCLAPI test_sqlite3rbu_vacuum(
ClientData clientData,
Tcl_Interp *interp,
int objc,
Tcl_Obj *CONST objv[]
){
sqlite3rbu *pRbu = 0;
const char *zCmd;
const char *zTarget;
const char *zStateDb = 0;
if( objc!=3 && objc!=4 ){
Tcl_WrongNumArgs(interp, 1, objv, "NAME TARGET-DB ?STATE-DB?");
return TCL_ERROR;
}
zCmd = Tcl_GetString(objv[1]);
zTarget = Tcl_GetString(objv[2]);
if( objc==4 ) zStateDb = Tcl_GetString(objv[3]);
pRbu = sqlite3rbu_vacuum(zTarget, zStateDb);
Tcl_CreateObjCommand(interp, zCmd, test_sqlite3rbu_cmd, (ClientData)pRbu, 0);
Tcl_SetObjResult(interp, objv[1]);
return TCL_OK;
}
/*
** Tclcmd: sqlite3rbu_create_vfs ?-default? NAME PARENT
*/
static int test_sqlite3rbu_create_vfs(
static int SQLITE_TCLAPI test_sqlite3rbu_create_vfs(
ClientData clientData,
Tcl_Interp *interp,
int objc,
@@ -212,7 +292,7 @@ static int test_sqlite3rbu_create_vfs(
/*
** Tclcmd: sqlite3rbu_destroy_vfs NAME
*/
static int test_sqlite3rbu_destroy_vfs(
static int SQLITE_TCLAPI test_sqlite3rbu_destroy_vfs(
ClientData clientData,
Tcl_Interp *interp,
int objc,
@@ -233,7 +313,7 @@ static int test_sqlite3rbu_destroy_vfs(
/*
** Tclcmd: sqlite3rbu_internal_test
*/
static int test_sqlite3rbu_internal_test(
static int SQLITE_TCLAPI test_sqlite3rbu_internal_test(
ClientData clientData,
Tcl_Interp *interp,
int objc,
@@ -261,6 +341,7 @@ int SqliteRbu_Init(Tcl_Interp *interp){
Tcl_ObjCmdProc *xProc;
} aObjCmd[] = {
{ "sqlite3rbu", test_sqlite3rbu },
{ "sqlite3rbu_vacuum", test_sqlite3rbu_vacuum },
{ "sqlite3rbu_create_vfs", test_sqlite3rbu_create_vfs },
{ "sqlite3rbu_destroy_vfs", test_sqlite3rbu_destroy_vfs },
{ "sqlite3rbu_internal_test", test_sqlite3rbu_internal_test },
@@ -273,7 +354,11 @@ int SqliteRbu_Init(Tcl_Interp *interp){
}
#else
#include <tcl.h>
#if defined(INCLUDE_SQLITE_TCL_H)
# include "sqlite_tcl.h"
#else
# include "tcl.h"
#endif
int SqliteRbu_Init(Tcl_Interp *interp){ return TCL_OK; }
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RBU) */
#endif /* defined(SQLITE_TEST) */

497
ext/rtree/rtree.c
View File

@@ -68,6 +68,7 @@
#ifndef SQLITE_AMALGAMATION
#include "sqlite3rtree.h"
typedef sqlite3_int64 i64;
typedef sqlite3_uint64 u64;
typedef unsigned char u8;
typedef unsigned short u16;
typedef unsigned int u32;
@@ -116,13 +117,16 @@ struct Rtree {
sqlite3 *db; /* Host database connection */
int iNodeSize; /* Size in bytes of each node in the node table */
u8 nDim; /* Number of dimensions */
u8 nDim2; /* Twice the number of dimensions */
u8 eCoordType; /* RTREE_COORD_REAL32 or RTREE_COORD_INT32 */
u8 nBytesPerCell; /* Bytes consumed per cell */
u8 inWrTrans; /* True if inside write transaction */
int iDepth; /* Current depth of the r-tree structure */
char *zDb; /* Name of database containing r-tree table */
char *zName; /* Name of r-tree table */
int nBusy; /* Current number of users of this structure */
u32 nBusy; /* Current number of users of this structure */
i64 nRowEst; /* Estimated number of rows in this table */
u32 nCursor; /* Number of open cursors */
/* List of nodes removed during a CondenseTree operation. List is
** linked together via the pointer normally used for hash chains -
@@ -132,8 +136,10 @@ struct Rtree {
RtreeNode *pDeleted;
int iReinsertHeight; /* Height of sub-trees Reinsert() has run on */
/* Blob I/O on xxx_node */
sqlite3_blob *pNodeBlob;
/* Statements to read/write/delete a record from xxx_node */
sqlite3_stmt *pReadNode;
sqlite3_stmt *pWriteNode;
sqlite3_stmt *pDeleteNode;
@@ -362,6 +368,58 @@ struct RtreeMatchArg {
# define MIN(x,y) ((x) > (y) ? (y) : (x))
#endif
/* What version of GCC is being used. 0 means GCC is not being used .
** Note that the GCC_VERSION macro will also be set correctly when using
** clang, since clang works hard to be gcc compatible. So the gcc
** optimizations will also work when compiling with clang.
*/
#ifndef GCC_VERSION
#if defined(__GNUC__) && !defined(SQLITE_DISABLE_INTRINSIC)
# define GCC_VERSION (__GNUC__*1000000+__GNUC_MINOR__*1000+__GNUC_PATCHLEVEL__)
#else
# define GCC_VERSION 0
#endif
#endif
/* The testcase() macro should already be defined in the amalgamation. If
** it is not, make it a no-op.
*/
#ifndef SQLITE_AMALGAMATION
# define testcase(X)
#endif
/*
** Macros to determine whether the machine is big or little endian,
** and whether or not that determination is run-time or compile-time.
**
** For best performance, an attempt is made to guess at the byte-order
** using C-preprocessor macros. If that is unsuccessful, or if
** -DSQLITE_RUNTIME_BYTEORDER=1 is set, then byte-order is determined
** at run-time.
*/
#ifndef SQLITE_BYTEORDER
#if defined(i386) || defined(__i386__) || defined(_M_IX86) || \
defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \
defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \
defined(__arm__)
# define SQLITE_BYTEORDER 1234
#elif defined(sparc) || defined(__ppc__)
# define SQLITE_BYTEORDER 4321
#else
# define SQLITE_BYTEORDER 0 /* 0 means "unknown at compile-time" */
#endif
#endif
/* What version of MSVC is being used. 0 means MSVC is not being used */
#ifndef MSVC_VERSION
#if defined(_MSC_VER) && !defined(SQLITE_DISABLE_INTRINSIC)
# define MSVC_VERSION _MSC_VER
#else
# define MSVC_VERSION 0
#endif
#endif
/*
** Functions to deserialize a 16 bit integer, 32 bit real number and
** 64 bit integer. The deserialized value is returned.
@@ -370,24 +428,47 @@ static int readInt16(u8 *p){
return (p[0]<<8) + p[1];
}
static void readCoord(u8 *p, RtreeCoord *pCoord){
assert( ((((char*)p) - (char*)0)&3)==0 ); /* p is always 4-byte aligned */
#if SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300
pCoord->u = _byteswap_ulong(*(u32*)p);
#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000
pCoord->u = __builtin_bswap32(*(u32*)p);
#elif SQLITE_BYTEORDER==4321
pCoord->u = *(u32*)p;
#else
pCoord->u = (
(((u32)p[0]) << 24) +
(((u32)p[1]) << 16) +
(((u32)p[2]) << 8) +
(((u32)p[3]) << 0)
);
#endif
}
static i64 readInt64(u8 *p){
return (
(((i64)p[0]) << 56) +
(((i64)p[1]) << 48) +
(((i64)p[2]) << 40) +
(((i64)p[3]) << 32) +
(((i64)p[4]) << 24) +
(((i64)p[5]) << 16) +
(((i64)p[6]) << 8) +
(((i64)p[7]) << 0)
#if SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300
u64 x;
memcpy(&x, p, 8);
return (i64)_byteswap_uint64(x);
#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000
u64 x;
memcpy(&x, p, 8);
return (i64)__builtin_bswap64(x);
#elif SQLITE_BYTEORDER==4321
i64 x;
memcpy(&x, p, 8);
return x;
#else
return (i64)(
(((u64)p[0]) << 56) +
(((u64)p[1]) << 48) +
(((u64)p[2]) << 40) +
(((u64)p[3]) << 32) +
(((u64)p[4]) << 24) +
(((u64)p[5]) << 16) +
(((u64)p[6]) << 8) +
(((u64)p[7]) << 0)
);
#endif
}
/*
@@ -395,23 +476,43 @@ static i64 readInt64(u8 *p){
** 64 bit integer. The value returned is the number of bytes written
** to the argument buffer (always 2, 4 and 8 respectively).
*/
static int writeInt16(u8 *p, int i){
static void writeInt16(u8 *p, int i){
p[0] = (i>> 8)&0xFF;
p[1] = (i>> 0)&0xFF;
return 2;
}
static int writeCoord(u8 *p, RtreeCoord *pCoord){
u32 i;
assert( ((((char*)p) - (char*)0)&3)==0 ); /* p is always 4-byte aligned */
assert( sizeof(RtreeCoord)==4 );
assert( sizeof(u32)==4 );
#if SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000
i = __builtin_bswap32(pCoord->u);
memcpy(p, &i, 4);
#elif SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300
i = _byteswap_ulong(pCoord->u);
memcpy(p, &i, 4);
#elif SQLITE_BYTEORDER==4321
i = pCoord->u;
memcpy(p, &i, 4);
#else
i = pCoord->u;
p[0] = (i>>24)&0xFF;
p[1] = (i>>16)&0xFF;
p[2] = (i>> 8)&0xFF;
p[3] = (i>> 0)&0xFF;
#endif
return 4;
}
static int writeInt64(u8 *p, i64 i){
#if SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000
i = (i64)__builtin_bswap64((u64)i);
memcpy(p, &i, 8);
#elif SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300
i = (i64)_byteswap_uint64((u64)i);
memcpy(p, &i, 8);
#elif SQLITE_BYTEORDER==4321
memcpy(p, &i, 8);
#else
p[0] = (i>>56)&0xFF;
p[1] = (i>>48)&0xFF;
p[2] = (i>>40)&0xFF;
@@ -420,6 +521,7 @@ static int writeInt64(u8 *p, i64 i){
p[5] = (i>>16)&0xFF;
p[6] = (i>> 8)&0xFF;
p[7] = (i>> 0)&0xFF;
#endif
return 8;
}
@@ -502,6 +604,17 @@ static RtreeNode *nodeNew(Rtree *pRtree, RtreeNode *pParent){
return pNode;
}
/*
** Clear the Rtree.pNodeBlob object
*/
static void nodeBlobReset(Rtree *pRtree){
if( pRtree->pNodeBlob && pRtree->inWrTrans==0 && pRtree->nCursor==0 ){
sqlite3_blob *pBlob = pRtree->pNodeBlob;
pRtree->pNodeBlob = 0;
sqlite3_blob_close(pBlob);
}
}
/*
** Obtain a reference to an r-tree node.
*/
@@ -511,9 +624,8 @@ static int nodeAcquire(
RtreeNode *pParent, /* Either the parent node or NULL */
RtreeNode **ppNode /* OUT: Acquired node */
){
int rc;
int rc2 = SQLITE_OK;
RtreeNode *pNode;
int rc = SQLITE_OK;
RtreeNode *pNode = 0;
/* Check if the requested node is already in the hash table. If so,
** increase its reference count and return it.
@@ -529,28 +641,45 @@ static int nodeAcquire(
return SQLITE_OK;
}
sqlite3_bind_int64(pRtree->pReadNode, 1, iNode);
rc = sqlite3_step(pRtree->pReadNode);
if( rc==SQLITE_ROW ){
const u8 *zBlob = sqlite3_column_blob(pRtree->pReadNode, 0);
if( pRtree->iNodeSize==sqlite3_column_bytes(pRtree->pReadNode, 0) ){
pNode = (RtreeNode *)sqlite3_malloc(sizeof(RtreeNode)+pRtree->iNodeSize);
if( !pNode ){
rc2 = SQLITE_NOMEM;
}else{
pNode->pParent = pParent;
pNode->zData = (u8 *)&pNode[1];
pNode->nRef = 1;
pNode->iNode = iNode;
pNode->isDirty = 0;
pNode->pNext = 0;
memcpy(pNode->zData, zBlob, pRtree->iNodeSize);
nodeReference(pParent);
}
if( pRtree->pNodeBlob ){
sqlite3_blob *pBlob = pRtree->pNodeBlob;
pRtree->pNodeBlob = 0;
rc = sqlite3_blob_reopen(pBlob, iNode);
pRtree->pNodeBlob = pBlob;
if( rc ){
nodeBlobReset(pRtree);
if( rc==SQLITE_NOMEM ) return SQLITE_NOMEM;
}
}
if( pRtree->pNodeBlob==0 ){
char *zTab = sqlite3_mprintf("%s_node", pRtree->zName);
if( zTab==0 ) return SQLITE_NOMEM;
rc = sqlite3_blob_open(pRtree->db, pRtree->zDb, zTab, "data", iNode, 0,
&pRtree->pNodeBlob);
sqlite3_free(zTab);
}
if( rc ){
nodeBlobReset(pRtree);
*ppNode = 0;
/* If unable to open an sqlite3_blob on the desired row, that can only
** be because the shadow tables hold erroneous data. */
if( rc==SQLITE_ERROR ) rc = SQLITE_CORRUPT_VTAB;
}else if( pRtree->iNodeSize==sqlite3_blob_bytes(pRtree->pNodeBlob) ){
pNode = (RtreeNode *)sqlite3_malloc(sizeof(RtreeNode)+pRtree->iNodeSize);
if( !pNode ){
rc = SQLITE_NOMEM;
}else{
pNode->pParent = pParent;
pNode->zData = (u8 *)&pNode[1];
pNode->nRef = 1;
pNode->iNode = iNode;
pNode->isDirty = 0;
pNode->pNext = 0;
rc = sqlite3_blob_read(pRtree->pNodeBlob, pNode->zData,
pRtree->iNodeSize, 0);
nodeReference(pParent);
}
}
rc = sqlite3_reset(pRtree->pReadNode);
if( rc==SQLITE_OK ) rc = rc2;
/* If the root node was just loaded, set pRtree->iDepth to the height
** of the r-tree structure. A height of zero means all data is stored on
@@ -602,7 +731,7 @@ static void nodeOverwriteCell(
int ii;
u8 *p = &pNode->zData[4 + pRtree->nBytesPerCell*iCell];
p += writeInt64(p, pCell->iRowid);
for(ii=0; ii<(pRtree->nDim*2); ii++){
for(ii=0; ii<pRtree->nDim2; ii++){
p += writeCoord(p, &pCell->aCoord[ii]);
}
pNode->isDirty = 1;
@@ -736,13 +865,16 @@ static void nodeGetCell(
){
u8 *pData;
RtreeCoord *pCoord;
int ii;
int ii = 0;
pCell->iRowid = nodeGetRowid(pRtree, pNode, iCell);
pData = pNode->zData + (12 + pRtree->nBytesPerCell*iCell);
pCoord = pCell->aCoord;
for(ii=0; ii<pRtree->nDim*2; ii++){
readCoord(&pData[ii*4], &pCoord[ii]);
}
do{
readCoord(pData, &pCoord[ii]);
readCoord(pData+4, &pCoord[ii+1]);
pData += 8;
ii += 2;
}while( ii<pRtree->nDim2 );
}
@@ -793,7 +925,9 @@ static void rtreeReference(Rtree *pRtree){
static void rtreeRelease(Rtree *pRtree){
pRtree->nBusy--;
if( pRtree->nBusy==0 ){
sqlite3_finalize(pRtree->pReadNode);
pRtree->inWrTrans = 0;
pRtree->nCursor = 0;
nodeBlobReset(pRtree);
sqlite3_finalize(pRtree->pWriteNode);
sqlite3_finalize(pRtree->pDeleteNode);
sqlite3_finalize(pRtree->pReadRowid);
@@ -831,6 +965,7 @@ static int rtreeDestroy(sqlite3_vtab *pVtab){
if( !zCreate ){
rc = SQLITE_NOMEM;
}else{
nodeBlobReset(pRtree);
rc = sqlite3_exec(pRtree->db, zCreate, 0, 0, 0);
sqlite3_free(zCreate);
}
@@ -846,6 +981,7 @@ static int rtreeDestroy(sqlite3_vtab *pVtab){
*/
static int rtreeOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
int rc = SQLITE_NOMEM;
Rtree *pRtree = (Rtree *)pVTab;
RtreeCursor *pCsr;
pCsr = (RtreeCursor *)sqlite3_malloc(sizeof(RtreeCursor));
@@ -853,6 +989,7 @@ static int rtreeOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
memset(pCsr, 0, sizeof(RtreeCursor));
pCsr->base.pVtab = pVTab;
rc = SQLITE_OK;
pRtree->nCursor++;
}
*ppCursor = (sqlite3_vtab_cursor *)pCsr;
@@ -885,10 +1022,13 @@ static int rtreeClose(sqlite3_vtab_cursor *cur){
Rtree *pRtree = (Rtree *)(cur->pVtab);
int ii;
RtreeCursor *pCsr = (RtreeCursor *)cur;
assert( pRtree->nCursor>0 );
freeCursorConstraints(pCsr);
sqlite3_free(pCsr->aPoint);
for(ii=0; ii<RTREE_CACHE_SZ; ii++) nodeRelease(pRtree, pCsr->aNode[ii]);
sqlite3_free(pCsr);
pRtree->nCursor--;
nodeBlobReset(pRtree);
return SQLITE_OK;
}
@@ -911,15 +1051,22 @@ static int rtreeEof(sqlite3_vtab_cursor *cur){
** false. a[] is the four bytes of the on-disk record to be decoded.
** Store the results in "r".
**
** There are three versions of this macro, one each for little-endian and
** big-endian processors and a third generic implementation. The endian-
** specific implementations are much faster and are preferred if the
** processor endianness is known at compile-time. The SQLITE_BYTEORDER
** macro is part of sqliteInt.h and hence the endian-specific
** implementation will only be used if this module is compiled as part
** of the amalgamation.
** There are five versions of this macro. The last one is generic. The
** other four are various architectures-specific optimizations.
*/
#if defined(SQLITE_BYTEORDER) && SQLITE_BYTEORDER==1234
#if SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300
#define RTREE_DECODE_COORD(eInt, a, r) { \
RtreeCoord c; /* Coordinate decoded */ \
c.u = _byteswap_ulong(*(u32*)a); \
r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \
}
#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000
#define RTREE_DECODE_COORD(eInt, a, r) { \
RtreeCoord c; /* Coordinate decoded */ \
c.u = __builtin_bswap32(*(u32*)a); \
r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \
}
#elif SQLITE_BYTEORDER==1234
#define RTREE_DECODE_COORD(eInt, a, r) { \
RtreeCoord c; /* Coordinate decoded */ \
memcpy(&c.u,a,4); \
@@ -927,7 +1074,7 @@ static int rtreeEof(sqlite3_vtab_cursor *cur){
((c.u&0xff)<<24)|((c.u&0xff00)<<8); \
r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \
}
#elif defined(SQLITE_BYTEORDER) && SQLITE_BYTEORDER==4321
#elif SQLITE_BYTEORDER==4321
#define RTREE_DECODE_COORD(eInt, a, r) { \
RtreeCoord c; /* Coordinate decoded */ \
memcpy(&c.u,a,4); \
@@ -954,10 +1101,10 @@ static int rtreeCallbackConstraint(
sqlite3_rtree_dbl *prScore, /* OUT: score for the cell */
int *peWithin /* OUT: visibility of the cell */
){
int i; /* Loop counter */
sqlite3_rtree_query_info *pInfo = pConstraint->pInfo; /* Callback info */
int nCoord = pInfo->nCoord; /* No. of coordinates */
int rc; /* Callback return code */
RtreeCoord c; /* Translator union */
sqlite3_rtree_dbl aCoord[RTREE_MAX_DIMENSIONS*2]; /* Decoded coordinates */
assert( pConstraint->op==RTREE_MATCH || pConstraint->op==RTREE_QUERY );
@@ -967,13 +1114,41 @@ static int rtreeCallbackConstraint(
pInfo->iRowid = readInt64(pCellData);
}
pCellData += 8;
for(i=0; i<nCoord; i++, pCellData += 4){
RTREE_DECODE_COORD(eInt, pCellData, aCoord[i]);
#ifndef SQLITE_RTREE_INT_ONLY
if( eInt==0 ){
switch( nCoord ){
case 10: readCoord(pCellData+36, &c); aCoord[9] = c.f;
readCoord(pCellData+32, &c); aCoord[8] = c.f;
case 8: readCoord(pCellData+28, &c); aCoord[7] = c.f;
readCoord(pCellData+24, &c); aCoord[6] = c.f;
case 6: readCoord(pCellData+20, &c); aCoord[5] = c.f;
readCoord(pCellData+16, &c); aCoord[4] = c.f;
case 4: readCoord(pCellData+12, &c); aCoord[3] = c.f;
readCoord(pCellData+8, &c); aCoord[2] = c.f;
default: readCoord(pCellData+4, &c); aCoord[1] = c.f;
readCoord(pCellData, &c); aCoord[0] = c.f;
}
}else
#endif
{
switch( nCoord ){
case 10: readCoord(pCellData+36, &c); aCoord[9] = c.i;
readCoord(pCellData+32, &c); aCoord[8] = c.i;
case 8: readCoord(pCellData+28, &c); aCoord[7] = c.i;
readCoord(pCellData+24, &c); aCoord[6] = c.i;
case 6: readCoord(pCellData+20, &c); aCoord[5] = c.i;
readCoord(pCellData+16, &c); aCoord[4] = c.i;
case 4: readCoord(pCellData+12, &c); aCoord[3] = c.i;
readCoord(pCellData+8, &c); aCoord[2] = c.i;
default: readCoord(pCellData+4, &c); aCoord[1] = c.i;
readCoord(pCellData, &c); aCoord[0] = c.i;
}
}
if( pConstraint->op==RTREE_MATCH ){
int eWithin = 0;
rc = pConstraint->u.xGeom((sqlite3_rtree_geometry*)pInfo,
nCoord, aCoord, &i);
if( i==0 ) *peWithin = NOT_WITHIN;
nCoord, aCoord, &eWithin);
if( eWithin==0 ) *peWithin = NOT_WITHIN;
*prScore = RTREE_ZERO;
}else{
pInfo->aCoord = aCoord;
@@ -1009,6 +1184,7 @@ static void rtreeNonleafConstraint(
assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE
|| p->op==RTREE_GT || p->op==RTREE_EQ );
assert( ((((char*)pCellData) - (char*)0)&3)==0 ); /* 4-byte aligned */
switch( p->op ){
case RTREE_LE:
case RTREE_LT:
@@ -1049,6 +1225,7 @@ static void rtreeLeafConstraint(
assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE
|| p->op==RTREE_GT || p->op==RTREE_EQ );
pCellData += 8 + p->iCoord*4;
assert( ((((char*)pCellData) - (char*)0)&3)==0 ); /* 4-byte aligned */
RTREE_DECODE_COORD(eInt, pCellData, xN);
switch( p->op ){
case RTREE_LE: if( xN <= p->u.rValue ) return; break;
@@ -1117,7 +1294,7 @@ static int rtreeSearchPointCompare(
}
/*
** Interchange to search points in a cursor.
** Interchange two search points in a cursor.
*/
static void rtreeSearchPointSwap(RtreeCursor *p, int i, int j){
RtreeSearchPoint t = p->aPoint[i];
@@ -1365,7 +1542,7 @@ static int rtreeStepToLeaf(RtreeCursor *pCur){
if( rScore<RTREE_ZERO ) rScore = RTREE_ZERO;
p = rtreeSearchPointNew(pCur, rScore, x.iLevel);
if( p==0 ) return SQLITE_NOMEM;
p->eWithin = eWithin;
p->eWithin = (u8)eWithin;
p->id = x.id;
p->iCell = x.iCell;
RTREE_QUEUE_TRACE(pCur, "PUSH-S:");
@@ -1424,7 +1601,6 @@ static int rtreeColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){
if( i==0 ){
sqlite3_result_int64(ctx, nodeGetRowid(pRtree, pNode, p->iCell));
}else{
if( rc ) return rc;
nodeGetCoord(pRtree, pNode, p->iCell, i-1, &c);
#ifndef SQLITE_RTREE_INT_ONLY
if( pRtree->eCoordType==RTREE_COORD_REAL32 ){
@@ -1542,7 +1718,7 @@ static int rtreeFilter(
if( idxNum==1 ){
/* Special case - lookup by rowid. */
RtreeNode *pLeaf; /* Leaf on which the required cell resides */
RtreeSearchPoint *p; /* Search point for the the leaf */
RtreeSearchPoint *p; /* Search point for the leaf */
i64 iRowid = sqlite3_value_int64(argv[0]);
i64 iNode = 0;
rc = findLeafNode(pRtree, iRowid, &pLeaf, &iNode);
@@ -1553,7 +1729,7 @@ static int rtreeFilter(
p->id = iNode;
p->eWithin = PARTLY_WITHIN;
rc = nodeRowidIndex(pRtree, pLeaf, iRowid, &iCell);
p->iCell = iCell;
p->iCell = (u8)iCell;
RTREE_QUEUE_TRACE(pCsr, "PUSH-F1:");
}else{
pCsr->atEOF = 1;
@@ -1586,7 +1762,7 @@ static int rtreeFilter(
if( rc!=SQLITE_OK ){
break;
}
p->pInfo->nCoord = pRtree->nDim*2;
p->pInfo->nCoord = pRtree->nDim2;
p->pInfo->anQueue = pCsr->anQueue;
p->pInfo->mxLevel = pRtree->iDepth + 1;
}else{
@@ -1601,7 +1777,7 @@ static int rtreeFilter(
}
if( rc==SQLITE_OK ){
RtreeSearchPoint *pNew;
pNew = rtreeSearchPointNew(pCsr, RTREE_ZERO, pRtree->iDepth+1);
pNew = rtreeSearchPointNew(pCsr, RTREE_ZERO, (u8)(pRtree->iDepth+1));
if( pNew==0 ) return SQLITE_NOMEM;
pNew->id = 1;
pNew->iCell = 0;
@@ -1619,19 +1795,6 @@ static int rtreeFilter(
return rc;
}
/*
** Set the pIdxInfo->estimatedRows variable to nRow. Unless this
** extension is currently being used by a version of SQLite too old to
** support estimatedRows. In that case this function is a no-op.
*/
static void setEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){
#if SQLITE_VERSION_NUMBER>=3008002
if( sqlite3_libversion_number()>=3008002 ){
pIdxInfo->estimatedRows = nRow;
}
#endif
}
/*
** Rtree virtual table module xBestIndex method. There are three
** table scan strategies to choose from (in order from most to
@@ -1711,7 +1874,7 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
** a single row.
*/
pIdxInfo->estimatedCost = 30.0;
setEstimatedRows(pIdxInfo, 1);
pIdxInfo->estimatedRows = 1;
return SQLITE_OK;
}
@@ -1729,7 +1892,7 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
break;
}
zIdxStr[iIdx++] = op;
zIdxStr[iIdx++] = p->iColumn - 1 + '0';
zIdxStr[iIdx++] = (char)(p->iColumn - 1 + '0');
pIdxInfo->aConstraintUsage[ii].argvIndex = (iIdx/2);
pIdxInfo->aConstraintUsage[ii].omit = 1;
}
@@ -1743,7 +1906,7 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
nRow = pRtree->nRowEst >> (iIdx/2);
pIdxInfo->estimatedCost = (double)6.0 * (double)nRow;
setEstimatedRows(pIdxInfo, nRow);
pIdxInfo->estimatedRows = nRow;
return rc;
}
@@ -1753,9 +1916,26 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
*/
static RtreeDValue cellArea(Rtree *pRtree, RtreeCell *p){
RtreeDValue area = (RtreeDValue)1;
int ii;
for(ii=0; ii<(pRtree->nDim*2); ii+=2){
area = (area * (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii])));
assert( pRtree->nDim>=1 && pRtree->nDim<=5 );
#ifndef SQLITE_RTREE_INT_ONLY
if( pRtree->eCoordType==RTREE_COORD_REAL32 ){
switch( pRtree->nDim ){
case 5: area = p->aCoord[9].f - p->aCoord[8].f;
case 4: area *= p->aCoord[7].f - p->aCoord[6].f;
case 3: area *= p->aCoord[5].f - p->aCoord[4].f;
case 2: area *= p->aCoord[3].f - p->aCoord[2].f;
default: area *= p->aCoord[1].f - p->aCoord[0].f;
}
}else
#endif
{
switch( pRtree->nDim ){
case 5: area = p->aCoord[9].i - p->aCoord[8].i;
case 4: area *= p->aCoord[7].i - p->aCoord[6].i;
case 3: area *= p->aCoord[5].i - p->aCoord[4].i;
case 2: area *= p->aCoord[3].i - p->aCoord[2].i;
default: area *= p->aCoord[1].i - p->aCoord[0].i;
}
}
return area;
}
@@ -1765,11 +1945,12 @@ static RtreeDValue cellArea(Rtree *pRtree, RtreeCell *p){
** of the objects size in each dimension.
*/
static RtreeDValue cellMargin(Rtree *pRtree, RtreeCell *p){
RtreeDValue margin = (RtreeDValue)0;
int ii;
for(ii=0; ii<(pRtree->nDim*2); ii+=2){
RtreeDValue margin = 0;
int ii = pRtree->nDim2 - 2;
do{
margin += (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii]));
}
ii -= 2;
}while( ii>=0 );
return margin;
}
@@ -1777,17 +1958,19 @@ static RtreeDValue cellMargin(Rtree *pRtree, RtreeCell *p){
** Store the union of cells p1 and p2 in p1.
*/
static void cellUnion(Rtree *pRtree, RtreeCell *p1, RtreeCell *p2){
int ii;
int ii = 0;
if( pRtree->eCoordType==RTREE_COORD_REAL32 ){
for(ii=0; ii<(pRtree->nDim*2); ii+=2){
do{
p1->aCoord[ii].f = MIN(p1->aCoord[ii].f, p2->aCoord[ii].f);
p1->aCoord[ii+1].f = MAX(p1->aCoord[ii+1].f, p2->aCoord[ii+1].f);
}
ii += 2;
}while( ii<pRtree->nDim2 );
}else{
for(ii=0; ii<(pRtree->nDim*2); ii+=2){
do{
p1->aCoord[ii].i = MIN(p1->aCoord[ii].i, p2->aCoord[ii].i);
p1->aCoord[ii+1].i = MAX(p1->aCoord[ii+1].i, p2->aCoord[ii+1].i);
}
ii += 2;
}while( ii<pRtree->nDim2 );
}
}
@@ -1798,7 +1981,7 @@ static void cellUnion(Rtree *pRtree, RtreeCell *p1, RtreeCell *p2){
static int cellContains(Rtree *pRtree, RtreeCell *p1, RtreeCell *p2){
int ii;
int isInt = (pRtree->eCoordType==RTREE_COORD_INT32);
for(ii=0; ii<(pRtree->nDim*2); ii+=2){
for(ii=0; ii<pRtree->nDim2; ii+=2){
RtreeCoord *a1 = &p1->aCoord[ii];
RtreeCoord *a2 = &p2->aCoord[ii];
if( (!isInt && (a2[0].f<a1[0].f || a2[1].f>a1[1].f))
@@ -1833,7 +2016,7 @@ static RtreeDValue cellOverlap(
for(ii=0; ii<nCell; ii++){
int jj;
RtreeDValue o = (RtreeDValue)1;
for(jj=0; jj<(pRtree->nDim*2); jj+=2){
for(jj=0; jj<pRtree->nDim2; jj+=2){
RtreeDValue x1, x2;
x1 = MAX(DCOORD(p->aCoord[jj]), DCOORD(aCell[ii].aCoord[jj]));
x2 = MIN(DCOORD(p->aCoord[jj+1]), DCOORD(aCell[ii].aCoord[jj+1]));
@@ -2800,6 +2983,53 @@ static RtreeValue rtreeValueUp(sqlite3_value *v){
}
#endif /* !defined(SQLITE_RTREE_INT_ONLY) */
/*
** A constraint has failed while inserting a row into an rtree table.
** Assuming no OOM error occurs, this function sets the error message
** (at pRtree->base.zErrMsg) to an appropriate value and returns
** SQLITE_CONSTRAINT.
**
** Parameter iCol is the index of the leftmost column involved in the
** constraint failure. If it is 0, then the constraint that failed is
** the unique constraint on the id column. Otherwise, it is the rtree
** (c1<=c2) constraint on columns iCol and iCol+1 that has failed.
**
** If an OOM occurs, SQLITE_NOMEM is returned instead of SQLITE_CONSTRAINT.
*/
static int rtreeConstraintError(Rtree *pRtree, int iCol){
sqlite3_stmt *pStmt = 0;
char *zSql;
int rc;
assert( iCol==0 || iCol%2 );
zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", pRtree->zDb, pRtree->zName);
if( zSql ){
rc = sqlite3_prepare_v2(pRtree->db, zSql, -1, &pStmt, 0);
}else{
rc = SQLITE_NOMEM;
}
sqlite3_free(zSql);
if( rc==SQLITE_OK ){
if( iCol==0 ){
const char *zCol = sqlite3_column_name(pStmt, 0);
pRtree->base.zErrMsg = sqlite3_mprintf(
"UNIQUE constraint failed: %s.%s", pRtree->zName, zCol
);
}else{
const char *zCol1 = sqlite3_column_name(pStmt, iCol);
const char *zCol2 = sqlite3_column_name(pStmt, iCol+1);
pRtree->base.zErrMsg = sqlite3_mprintf(
"rtree constraint failed: %s.(%s<=%s)", pRtree->zName, zCol1, zCol2
);
}
}
sqlite3_finalize(pStmt);
return (rc==SQLITE_OK ? SQLITE_CONSTRAINT : rc);
}
/*
** The xUpdate method for rtree module virtual tables.
@@ -2842,7 +3072,7 @@ static int rtreeUpdate(
** This problem was discovered after years of use, so we silently ignore
** these kinds of misdeclared tables to avoid breaking any legacy.
*/
assert( nData<=(pRtree->nDim*2 + 3) );
assert( nData<=(pRtree->nDim2 + 3) );
#ifndef SQLITE_RTREE_INT_ONLY
if( pRtree->eCoordType==RTREE_COORD_REAL32 ){
@@ -2850,7 +3080,7 @@ static int rtreeUpdate(
cell.aCoord[ii].f = rtreeValueDown(azData[ii+3]);
cell.aCoord[ii+1].f = rtreeValueUp(azData[ii+4]);
if( cell.aCoord[ii].f>cell.aCoord[ii+1].f ){
rc = SQLITE_CONSTRAINT;
rc = rtreeConstraintError(pRtree, ii+1);
goto constraint;
}
}
@@ -2861,7 +3091,7 @@ static int rtreeUpdate(
cell.aCoord[ii].i = sqlite3_value_int(azData[ii+3]);
cell.aCoord[ii+1].i = sqlite3_value_int(azData[ii+4]);
if( cell.aCoord[ii].i>cell.aCoord[ii+1].i ){
rc = SQLITE_CONSTRAINT;
rc = rtreeConstraintError(pRtree, ii+1);
goto constraint;
}
}
@@ -2882,7 +3112,7 @@ static int rtreeUpdate(
if( sqlite3_vtab_on_conflict(pRtree->db)==SQLITE_REPLACE ){
rc = rtreeDeleteRowid(pRtree, cell.iRowid);
}else{
rc = SQLITE_CONSTRAINT;
rc = rtreeConstraintError(pRtree, 0);
goto constraint;
}
}
@@ -2932,6 +3162,27 @@ constraint:
return rc;
}
/*
** Called when a transaction starts.
*/
static int rtreeBeginTransaction(sqlite3_vtab *pVtab){
Rtree *pRtree = (Rtree *)pVtab;
assert( pRtree->inWrTrans==0 );
pRtree->inWrTrans++;
return SQLITE_OK;
}
/*
** Called when a transaction completes (either by COMMIT or ROLLBACK).
** The sqlite3_blob object should be released at this point.
*/
static int rtreeEndTransaction(sqlite3_vtab *pVtab){
Rtree *pRtree = (Rtree *)pVtab;
pRtree->inWrTrans = 0;
nodeBlobReset(pRtree);
return SQLITE_OK;
}
/*
** The xRename method for rtree module virtual tables.
*/
@@ -2953,6 +3204,7 @@ static int rtreeRename(sqlite3_vtab *pVtab, const char *zNewName){
return rc;
}
/*
** This function populates the pRtree->nRowEst variable with an estimate
** of the number of rows in the virtual table. If possible, this is based
@@ -2965,6 +3217,13 @@ static int rtreeQueryStat1(sqlite3 *db, Rtree *pRtree){
int rc;
i64 nRow = 0;
rc = sqlite3_table_column_metadata(
db, pRtree->zDb, "sqlite_stat1",0,0,0,0,0,0
);
if( rc!=SQLITE_OK ){
pRtree->nRowEst = RTREE_DEFAULT_ROWEST;
return rc==SQLITE_ERROR ? SQLITE_OK : rc;
}
zSql = sqlite3_mprintf(zFmt, pRtree->zDb, pRtree->zName);
if( zSql==0 ){
rc = SQLITE_NOMEM;
@@ -3005,15 +3264,15 @@ static sqlite3_module rtreeModule = {
rtreeColumn, /* xColumn - read data */
rtreeRowid, /* xRowid - read data */
rtreeUpdate, /* xUpdate - write data */
0, /* xBegin - begin transaction */
0, /* xSync - sync transaction */
0, /* xCommit - commit transaction */
0, /* xRollback - rollback transaction */
rtreeBeginTransaction, /* xBegin - begin transaction */
rtreeEndTransaction, /* xSync - sync transaction */
rtreeEndTransaction, /* xCommit - commit transaction */
rtreeEndTransaction, /* xRollback - rollback transaction */
0, /* xFindFunction - function overloading */
rtreeRename, /* xRename - rename the table */
0, /* xSavepoint */
0, /* xRelease */
0 /* xRollbackTo */
0, /* xRollbackTo */
};
static int rtreeSqlInit(
@@ -3025,10 +3284,9 @@ static int rtreeSqlInit(
){
int rc = SQLITE_OK;
#define N_STATEMENT 9
#define N_STATEMENT 8
static const char *azSql[N_STATEMENT] = {
/* Read and write the xxx_node table */
"SELECT data FROM '%q'.'%q_node' WHERE nodeno = :1",
/* Write the xxx_node table */
"INSERT OR REPLACE INTO '%q'.'%q_node' VALUES(:1, :2)",
"DELETE FROM '%q'.'%q_node' WHERE nodeno = :1",
@@ -3066,15 +3324,14 @@ static int rtreeSqlInit(
}
}
appStmt[0] = &pRtree->pReadNode;
appStmt[1] = &pRtree->pWriteNode;
appStmt[2] = &pRtree->pDeleteNode;
appStmt[3] = &pRtree->pReadRowid;
appStmt[4] = &pRtree->pWriteRowid;
appStmt[5] = &pRtree->pDeleteRowid;
appStmt[6] = &pRtree->pReadParent;
appStmt[7] = &pRtree->pWriteParent;
appStmt[8] = &pRtree->pDeleteParent;
appStmt[0] = &pRtree->pWriteNode;
appStmt[1] = &pRtree->pDeleteNode;
appStmt[2] = &pRtree->pReadRowid;
appStmt[3] = &pRtree->pWriteRowid;
appStmt[4] = &pRtree->pDeleteRowid;
appStmt[5] = &pRtree->pReadParent;
appStmt[6] = &pRtree->pWriteParent;
appStmt[7] = &pRtree->pDeleteParent;
rc = rtreeQueryStat1(db, pRtree);
for(i=0; i<N_STATEMENT && rc==SQLITE_OK; i++){
@@ -3212,9 +3469,10 @@ static int rtreeInit(
pRtree->base.pModule = &rtreeModule;
pRtree->zDb = (char *)&pRtree[1];
pRtree->zName = &pRtree->zDb[nDb+1];
pRtree->nDim = (argc-4)/2;
pRtree->nBytesPerCell = 8 + pRtree->nDim*4*2;
pRtree->eCoordType = eCoordType;
pRtree->nDim = (u8)((argc-4)/2);
pRtree->nDim2 = pRtree->nDim*2;
pRtree->nBytesPerCell = 8 + pRtree->nDim2*4;
pRtree->eCoordType = (u8)eCoordType;
memcpy(pRtree->zDb, argv[1], nDb);
memcpy(pRtree->zName, argv[2], nName);
@@ -3287,7 +3545,8 @@ static void rtreenode(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){
UNUSED_PARAMETER(nArg);
memset(&node, 0, sizeof(RtreeNode));
memset(&tree, 0, sizeof(Rtree));
tree.nDim = sqlite3_value_int(apArg[0]);
tree.nDim = (u8)sqlite3_value_int(apArg[0]);
tree.nDim2 = tree.nDim*2;
tree.nBytesPerCell = 8 + 8 * tree.nDim;
node.zData = (u8 *)sqlite3_value_blob(apArg[1]);
@@ -3300,7 +3559,7 @@ static void rtreenode(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){
nodeGetCell(&tree, &node, ii, &cell);
sqlite3_snprintf(512-nCell,&zCell[nCell],"%lld", cell.iRowid);
nCell = (int)strlen(zCell);
for(jj=0; jj<tree.nDim*2; jj++){
for(jj=0; jj<tree.nDim2; jj++){
#ifndef SQLITE_RTREE_INT_ONLY
sqlite3_snprintf(512-nCell,&zCell[nCell], " %g",
(double)cell.aCoord[jj].f);

20
ext/rtree/rtree1.test
View File

@@ -194,13 +194,13 @@ do_test rtree-2.1.3 {
do_test rtree-2.2.1 {
catchsql { INSERT INTO t1 VALUES(2, 1, 3, 2, 4) }
} {1 {constraint failed}}
} {1 {UNIQUE constraint failed: t1.ii}}
do_test rtree-2.2.2 {
catchsql { INSERT INTO t1 VALUES(4, 1, 3, 4, 2) }
} {1 {constraint failed}}
} {1 {rtree constraint failed: t1.(y1<=y2)}}
do_test rtree-2.2.3 {
catchsql { INSERT INTO t1 VALUES(4, 3, 1, 2, 4) }
} {1 {constraint failed}}
} {1 {rtree constraint failed: t1.(x1<=x2)}}
do_test rtree-2.2.4 {
execsql { SELECT ii FROM t1 ORDER BY ii }
} {1 2 3}
@@ -236,7 +236,7 @@ do_test rtree-3.1.3 {
# Test the constraint on the coordinates (c[i]<=c[i+1] where (i%2==0)):
do_test rtree-3.2.1 {
catchsql { INSERT INTO t1 VALUES(7, 2, 6, 4, 3) }
} {1 {constraint failed}}
} {1 {rtree constraint failed: t1.(y1<=y2)}}
do_test rtree-3.2.2 {
catchsql { INSERT INTO t1 VALUES(8, 2, 6, 3, 3) }
} {0 {}}
@@ -490,11 +490,11 @@ foreach {tn sql_template testdata} {
}
4 "INSERT %CONF% INTO t1 VALUES(2, 7, 6, 7, 7)" {
ROLLBACK 0 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6}
ABORT 0 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7}
ROLLBACK 0 2 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6}
ABORT 0 2 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7}
IGNORE 0 0 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7}
FAIL 0 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7}
REPLACE 0 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7}
FAIL 0 2 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7}
REPLACE 0 2 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7}
}
} {
@@ -510,7 +510,9 @@ foreach {tn sql_template testdata} {
}
set res(0) {0 {}}
set res(1) {1 {constraint failed}}
set res(1) {1 {UNIQUE constraint failed: t1.idx}}
set res(2) {1 {rtree constraint failed: t1.(x1<=x2)}}
do_catchsql_test $testname.1 $sql $res($error)
do_test $testname.2 [list sql_uses_stmt db $sql] $uses
do_execsql_test $testname.3 { SELECT * FROM t1 ORDER BY idx } $data

31
ext/rtree/rtree3.test
View File

@@ -47,7 +47,8 @@ ifcapable !rtree {
#
# rtree3-8: Test OOM while registering the r-tree module with sqlite.
#
# rtree3-11: OOM following a constraint failure
#
do_faultsim_test rtree3-1 -faults oom* -prep {
faultsim_delete_and_reopen
} -body {
@@ -234,4 +235,32 @@ do_faultsim_test rtree3-10 -faults oom-* -prep {
faultsim_test_result {0 2}
}
do_test rtree3-11.prep {
faultsim_delete_and_reopen
execsql {
CREATE VIRTUAL TABLE rt USING rtree(ii, x1, x2, y1, y2);
INSERT INTO rt VALUES(1, 2, 3, 4, 5);
}
faultsim_save_and_close
} {}
do_faultsim_test rtree3-10.1 -faults oom-* -prep {
faultsim_restore_and_reopen
execsql { SELECT * FROM rt }
} -body {
execsql { INSERT INTO rt VALUES(1, 2, 3, 4, 5) }
} -test {
faultsim_test_result {1 {UNIQUE constraint failed: rt.ii}} \
{1 {constraint failed}}
}
do_faultsim_test rtree3-10.2 -faults oom-* -prep {
faultsim_restore_and_reopen
execsql { SELECT * FROM rt }
} -body {
execsql { INSERT INTO rt VALUES(2, 2, 3, 5, 4) }
} -test {
faultsim_test_result {1 {rtree constraint failed: rt.(y1<=y2)}} \
{1 {constraint failed}}
}
finish_test

2
ext/rtree/rtreeA.test
View File

@@ -109,7 +109,7 @@ do_corruption_tests rtreeA-1.1 {
}
do_execsql_test rtreeA-1.2.0 { DROP TABLE t1_node } {}
do_corruption_tests rtreeA-1.2 -error "SQL logic error or missing database" {
do_corruption_tests rtreeA-1.2 -error "database disk image is malformed" {
1 "SELECT * FROM t1"
2 "SELECT * FROM t1 WHERE rowid=5"
3 "INSERT INTO t1 VALUES(1000, 1, 2, 3, 4)"

4
ext/rtree/rtreeC.test
View File

@@ -350,7 +350,3 @@ do_eqp_execsql_test 7.4 {
}
finish_test
finish_test

2
ext/rtree/rtreeD.test
View File

@@ -19,7 +19,7 @@ if {![info exists testdir]} {
source [file join [file dirname [info script]] rtree_util.tcl]
source $testdir/tester.tcl
source $testdir/lock_common.tcl
ifcapable !rtree {
ifcapable !rtree||!builtin_test {
finish_test
return
}

66
ext/rtree/rtreeG.test Normal file
View File

@@ -0,0 +1,66 @@
# 2016-05-32
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
# This file contains tests for the r-tree module.
#
# Verify that no invalid SQL is run during initialization
if {![info exists testdir]} {
set testdir [file join [file dirname [info script]] .. .. test]
}
source $testdir/tester.tcl
ifcapable !rtree { finish_test ; return }
db close
sqlite3_shutdown
test_sqlite3_log [list lappend ::log]
set ::log [list]
sqlite3 db test.db
set ::log {}
do_execsql_test rtreeG-1.1 {
CREATE VIRTUAL TABLE t1 USING rtree(id,x0,x1,y0,y1);
} {}
do_test rtreeG-1.1log {
set ::log
} {}
do_execsql_test rtreeG-1.2 {
INSERT INTO t1 VALUES(1,10,15,5,23),(2,20,21,5,23),(3,10,15,20,30);
SELECT id from t1 WHERE x0>8 AND x1<16 AND y0>2 AND y1<25;
} {1}
do_test rtreeG-1.2log {
set ::log
} {}
db close
sqlite3 db test.db
do_execsql_test rtreeG-1.3 {
SELECT id from t1 WHERE x0>8 AND x1<16 AND y0>2 AND y1<25;
} {1}
do_test rtreeG-1.3log {
set ::log
} {}
do_execsql_test rtreeG-1.4 {
DROP TABLE t1;
} {}
do_test rtreeG-1.4log {
set ::log
} {}
db close
sqlite3_shutdown
test_sqlite3_log
sqlite3_initialize
sqlite3 db test.db
finish_test

Some files were not shown because too many files have changed in this diff Show More