diff --git a/Makefile.in b/Makefile.in index b0871ad940..4a81e15664 100644 --- a/Makefile.in +++ b/Makefile.in @@ -379,7 +379,8 @@ TESTSRC = \ $(TOP)/src/test_vfs.c \ $(TOP)/src/test_wholenumber.c \ $(TOP)/src/test_wsd.c \ - $(TOP)/ext/fts3/fts3_term.c + $(TOP)/ext/fts3/fts3_term.c \ + $(TOP)/ext/fts3/fts3_test.c # Source code to the library files needed by the test fixture # diff --git a/Makefile.msc b/Makefile.msc new file mode 100644 index 0000000000..68a66bcb45 --- /dev/null +++ b/Makefile.msc @@ -0,0 +1,864 @@ +# +# nmake Makefile for SQLite +# + +# The toplevel directory of the source tree. This is the directory +# that contains this "Makefile.msc". +# +TOP = . + +# Set this non-0 to create and use the SQLite amalgamation file. +# +USE_AMALGAMATION = 1 + +# Version numbers and release number for the SQLite being compiled. +# +VERSION = 3.7 +VERSION_NUMBER = 3007007 +RELEASE = 3.7.7 + +# C Compiler and options for use in building executables that +# will run on the platform that is doing the build. +# +BCC = cl.exe -O2 + +# C Compile and options for use in building executables that +# will run on the target platform. (BCC and TCC are usually the +# same unless your are cross-compiling.) +# +TCC = cl.exe -W3 -O2 -DSQLITE_OS_WIN=1 -I. -I$(TOP)\src + +# The mksqlite3c.tcl and mksqlite3h.tcl scripts will pull in +# any extension header files by default. For non-amalgamation +# builds, we need to make sure the compiler can find these. +# +!IF $(USE_AMALGAMATION)==0 +TCC = $(TCC) -I$(TOP)\ext\fts3 +TCC = $(TCC) -I$(TOP)\ext\rtree +!ENDIF + +# Define -DNDEBUG to compile without debugging (i.e., for production usage) +# Omitting the define will cause extra debugging code to be inserted and +# includes extra comments when "EXPLAIN stmt" is used. +# +TCC = $(TCC) -DNDEBUG + +# The library that programs using TCL must link against. +# +LIBTCL = tcl85.lib +TCLINCDIR = c:\tcl\include +TCLLIBDIR = c:\tcl\lib + +# This is the command to use for tclsh - normally just "tclsh", but we may +# know the specific version we want to use +# +TCLSH_CMD = tclsh85 + +# Compiler options needed for programs that use the readline() library. +# +READLINE_FLAGS = -DHAVE_READLINE=0 + +# The library that programs using readline() must link against. +# +LIBREADLINE = + +# Should the database engine be compiled threadsafe +# +TCC = $(TCC) -DSQLITE_THREADSAFE=1 + +# Do threads override each others locks by default (1), or do we test (-1) +# +TCC = $(TCC) -DSQLITE_THREAD_OVERRIDE_LOCK=-1 + +# Any target libraries which libsqlite must be linked against +# +TLIBS = + +# Flags controlling use of the in memory btree implementation +# +# SQLITE_TEMP_STORE is 0 to force temporary tables to be in a file, 1 to +# default to file, 2 to default to memory, and 3 to force temporary +# tables to always be in memory. +# +TCC = $(TCC) -DSQLITE_TEMP_STORE=1 + +# Enable/disable loadable extensions, and other optional features +# based on configuration. (-DSQLITE_OMIT*, -DSQLITE_ENABLE*). +# The same set of OMIT and ENABLE flags should be passed to the +# LEMON parser generator and the mkkeywordhash tool as well. + +# BEGIN standard options +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1 +# END standard options + +# BEGIN required Windows option +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_MAX_TRIGGER_DEPTH=100 +# END required Windows option + +TCC = $(TCC) $(OPT_FEATURE_FLAGS) + +# Add in any optional parameters specified on the make commane line +# ie. make "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1". +TCC = $(TCC) $(OPTS) + +# libtool compile/link +LTCOMPILE = $(TCC) -Fo$@ +LTLINK = $(TCC) -Fe$@ +LTLIB = lib.exe + +# nawk compatible awk. +NAWK = .\gawk.exe + +# You should not have to change anything below this line +############################################################################### + +# Object files for the SQLite library (non-amalgamation). +# +LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \ + backup.lo bitvec.lo btmutex.lo btree.lo build.lo \ + callback.lo complete.lo ctime.lo date.lo delete.lo \ + expr.lo fault.lo fkey.lo \ + fts3.lo fts3_aux.lo fts3_expr.lo fts3_hash.lo fts3_icu.lo fts3_porter.lo \ + fts3_snippet.lo fts3_tokenizer.lo fts3_tokenizer1.lo fts3_write.lo \ + func.lo global.lo hash.lo \ + icu.lo insert.lo journal.lo legacy.lo loadext.lo \ + main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \ + memjournal.lo \ + mutex.lo mutex_noop.lo mutex_os2.lo mutex_unix.lo mutex_w32.lo \ + notify.lo opcodes.lo os.lo os_os2.lo os_unix.lo os_win.lo \ + pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \ + random.lo resolve.lo rowset.lo rtree.lo select.lo status.lo \ + table.lo tokenize.lo trigger.lo \ + update.lo util.lo vacuum.lo \ + vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbetrace.lo \ + wal.lo walker.lo where.lo utf.lo vtab.lo + +# Object files for the amalgamation. +# +LIBOBJS1 = sqlite3.lo + +# Determine the real value of LIBOBJ based on the 'configure' script +# +!IF $(USE_AMALGAMATION)==0 +LIBOBJ = $(LIBOBJS0) +!ELSE +LIBOBJ = $(LIBOBJS1) +!ENDIF + +# All of the source code files. +# +SRC = \ + $(TOP)\src\alter.c \ + $(TOP)\src\analyze.c \ + $(TOP)\src\attach.c \ + $(TOP)\src\auth.c \ + $(TOP)\src\backup.c \ + $(TOP)\src\bitvec.c \ + $(TOP)\src\btmutex.c \ + $(TOP)\src\btree.c \ + $(TOP)\src\btree.h \ + $(TOP)\src\btreeInt.h \ + $(TOP)\src\build.c \ + $(TOP)\src\callback.c \ + $(TOP)\src\complete.c \ + $(TOP)\src\ctime.c \ + $(TOP)\src\date.c \ + $(TOP)\src\delete.c \ + $(TOP)\src\expr.c \ + $(TOP)\src\fault.c \ + $(TOP)\src\fkey.c \ + $(TOP)\src\func.c \ + $(TOP)\src\global.c \ + $(TOP)\src\hash.c \ + $(TOP)\src\hash.h \ + $(TOP)\src\hwtime.h \ + $(TOP)\src\insert.c \ + $(TOP)\src\journal.c \ + $(TOP)\src\legacy.c \ + $(TOP)\src\loadext.c \ + $(TOP)\src\main.c \ + $(TOP)\src\malloc.c \ + $(TOP)\src\mem0.c \ + $(TOP)\src\mem1.c \ + $(TOP)\src\mem2.c \ + $(TOP)\src\mem3.c \ + $(TOP)\src\mem5.c \ + $(TOP)\src\memjournal.c \ + $(TOP)\src\mutex.c \ + $(TOP)\src\mutex.h \ + $(TOP)\src\mutex_noop.c \ + $(TOP)\src\mutex_os2.c \ + $(TOP)\src\mutex_unix.c \ + $(TOP)\src\mutex_w32.c \ + $(TOP)\src\notify.c \ + $(TOP)\src\os.c \ + $(TOP)\src\os.h \ + $(TOP)\src\os_common.h \ + $(TOP)\src\os_os2.c \ + $(TOP)\src\os_unix.c \ + $(TOP)\src\os_win.c \ + $(TOP)\src\pager.c \ + $(TOP)\src\pager.h \ + $(TOP)\src\parse.y \ + $(TOP)\src\pcache.c \ + $(TOP)\src\pcache.h \ + $(TOP)\src\pcache1.c \ + $(TOP)\src\pragma.c \ + $(TOP)\src\prepare.c \ + $(TOP)\src\printf.c \ + $(TOP)\src\random.c \ + $(TOP)\src\resolve.c \ + $(TOP)\src\rowset.c \ + $(TOP)\src\select.c \ + $(TOP)\src\status.c \ + $(TOP)\src\shell.c \ + $(TOP)\src\sqlite.h.in \ + $(TOP)\src\sqlite3ext.h \ + $(TOP)\src\sqliteInt.h \ + $(TOP)\src\sqliteLimit.h \ + $(TOP)\src\table.c \ + $(TOP)\src\tclsqlite.c \ + $(TOP)\src\tokenize.c \ + $(TOP)\src\trigger.c \ + $(TOP)\src\utf.c \ + $(TOP)\src\update.c \ + $(TOP)\src\util.c \ + $(TOP)\src\vacuum.c \ + $(TOP)\src\vdbe.c \ + $(TOP)\src\vdbe.h \ + $(TOP)\src\vdbeapi.c \ + $(TOP)\src\vdbeaux.c \ + $(TOP)\src\vdbeblob.c \ + $(TOP)\src\vdbemem.c \ + $(TOP)\src\vdbetrace.c \ + $(TOP)\src\vdbeInt.h \ + $(TOP)\src\vtab.c \ + $(TOP)\src\wal.c \ + $(TOP)\src\wal.h \ + $(TOP)\src\walker.c \ + $(TOP)\src\where.c + +# Source code for extensions +# +SRC = $(SRC) \ + $(TOP)\ext\fts1\fts1.c \ + $(TOP)\ext\fts1\fts1.h \ + $(TOP)\ext\fts1\fts1_hash.c \ + $(TOP)\ext\fts1\fts1_hash.h \ + $(TOP)\ext\fts1\fts1_porter.c \ + $(TOP)\ext\fts1\fts1_tokenizer.h \ + $(TOP)\ext\fts1\fts1_tokenizer1.c +SRC = $(SRC) \ + $(TOP)\ext\fts2\fts2.c \ + $(TOP)\ext\fts2\fts2.h \ + $(TOP)\ext\fts2\fts2_hash.c \ + $(TOP)\ext\fts2\fts2_hash.h \ + $(TOP)\ext\fts2\fts2_icu.c \ + $(TOP)\ext\fts2\fts2_porter.c \ + $(TOP)\ext\fts2\fts2_tokenizer.h \ + $(TOP)\ext\fts2\fts2_tokenizer.c \ + $(TOP)\ext\fts2\fts2_tokenizer1.c +SRC = $(SRC) \ + $(TOP)\ext\fts3\fts3.c \ + $(TOP)\ext\fts3\fts3.h \ + $(TOP)\ext\fts3\fts3Int.h \ + $(TOP)\ext\fts3\fts3_aux.c \ + $(TOP)\ext\fts3\fts3_expr.c \ + $(TOP)\ext\fts3\fts3_hash.c \ + $(TOP)\ext\fts3\fts3_hash.h \ + $(TOP)\ext\fts3\fts3_icu.c \ + $(TOP)\ext\fts3\fts3_porter.c \ + $(TOP)\ext\fts3\fts3_snippet.c \ + $(TOP)\ext\fts3\fts3_tokenizer.h \ + $(TOP)\ext\fts3\fts3_tokenizer.c \ + $(TOP)\ext\fts3\fts3_tokenizer1.c \ + $(TOP)\ext\fts3\fts3_write.c +SRC = $(SRC) \ + $(TOP)\ext\icu\sqliteicu.h \ + $(TOP)\ext\icu\icu.c +SRC = $(SRC) \ + $(TOP)\ext\rtree\rtree.h \ + $(TOP)\ext\rtree\rtree.c + + +# Generated source code files +# +SRC = $(SRC) \ + keywordhash.h \ + opcodes.c \ + opcodes.h \ + parse.c \ + parse.h \ + sqlite3.h + +# Source code to the test files. +# +TESTSRC = \ + $(TOP)\src\test1.c \ + $(TOP)\src\test2.c \ + $(TOP)\src\test3.c \ + $(TOP)\src\test4.c \ + $(TOP)\src\test5.c \ + $(TOP)\src\test6.c \ + $(TOP)\src\test7.c \ + $(TOP)\src\test8.c \ + $(TOP)\src\test9.c \ + $(TOP)\src\test_autoext.c \ + $(TOP)\src\test_async.c \ + $(TOP)\src\test_backup.c \ + $(TOP)\src\test_btree.c \ + $(TOP)\src\test_config.c \ + $(TOP)\src\test_demovfs.c \ + $(TOP)\src\test_devsym.c \ + $(TOP)\src\test_func.c \ + $(TOP)\src\test_fuzzer.c \ + $(TOP)\src\test_hexio.c \ + $(TOP)\src\test_init.c \ + $(TOP)\src\test_intarray.c \ + $(TOP)\src\test_journal.c \ + $(TOP)\src\test_malloc.c \ + $(TOP)\src\test_multiplex.c \ + $(TOP)\src\test_mutex.c \ + $(TOP)\src\test_onefile.c \ + $(TOP)\src\test_osinst.c \ + $(TOP)\src\test_pcache.c \ + $(TOP)\src\test_quota.c \ + $(TOP)\src\test_rtree.c \ + $(TOP)\src\test_schema.c \ + $(TOP)\src\test_server.c \ + $(TOP)\src\test_superlock.c \ + $(TOP)\src\test_syscall.c \ + $(TOP)\src\test_stat.c \ + $(TOP)\src\test_tclvar.c \ + $(TOP)\src\test_thread.c \ + $(TOP)\src\test_vfs.c \ + $(TOP)\src\test_wholenumber.c \ + $(TOP)\src\test_wsd.c \ + $(TOP)\ext\fts3\fts3_term.c \ + $(TOP)\ext\fts3\fts3_test.c + +# Source code to the library files needed by the test fixture +# +TESTSRC2 = \ + $(TOP)\src\attach.c \ + $(TOP)\src\backup.c \ + $(TOP)\src\bitvec.c \ + $(TOP)\src\btree.c \ + $(TOP)\src\build.c \ + $(TOP)\src\ctime.c \ + $(TOP)\src\date.c \ + $(TOP)\src\expr.c \ + $(TOP)\src\func.c \ + $(TOP)\src\insert.c \ + $(TOP)\src\wal.c \ + $(TOP)\src\mem5.c \ + $(TOP)\src\os.c \ + $(TOP)\src\os_os2.c \ + $(TOP)\src\os_unix.c \ + $(TOP)\src\os_win.c \ + $(TOP)\src\pager.c \ + $(TOP)\src\pragma.c \ + $(TOP)\src\prepare.c \ + $(TOP)\src\printf.c \ + $(TOP)\src\random.c \ + $(TOP)\src\pcache.c \ + $(TOP)\src\pcache1.c \ + $(TOP)\src\select.c \ + $(TOP)\src\tokenize.c \ + $(TOP)\src\utf.c \ + $(TOP)\src\util.c \ + $(TOP)\src\vdbeapi.c \ + $(TOP)\src\vdbeaux.c \ + $(TOP)\src\vdbe.c \ + $(TOP)\src\vdbemem.c \ + $(TOP)\src\vdbetrace.c \ + $(TOP)\src\where.c \ + parse.c \ + $(TOP)\ext\fts3\fts3.c \ + $(TOP)\ext\fts3\fts3_aux.c \ + $(TOP)\ext\fts3\fts3_expr.c \ + $(TOP)\ext\fts3\fts3_tokenizer.c \ + $(TOP)\ext\fts3\fts3_write.c \ + $(TOP)\ext\async\sqlite3async.c + +# Header files used by all library source files. +# +HDR = \ + $(TOP)\src\btree.h \ + $(TOP)\src\btreeInt.h \ + $(TOP)\src\hash.h \ + $(TOP)\src\hwtime.h \ + keywordhash.h \ + $(TOP)\src\mutex.h \ + opcodes.h \ + $(TOP)\src\os.h \ + $(TOP)\src\os_common.h \ + $(TOP)\src\pager.h \ + $(TOP)\src\pcache.h \ + parse.h \ + sqlite3.h \ + $(TOP)\src\sqlite3ext.h \ + $(TOP)\src\sqliteInt.h \ + $(TOP)\src\sqliteLimit.h \ + $(TOP)\src\vdbe.h \ + $(TOP)\src\vdbeInt.h + +# Header files used by extensions +# +EXTHDR = $(EXTHDR) \ + $(TOP)\ext\fts1\fts1.h \ + $(TOP)\ext\fts1\fts1_hash.h \ + $(TOP)\ext\fts1\fts1_tokenizer.h +EXTHDR = $(EXTHDR) \ + $(TOP)\ext\fts2\fts2.h \ + $(TOP)\ext\fts2\fts2_hash.h \ + $(TOP)\ext\fts2\fts2_tokenizer.h +EXTHDR = $(EXTHDR) \ + $(TOP)\ext\fts3\fts3.h \ + $(TOP)\ext\fts3\fts3Int.h \ + $(TOP)\ext\fts3\fts3_hash.h \ + $(TOP)\ext\fts3\fts3_tokenizer.h +EXTHDR = $(EXTHDR) \ + $(TOP)\ext\rtree\rtree.h +EXTHDR = $(EXTHDR) \ + $(TOP)\ext\icu\sqliteicu.h +EXTHDR = $(EXTHDR) \ + $(TOP)\ext\rtree\sqlite3rtree.h + +# This is the default Makefile target. The objects listed here +# are what get build when you type just "make" with no arguments. +# +all: libsqlite3.lib sqlite3.exe libtclsqlite3.lib + +libsqlite3.lib: $(LIBOBJ) + $(LTLIB) -OUT:$@ $(LIBOBJ) $(TLIBS) + +libtclsqlite3.lib: tclsqlite.lo libsqlite3.lib + $(LTLIB) /LIBPATH:$(TCLLIBDIR) -OUT:$@ tclsqlite.lo libsqlite3.lib $(LIBTCL:tcl=tclstub) $(TLIBS) + +sqlite3.exe: $(TOP)\src\shell.c libsqlite3.lib sqlite3.h + $(LTLINK) $(READLINE_FLAGS) \ + $(TOP)\src\shell.c libsqlite3.lib \ + $(LIBREADLINE) $(TLIBS) + +# This target creates a directory named "tsrc" and fills it with +# copies of all of the C source code and header files needed to +# build on the target system. Some of the C source code and header +# files are automatically generated. This target takes care of +# all that automatic generation. +# +.target_source: $(SRC) $(TOP)\tool\vdbe-compress.tcl + -rmdir /S/Q tsrc + -mkdir tsrc + for %i in ($(SRC)) do copy /Y %i tsrc + del /Q tsrc\sqlite.h.in tsrc\parse.y + $(TCLSH_CMD) $(TOP)\tool\vdbe-compress.tcl vdbe.new + move vdbe.new tsrc\vdbe.c + echo > .target_source + +sqlite3.c: .target_source $(TOP)\tool\mksqlite3c.tcl + $(TCLSH_CMD) $(TOP)\tool\mksqlite3c.tcl + +# Rule to build the amalgamation +# +sqlite3.lo: sqlite3.c + $(LTCOMPILE) -c sqlite3.c + +# Rules to build the LEMON compiler generator +# +lempar.c: $(TOP)\src\lempar.c + copy $(TOP)\src\lempar.c . + +lemon.exe: $(TOP)\tool\lemon.c lempar.c + $(BCC) -Fe$@ $(TOP)\tool\lemon.c + +# Rules to build individual *.lo files from generated *.c files. This +# applies to: +# +# parse.lo +# opcodes.lo +# +parse.lo: parse.c $(HDR) + $(LTCOMPILE) -c parse.c + +opcodes.lo: opcodes.c + $(LTCOMPILE) -c opcodes.c + +# Rules to build individual *.lo files from files in the src directory. +# +alter.lo: $(TOP)\src\alter.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\alter.c + +analyze.lo: $(TOP)\src\analyze.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\analyze.c + +attach.lo: $(TOP)\src\attach.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\attach.c + +auth.lo: $(TOP)\src\auth.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\auth.c + +backup.lo: $(TOP)\src\backup.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\backup.c + +bitvec.lo: $(TOP)\src\bitvec.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\bitvec.c + +btmutex.lo: $(TOP)\src\btmutex.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\btmutex.c + +btree.lo: $(TOP)\src\btree.c $(HDR) $(TOP)\src\pager.h + $(LTCOMPILE) -c $(TOP)\src\btree.c + +build.lo: $(TOP)\src\build.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\build.c + +callback.lo: $(TOP)\src\callback.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\callback.c + +complete.lo: $(TOP)\src\complete.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\complete.c + +ctime.lo: $(TOP)\src\ctime.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\ctime.c + +date.lo: $(TOP)\src\date.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\date.c + +delete.lo: $(TOP)\src\delete.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\delete.c + +expr.lo: $(TOP)\src\expr.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\expr.c + +fault.lo: $(TOP)\src\fault.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\fault.c + +fkey.lo: $(TOP)\src\fkey.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\fkey.c + +func.lo: $(TOP)\src\func.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\func.c + +global.lo: $(TOP)\src\global.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\global.c + +hash.lo: $(TOP)\src\hash.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\hash.c + +insert.lo: $(TOP)\src\insert.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\insert.c + +journal.lo: $(TOP)\src\journal.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\journal.c + +legacy.lo: $(TOP)\src\legacy.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\legacy.c + +loadext.lo: $(TOP)\src\loadext.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\loadext.c + +main.lo: $(TOP)\src\main.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\main.c + +malloc.lo: $(TOP)\src\malloc.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\malloc.c + +mem0.lo: $(TOP)\src\mem0.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\mem0.c + +mem1.lo: $(TOP)\src\mem1.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\mem1.c + +mem2.lo: $(TOP)\src\mem2.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\mem2.c + +mem3.lo: $(TOP)\src\mem3.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\mem3.c + +mem5.lo: $(TOP)\src\mem5.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\mem5.c + +memjournal.lo: $(TOP)\src\memjournal.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\memjournal.c + +mutex.lo: $(TOP)\src\mutex.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\mutex.c + +mutex_noop.lo: $(TOP)\src\mutex_noop.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\mutex_noop.c + +mutex_os2.lo: $(TOP)\src\mutex_os2.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\mutex_os2.c + +mutex_unix.lo: $(TOP)\src\mutex_unix.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\mutex_unix.c + +mutex_w32.lo: $(TOP)\src\mutex_w32.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\mutex_w32.c + +notify.lo: $(TOP)\src\notify.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\notify.c + +pager.lo: $(TOP)\src\pager.c $(HDR) $(TOP)\src\pager.h + $(LTCOMPILE) -c $(TOP)\src\pager.c + +pcache.lo: $(TOP)\src\pcache.c $(HDR) $(TOP)\src\pcache.h + $(LTCOMPILE) -c $(TOP)\src\pcache.c + +pcache1.lo: $(TOP)\src\pcache1.c $(HDR) $(TOP)\src\pcache.h + $(LTCOMPILE) -c $(TOP)\src\pcache1.c + +os.lo: $(TOP)\src\os.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\os.c + +os_unix.lo: $(TOP)\src\os_unix.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\os_unix.c + +os_win.lo: $(TOP)\src\os_win.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\os_win.c + +os_os2.lo: $(TOP)\src\os_os2.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\os_os2.c + +pragma.lo: $(TOP)\src\pragma.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\pragma.c + +prepare.lo: $(TOP)\src\prepare.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\prepare.c + +printf.lo: $(TOP)\src\printf.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\printf.c + +random.lo: $(TOP)\src\random.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\random.c + +resolve.lo: $(TOP)\src\resolve.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\resolve.c + +rowset.lo: $(TOP)\src\rowset.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\rowset.c + +select.lo: $(TOP)\src\select.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\select.c + +status.lo: $(TOP)\src\status.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\status.c + +table.lo: $(TOP)\src\table.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\table.c + +tokenize.lo: $(TOP)\src\tokenize.c keywordhash.h $(HDR) + $(LTCOMPILE) -c $(TOP)\src\tokenize.c + +trigger.lo: $(TOP)\src\trigger.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\trigger.c + +update.lo: $(TOP)\src\update.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\update.c + +utf.lo: $(TOP)\src\utf.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\utf.c + +util.lo: $(TOP)\src\util.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\util.c + +vacuum.lo: $(TOP)\src\vacuum.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\vacuum.c + +vdbe.lo: $(TOP)\src\vdbe.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\vdbe.c + +vdbeapi.lo: $(TOP)\src\vdbeapi.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\vdbeapi.c + +vdbeaux.lo: $(TOP)\src\vdbeaux.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\vdbeaux.c + +vdbeblob.lo: $(TOP)\src\vdbeblob.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\vdbeblob.c + +vdbemem.lo: $(TOP)\src\vdbemem.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\vdbemem.c + +vdbetrace.lo: $(TOP)\src\vdbetrace.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\vdbetrace.c + +vtab.lo: $(TOP)\src\vtab.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\vtab.c + +wal.lo: $(TOP)\src\wal.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\wal.c + +walker.lo: $(TOP)\src\walker.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\walker.c + +where.lo: $(TOP)\src\where.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\where.c + +tclsqlite.lo: $(TOP)\src\tclsqlite.c $(HDR) + $(LTCOMPILE) -DUSE_TCL_STUBS=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c + +tclsqlite-shell.lo: $(TOP)\src\tclsqlite.c $(HDR) + $(LTCOMPILE) -DTCLSH=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c + +tclsqlite3.exe: tclsqlite-shell.lo libsqlite3.lib + $(LTLINK) tclsqlite-shell.lo \ + /link /LIBPATH:$(TCLLIBDIR) libsqlite3.lib $(LIBTCL) + +# Rules to build opcodes.c and opcodes.h +# +opcodes.c: opcodes.h $(TOP)\mkopcodec.awk + $(NAWK) "/#define OP_/ { print }" opcodes.h | sort /+45 | $(NAWK) -f $(TOP)\mkopcodec.awk >opcodes.c + +opcodes.h: parse.h $(TOP)\src\vdbe.c $(TOP)\mkopcodeh.awk + type parse.h $(TOP)\src\vdbe.c | $(NAWK) -f $(TOP)\mkopcodeh.awk >opcodes.h + +# Rules to build parse.c and parse.h - the outputs of lemon. +# +parse.h: parse.c + +parse.c: $(TOP)\src\parse.y lemon.exe $(TOP)\addopcodes.awk + del /Q parse.y parse.h parse.h.temp + copy $(TOP)\src\parse.y . + .\lemon.exe $(OPT_FEATURE_FLAGS) $(OPTS) parse.y + move parse.h parse.h.temp + $(NAWK) -f $(TOP)\addopcodes.awk parse.h.temp >parse.h + +sqlite3.h: $(TOP)\src\sqlite.h.in $(TOP)\manifest.uuid $(TOP)\VERSION + $(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP) >sqlite3.h + +mkkeywordhash.exe: $(TOP)\tool\mkkeywordhash.c + $(BCC) -Femkkeywordhash.exe $(OPT_FEATURE_FLAGS) $(OPTS) $(TOP)\tool\mkkeywordhash.c + +keywordhash.h: $(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe + .\mkkeywordhash.exe >keywordhash.h + + + +# Rules to build the extension objects. +# +icu.lo: $(TOP)\ext\icu\icu.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\icu\icu.c + +fts2.lo: $(TOP)\ext\fts2\fts2.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts2\fts2.c + +fts2_hash.lo: $(TOP)\ext\fts2\fts2_hash.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts2\fts2_hash.c + +fts2_icu.lo: $(TOP)\ext\fts2\fts2_icu.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts2\fts2_icu.c + +fts2_porter.lo: $(TOP)\ext\fts2\fts2_porter.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts2\fts2_porter.c + +fts2_tokenizer.lo: $(TOP)\ext\fts2\fts2_tokenizer.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts2\fts2_tokenizer.c + +fts2_tokenizer1.lo: $(TOP)\ext\fts2\fts2_tokenizer1.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts2\fts2_tokenizer1.c + +fts3.lo: $(TOP)\ext\fts3\fts3.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3.c + +fts3_aux.lo: $(TOP)\ext\fts3\fts3_aux.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_aux.c + +fts3_expr.lo: $(TOP)\ext\fts3\fts3_expr.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_expr.c + +fts3_hash.lo: $(TOP)\ext\fts3\fts3_hash.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_hash.c + +fts3_icu.lo: $(TOP)\ext\fts3\fts3_icu.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_icu.c + +fts3_snippet.lo: $(TOP)\ext\fts3\fts3_snippet.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_snippet.c + +fts3_porter.lo: $(TOP)\ext\fts3\fts3_porter.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_porter.c + +fts3_tokenizer.lo: $(TOP)\ext\fts3\fts3_tokenizer.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_tokenizer.c + +fts3_tokenizer1.lo: $(TOP)\ext\fts3\fts3_tokenizer1.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_tokenizer1.c + +fts3_write.lo: $(TOP)\ext\fts3\fts3_write.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_write.c + +rtree.lo: $(TOP)\ext\rtree\rtree.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\rtree\rtree.c + + +# Rules to build the 'testfixture' application. +# +# If using the amalgamation, use sqlite3.c directly to build the test +# fixture. Otherwise link against libsqlite3.lib. (This distinction is +# necessary because the test fixture requires non-API symbols which are +# hidden when the library is built via the amalgamation). +# +TESTFIXTURE_FLAGS = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 +TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE + +TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.lib +TESTFIXTURE_SRC1 = sqlite3.c +!IF $(USE_AMALGAMATION)==0 +TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC0) +!ELSE +TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC1) +!ENDIF + +testfixture.exe: $(TESTFIXTURE_SRC) $(HDR) + $(LTLINK) -DSQLITE_NO_SYNC=1 $(TESTFIXTURE_FLAGS) \ + -DBUILD_sqlite -I$(TCLINCDIR) \ + $(TESTFIXTURE_SRC) /link /LIBPATH:$(TCLLIBDIR) $(LIBTCL) $(TLIBS) + +fulltest: testfixture.exe sqlite3.exe + .\testfixture.exe $(TOP)\test\all.test + +soaktest: testfixture.exe sqlite3.exe + .\testfixture.exe $(TOP)\test\all.test -soak=1 + +test: testfixture.exe sqlite3.exe + .\testfixture.exe $(TOP)\test\veryquick.test + +spaceanal_tcl.h: $(TOP)\tool\spaceanal.tcl + $(NAWK) "/^[^#]/ { gsub(/\\/,\"\\\\\\\\\");gsub(/\\\"/,\"\\\\\\\"\");gsub(/^/,\"\\\"\");gsub(/$$/,\"\\n\\\"\");print }" \ + $(TOP)\tool\spaceanal.tcl >spaceanal_tcl.h + +sqlite3_analyzer.exe: $(TESTFIXTURE_SRC) spaceanal_tcl.h + $(LTLINK) -DTCLSH=2 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 \ + -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE \ + -DBUILD_sqlite -I$(TCLINCDIR) \ + $(TESTFIXTURE_SRC) /link /LIBPATH:$(TCLLIBDIR) $(LIBTCL) $(TLIBS) + +clean: + del /Q *.lo *.lib *.obj sqlite3.exe libsqlite3.lib + del /Q sqlite3.h opcodes.c opcodes.h + del /Q lemon.exe lempar.c parse.* + del /Q mkkeywordhash.exe keywordhash.h + -rmdir /Q/S tsrc + del /Q .target_source + del /Q testfixture.exe testfixture.exp test.db + del /Q sqlite3.dll sqlite3.lib sqlite3.exp sqlite3.def + del /Q sqlite3.c + del /Q sqlite3_analyzer.exe sqlite3_analyzer.exp spaceanal_tcl.h + +# +# Windows section +# +dll: sqlite3.dll + +sqlite3.def: libsqlite3.lib + echo EXPORTS >sqlite3.def + dumpbin /all libsqlite3.lib \ + | $(NAWK) "/ 1 _sqlite3_/ { sub(/^.* _/,\"\");print }" \ + | sort >>sqlite3.def + +sqlite3.dll: $(LIBOBJ) sqlite3.def + link /DLL /OUT:$@ /DEF:sqlite3.def $(LIBOBJ) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 0aba0546c3..b67cb8c5a1 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -292,14 +292,13 @@ ** into a single segment. */ +#include "fts3Int.h" #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) #if defined(SQLITE_ENABLE_FTS3) && !defined(SQLITE_CORE) # define SQLITE_CORE 1 #endif -#include "fts3Int.h" - #include #include #include @@ -423,12 +422,12 @@ static void fts3GetDeltaVarint(char **pp, sqlite3_int64 *pVal){ ** When this function is called, *pp points to the first byte following a ** varint that is part of a doclist (or position-list, or any other list ** of varints). This function moves *pp to point to the start of that varint, -** and decrements the value stored in *pVal by the varint value. +** and sets *pVal by the varint value. ** ** Argument pStart points to the first byte of the doclist that the ** varint is part of. */ -static void fts3GetReverseDeltaVarint( +static void fts3GetReverseVarint( char **pp, char *pStart, sqlite3_int64 *pVal @@ -444,21 +443,7 @@ static void fts3GetReverseDeltaVarint( *pp = p; sqlite3Fts3GetVarint(p, &iVal); - *pVal -= iVal; -} - -/* -** As long as *pp has not reached its end (pEnd), then do the same -** as fts3GetDeltaVarint(): read a single varint and add it to *pVal. -** But if we have reached the end of the varint, just set *pp=0 and -** leave *pVal unchanged. -*/ -static void fts3GetDeltaVarint2(char **pp, char *pEnd, sqlite3_int64 *pVal){ - if( *pp>=pEnd ){ - *pp = 0; - }else{ - fts3GetDeltaVarint(pp, pVal); - } + *pVal = iVal; } /* @@ -832,6 +817,58 @@ static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){ return zRet; } +static int fts3GobbleInt(const char **pp, int *pnOut){ + const char *p = *pp; + int nInt = 0; + for(p=*pp; p[0]>='0' && p[0]<='9'; p++){ + nInt = nInt * 10 + (p[0] - '0'); + } + if( p==*pp ) return SQLITE_ERROR; + *pnOut = nInt; + *pp = p; + return SQLITE_OK; +} + + +static int fts3PrefixParameter( + const char *zParam, /* ABC in prefix=ABC parameter to parse */ + int *pnIndex, /* OUT: size of *apIndex[] array */ + struct Fts3Index **apIndex, /* OUT: Array of indexes for this table */ + struct Fts3Index **apFree /* OUT: Free this with sqlite3_free() */ +){ + struct Fts3Index *aIndex; + int nIndex = 1; + + if( zParam && zParam[0] ){ + const char *p; + nIndex++; + for(p=zParam; *p; p++){ + if( *p==',' ) nIndex++; + } + } + + aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex); + *apIndex = *apFree = aIndex; + *pnIndex = nIndex; + if( !aIndex ){ + return SQLITE_NOMEM; + } + + memset(aIndex, 0, sizeof(struct Fts3Index) * nIndex); + if( zParam ){ + const char *p = zParam; + int i; + for(i=1; i MATCHINFO */ + { "prefix", 6, 0 }, /* 1 -> PREFIX */ + { "compress", 8, 0 }, /* 2 -> COMPRESS */ + { "uncompress", 10, 0 }, /* 3 -> UNCOMPRESS */ + { "order", 5, 0 } /* 4 -> ORDER */ + }; + + int iOpt; if( !zVal ){ rc = SQLITE_NOMEM; - goto fts3_init_out; - } - if( nKey==9 && 0==sqlite3_strnicmp(z, "matchinfo", 9) ){ - if( strlen(zVal)==4 && 0==sqlite3_strnicmp(zVal, "fts3", 4) ){ - bNoDocsize = 1; - }else{ - *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal); - rc = SQLITE_ERROR; - } - }else if( nKey==8 && 0==sqlite3_strnicmp(z, "compress", 8) ){ - zCompress = zVal; - zVal = 0; - }else if( nKey==10 && 0==sqlite3_strnicmp(z, "uncompress", 10) ){ - zUncompress = zVal; - zVal = 0; }else{ - *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z); - rc = SQLITE_ERROR; + for(iOpt=0; iOptnOpt && !sqlite3_strnicmp(z, pOp->zOpt, pOp->nOpt) ){ + break; + } + } + if( iOpt==SizeofArray(aFts4Opt) ){ + *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z); + rc = SQLITE_ERROR; + }else{ + switch( iOpt ){ + case 0: /* MATCHINFO */ + if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){ + *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal); + rc = SQLITE_ERROR; + } + bNoDocsize = 1; + break; + + case 1: /* PREFIX */ + sqlite3_free(zPrefix); + zPrefix = zVal; + zVal = 0; + break; + + case 2: /* COMPRESS */ + sqlite3_free(zCompress); + zCompress = zVal; + zVal = 0; + break; + + case 3: /* UNCOMPRESS */ + sqlite3_free(zUncompress); + zUncompress = zVal; + zVal = 0; + break; + + case 4: /* ORDER */ + if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) + && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 3)) + ){ + *pzErr = sqlite3_mprintf("unrecognized order: %s", zVal); + rc = SQLITE_ERROR; + } + bDescIdx = (zVal[0]=='d' || zVal[0]=='D'); + break; + } + } + sqlite3_free(zVal); } - sqlite3_free(zVal); } /* Otherwise, the argument is a column name. */ @@ -955,10 +1043,17 @@ static int fts3InitVtab( } assert( pTokenizer ); + rc = fts3PrefixParameter(zPrefix, &nIndex, &aIndex, &aFree); + if( rc==SQLITE_ERROR ){ + assert( zPrefix ); + *pzErr = sqlite3_mprintf("error parsing prefix parameter: %s", zPrefix); + } + if( rc!=SQLITE_OK ) goto fts3_init_out; /* Allocate and populate the Fts3Table structure. */ - nByte = sizeof(Fts3Table) + /* Fts3Table */ + nByte = sizeof(Fts3Table) + /* Fts3Table */ nCol * sizeof(char *) + /* azColumn */ + nIndex * sizeof(struct Fts3Index) + /* aIndex */ nName + /* zName */ nDb + /* zDb */ nString; /* Space for azColumn strings */ @@ -973,16 +1068,22 @@ static int fts3InitVtab( p->nPendingData = 0; p->azColumn = (char **)&p[1]; p->pTokenizer = pTokenizer; - p->nNodeSize = 1000; p->nMaxPendingData = FTS3_MAX_PENDING_DATA; p->bHasDocsize = (isFts4 && bNoDocsize==0); p->bHasStat = isFts4; + p->bDescIdx = bDescIdx; TESTONLY( p->inTransaction = -1 ); TESTONLY( p->mxSavepoint = -1 ); - fts3HashInit(&p->pendingTerms, FTS3_HASH_STRING, 1); + + p->aIndex = (struct Fts3Index *)&p->azColumn[nCol]; + memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex); + p->nIndex = nIndex; + for(i=0; iaIndex[i].hPending, FTS3_HASH_STRING, 1); + } /* Fill in the zName and zDb fields of the vtab structure. */ - zCsr = (char *)&p->azColumn[nCol]; + zCsr = (char *)&p->aIndex[nIndex]; p->zName = zCsr; memcpy(zCsr, argv[2], nName); zCsr += nName; @@ -1020,15 +1121,16 @@ static int fts3InitVtab( } /* Figure out the page-size for the database. This is required in order to - ** estimate the cost of loading large doclists from the database (see - ** function sqlite3Fts3SegReaderCost() for details). - */ + ** estimate the cost of loading large doclists from the database. */ fts3DatabasePageSize(&rc, p); + p->nNodeSize = p->nPgsz-35; /* Declare the table schema to SQLite. */ fts3DeclareVtab(&rc, p); fts3_init_out: + sqlite3_free(zPrefix); + sqlite3_free(aFree); sqlite3_free(zCompress); sqlite3_free(zUncompress); sqlite3_free((void *)aCol); @@ -1039,6 +1141,7 @@ fts3_init_out: pTokenizer->pModule->xDestroy(pTokenizer); } }else{ + assert( p->pSegments==0 ); *ppVTab = &p->base; } return rc; @@ -1136,10 +1239,11 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ }else{ pInfo->idxStr = "ASC"; } + pInfo->orderByConsumed = 1; } - pInfo->orderByConsumed = 1; } + assert( p->pSegments==0 ); return SQLITE_OK; } @@ -1175,6 +1279,7 @@ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ sqlite3Fts3FreeDeferredTokens(pCsr); sqlite3_free(pCsr->aDoclist); sqlite3_free(pCsr->aMatchinfo); + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); sqlite3_free(pCsr); return SQLITE_OK; } @@ -1186,8 +1291,8 @@ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ */ static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ if( pCsr->isRequireSeek ){ - pCsr->isRequireSeek = 0; sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId); + pCsr->isRequireSeek = 0; if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){ return SQLITE_OK; }else{ @@ -1368,7 +1473,7 @@ static int fts3SelectLeaf( int nBlob; /* Size of zBlob in bytes */ if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){ - rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob); + rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob, 0); if( rc==SQLITE_OK ){ rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0); } @@ -1378,7 +1483,7 @@ static int fts3SelectLeaf( } if( rc==SQLITE_OK ){ - rc = sqlite3Fts3ReadBlock(p, piLeaf ? *piLeaf : *piLeaf2, &zBlob, &nBlob); + rc = sqlite3Fts3ReadBlock(p, piLeaf?*piLeaf:*piLeaf2, &zBlob, &nBlob, 0); } if( rc==SQLITE_OK ){ rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2); @@ -1754,7 +1859,19 @@ static int fts3PoslistPhraseMerge( } /* -** Merge two position-lists as required by the NEAR operator. +** Merge two position-lists as required by the NEAR operator. The argument +** position lists correspond to the left and right phrases of an expression +** like: +** +** "phrase 1" NEAR "phrase number 2" +** +** Position list *pp1 corresponds to the left-hand side of the NEAR +** expression and *pp2 to the right. As usual, the indexes in the position +** lists are the offsets of the last token in each phrase (tokens "1" and "2" +** in the example above). +** +** The output position list - written to *pp - is a copy of *pp2 with those +** entries that are not sufficiently NEAR entries in *pp1 removed. */ static int fts3PoslistNearMerge( char **pp, /* Output buffer */ @@ -1767,214 +1884,27 @@ static int fts3PoslistNearMerge( char *p1 = *pp1; char *p2 = *pp2; - if( !pp ){ - if( fts3PoslistPhraseMerge(0, nRight, 0, 0, pp1, pp2) ) return 1; - *pp1 = p1; - *pp2 = p2; - return fts3PoslistPhraseMerge(0, nLeft, 0, 0, pp2, pp1); + char *pTmp1 = aTmp; + char *pTmp2; + char *aTmp2; + int res = 1; + + fts3PoslistPhraseMerge(&pTmp1, nRight, 0, 0, pp1, pp2); + aTmp2 = pTmp2 = pTmp1; + *pp1 = p1; + *pp2 = p2; + fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, 0, pp2, pp1); + if( pTmp1!=aTmp && pTmp2!=aTmp2 ){ + fts3PoslistMerge(pp, &aTmp, &aTmp2); + }else if( pTmp1!=aTmp ){ + fts3PoslistCopy(pp, &aTmp); + }else if( pTmp2!=aTmp2 ){ + fts3PoslistCopy(pp, &aTmp2); }else{ - char *pTmp1 = aTmp; - char *pTmp2; - char *aTmp2; - int res = 1; - - fts3PoslistPhraseMerge(&pTmp1, nRight, 0, 0, pp1, pp2); - aTmp2 = pTmp2 = pTmp1; - *pp1 = p1; - *pp2 = p2; - fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, 0, pp2, pp1); - if( pTmp1!=aTmp && pTmp2!=aTmp2 ){ - fts3PoslistMerge(pp, &aTmp, &aTmp2); - }else if( pTmp1!=aTmp ){ - fts3PoslistCopy(pp, &aTmp); - }else if( pTmp2!=aTmp2 ){ - fts3PoslistCopy(pp, &aTmp2); - }else{ - res = 0; - } - - return res; - } -} - -/* -** Values that may be used as the first parameter to fts3DoclistMerge(). -*/ -#define MERGE_NOT 2 /* D + D -> D */ -#define MERGE_AND 3 /* D + D -> D */ -#define MERGE_OR 4 /* D + D -> D */ -#define MERGE_POS_OR 5 /* P + P -> P */ -#define MERGE_PHRASE 6 /* P + P -> D */ -#define MERGE_POS_PHRASE 7 /* P + P -> P */ -#define MERGE_NEAR 8 /* P + P -> D */ -#define MERGE_POS_NEAR 9 /* P + P -> P */ - -/* -** Merge the two doclists passed in buffer a1 (size n1 bytes) and a2 -** (size n2 bytes). The output is written to pre-allocated buffer aBuffer, -** which is guaranteed to be large enough to hold the results. The number -** of bytes written to aBuffer is stored in *pnBuffer before returning. -** -** If successful, SQLITE_OK is returned. Otherwise, if a malloc error -** occurs while allocating a temporary buffer as part of the merge operation, -** SQLITE_NOMEM is returned. -*/ -static int fts3DoclistMerge( - int mergetype, /* One of the MERGE_XXX constants */ - int nParam1, /* Used by MERGE_NEAR and MERGE_POS_NEAR */ - int nParam2, /* Used by MERGE_NEAR and MERGE_POS_NEAR */ - char *aBuffer, /* Pre-allocated output buffer */ - int *pnBuffer, /* OUT: Bytes written to aBuffer */ - char *a1, /* Buffer containing first doclist */ - int n1, /* Size of buffer a1 */ - char *a2, /* Buffer containing second doclist */ - int n2, /* Size of buffer a2 */ - int *pnDoc /* OUT: Number of docids in output */ -){ - sqlite3_int64 i1 = 0; - sqlite3_int64 i2 = 0; - sqlite3_int64 iPrev = 0; - - char *p = aBuffer; - char *p1 = a1; - char *p2 = a2; - char *pEnd1 = &a1[n1]; - char *pEnd2 = &a2[n2]; - int nDoc = 0; - - assert( mergetype==MERGE_OR || mergetype==MERGE_POS_OR - || mergetype==MERGE_AND || mergetype==MERGE_NOT - || mergetype==MERGE_PHRASE || mergetype==MERGE_POS_PHRASE - || mergetype==MERGE_NEAR || mergetype==MERGE_POS_NEAR - ); - - if( !aBuffer ){ - *pnBuffer = 0; - return SQLITE_NOMEM; + res = 0; } - /* Read the first docid from each doclist */ - fts3GetDeltaVarint2(&p1, pEnd1, &i1); - fts3GetDeltaVarint2(&p2, pEnd2, &i2); - - switch( mergetype ){ - case MERGE_OR: - case MERGE_POS_OR: - while( p1 || p2 ){ - if( p2 && p1 && i1==i2 ){ - fts3PutDeltaVarint(&p, &iPrev, i1); - if( mergetype==MERGE_POS_OR ) fts3PoslistMerge(&p, &p1, &p2); - fts3GetDeltaVarint2(&p1, pEnd1, &i1); - fts3GetDeltaVarint2(&p2, pEnd2, &i2); - }else if( !p2 || (p1 && i1=pEnd ){ + *pp = 0; + }else{ + sqlite3_int64 iVal; + *pp += sqlite3Fts3GetVarint(*pp, &iVal); + if( bDescIdx ){ + *pVal -= iVal; + }else{ + *pVal += iVal; + } + } +} + +static void fts3PutDeltaVarint3( + char **pp, /* IN/OUT: Output pointer */ + int bDescIdx, /* True for descending docids */ + sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */ + int *pbFirst, /* IN/OUT: True after first int written */ + sqlite3_int64 iVal /* Write this value to the list */ +){ + sqlite3_int64 iWrite; + if( bDescIdx==0 || *pbFirst==0 ){ + iWrite = iVal - *piPrev; + }else{ + iWrite = *piPrev - iVal; + } + assert( *pbFirst || *piPrev==0 ); + assert( *pbFirst==0 || iWrite>0 ); + *pp += sqlite3Fts3PutVarint(*pp, iWrite); + *piPrev = iVal; + *pbFirst = 1; +} + +#define COMPARE_DOCID(i1, i2) ((bDescIdx?-1:1) * (i1-i2)) + +static int fts3DoclistOrMerge( + int bDescIdx, /* True if arguments are desc */ + char *a1, int n1, /* First doclist */ + char *a2, int n2, /* Second doclist */ + char **paOut, int *pnOut /* OUT: Malloc'd doclist */ +){ + sqlite3_int64 i1 = 0; + sqlite3_int64 i2 = 0; + sqlite3_int64 iPrev = 0; + char *pEnd1 = &a1[n1]; + char *pEnd2 = &a2[n2]; + char *p1 = a1; + char *p2 = a2; + char *p; + char *aOut; + int bFirstOut = 0; + + *paOut = 0; + *pnOut = 0; + aOut = sqlite3_malloc(n1+n2); + if( !aOut ) return SQLITE_NOMEM; + + p = aOut; + fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); + while( p1 || p2 ){ + sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2); + + if( p2 && p1 && iDiff==0 ){ + fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); + fts3PoslistMerge(&p, &p1, &p2); + fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + }else if( !p2 || (p1 && iDiff<0) ){ + fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); + fts3PoslistCopy(&p, &p1); + fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + }else{ + fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i2); + fts3PoslistCopy(&p, &p2); + fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + } + } + + *paOut = aOut; + *pnOut = (p-aOut); + return SQLITE_OK; +} + +static void fts3DoclistPhraseMerge( + int bDescIdx, /* True if arguments are desc */ + int nDist, /* Distance from left to right (1=adjacent) */ + char *aLeft, int nLeft, /* Left doclist */ + char *aRight, int *pnRight /* IN/OUT: Right/output doclist */ +){ + sqlite3_int64 i1 = 0; + sqlite3_int64 i2 = 0; + sqlite3_int64 iPrev = 0; + char *pEnd1 = &aLeft[nLeft]; + char *pEnd2 = &aRight[*pnRight]; + char *p1 = aLeft; + char *p2 = aRight; + char *p; + int bFirstOut = 0; + char *aOut = aRight; + + assert( nDist>0 ); + + p = aOut; + fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); + + while( p1 && p2 ){ + sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2); + if( iDiff==0 ){ + char *pSave = p; + sqlite3_int64 iPrevSave = iPrev; + int bFirstOutSave = bFirstOut; + + fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); + if( 0==fts3PoslistPhraseMerge(&p, nDist, 0, 1, &p1, &p2) ){ + p = pSave; + iPrev = iPrevSave; + bFirstOut = bFirstOutSave; + } + fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + }else if( iDiff<0 ){ + fts3PoslistCopy(0, &p1); + fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + }else{ + fts3PoslistCopy(0, &p2); + fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + } + } + + *pnRight = p - aOut; +} + + /* ** Merge all doclists in the TermSelect.aaOutput[] array into a single ** doclist stored in TermSelect.aaOutput[0]. If successful, delete all @@ -1997,8 +2069,7 @@ struct TermSelect { ** the responsibility of the caller to free any doclists left in the ** TermSelect.aaOutput[] array. */ -static int fts3TermSelectMerge(TermSelect *pTS){ - int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR); +static int fts3TermSelectMerge(Fts3Table *p, TermSelect *pTS){ char *aOut = 0; int nOut = 0; int i; @@ -2013,15 +2084,17 @@ static int fts3TermSelectMerge(TermSelect *pTS){ nOut = pTS->anOutput[i]; pTS->aaOutput[i] = 0; }else{ - int nNew = nOut + pTS->anOutput[i]; - char *aNew = sqlite3_malloc(nNew); - if( !aNew ){ - sqlite3_free(aOut); - return SQLITE_NOMEM; - } - fts3DoclistMerge(mergetype, 0, 0, - aNew, &nNew, pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, 0 + int nNew; + char *aNew; + + int rc = fts3DoclistOrMerge(p->bDescIdx, + pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, &aNew, &nNew ); + if( rc!=SQLITE_OK ){ + sqlite3_free(aOut); + return rc; + } + sqlite3_free(pTS->aaOutput[i]); sqlite3_free(aOut); pTS->aaOutput[i] = 0; @@ -2057,9 +2130,7 @@ static int fts3TermSelectCb( if( pTS->aaOutput[0]==0 ){ /* If this is the first term selected, copy the doclist to the output - ** buffer using memcpy(). TODO: Add a way to transfer control of the - ** aDoclist buffer from the caller so as to avoid the memcpy(). - */ + ** buffer using memcpy(). */ pTS->aaOutput[0] = sqlite3_malloc(nDoclist); pTS->anOutput[0] = nDoclist; if( pTS->aaOutput[0] ){ @@ -2068,126 +2139,100 @@ static int fts3TermSelectCb( return SQLITE_NOMEM; } }else{ - int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR); char *aMerge = aDoclist; int nMerge = nDoclist; int iOut; for(iOut=0; iOutaaOutput); iOut++){ - char *aNew; - int nNew; if( pTS->aaOutput[iOut]==0 ){ assert( iOut>0 ); pTS->aaOutput[iOut] = aMerge; pTS->anOutput[iOut] = nMerge; break; - } + }else{ + char *aNew; + int nNew; - nNew = nMerge + pTS->anOutput[iOut]; - aNew = sqlite3_malloc(nNew); - if( !aNew ){ - if( aMerge!=aDoclist ){ - sqlite3_free(aMerge); + int rc = fts3DoclistOrMerge(p->bDescIdx, aMerge, nMerge, + pTS->aaOutput[iOut], pTS->anOutput[iOut], &aNew, &nNew + ); + if( rc!=SQLITE_OK ){ + if( aMerge!=aDoclist ) sqlite3_free(aMerge); + return rc; } - return SQLITE_NOMEM; - } - fts3DoclistMerge(mergetype, 0, 0, aNew, &nNew, - pTS->aaOutput[iOut], pTS->anOutput[iOut], aMerge, nMerge, 0 - ); - if( iOut>0 ) sqlite3_free(aMerge); - sqlite3_free(pTS->aaOutput[iOut]); - pTS->aaOutput[iOut] = 0; - - aMerge = aNew; - nMerge = nNew; - if( (iOut+1)==SizeofArray(pTS->aaOutput) ){ - pTS->aaOutput[iOut] = aMerge; - pTS->anOutput[iOut] = nMerge; + if( aMerge!=aDoclist ) sqlite3_free(aMerge); + sqlite3_free(pTS->aaOutput[iOut]); + pTS->aaOutput[iOut] = 0; + + aMerge = aNew; + nMerge = nNew; + if( (iOut+1)==SizeofArray(pTS->aaOutput) ){ + pTS->aaOutput[iOut] = aMerge; + pTS->anOutput[iOut] = nMerge; + } } } } return SQLITE_OK; } -static int fts3DeferredTermSelect( - Fts3DeferredToken *pToken, /* Phrase token */ - int isTermPos, /* True to include positions */ - int *pnOut, /* OUT: Size of list */ - char **ppOut /* OUT: Body of list */ +/* +** Append SegReader object pNew to the end of the pCsr->apSegment[] array. +*/ +static int fts3SegReaderCursorAppend( + Fts3MultiSegReader *pCsr, + Fts3SegReader *pNew ){ - char *aSource; - int nSource; - - aSource = sqlite3Fts3DeferredDoclist(pToken, &nSource); - if( !aSource ){ - *pnOut = 0; - *ppOut = 0; - }else if( isTermPos ){ - *ppOut = sqlite3_malloc(nSource); - if( !*ppOut ) return SQLITE_NOMEM; - memcpy(*ppOut, aSource, nSource); - *pnOut = nSource; - }else{ - sqlite3_int64 docid; - *pnOut = sqlite3Fts3GetVarint(aSource, &docid); - *ppOut = sqlite3_malloc(*pnOut); - if( !*ppOut ) return SQLITE_NOMEM; - sqlite3Fts3PutVarint(*ppOut, docid); + if( (pCsr->nSegment%16)==0 ){ + Fts3SegReader **apNew; + int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*); + apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte); + if( !apNew ){ + sqlite3Fts3SegReaderFree(pNew); + return SQLITE_NOMEM; + } + pCsr->apSegment = apNew; } - + pCsr->apSegment[pCsr->nSegment++] = pNew; return SQLITE_OK; } -int sqlite3Fts3SegReaderCursor( +static int fts3SegReaderCursor( Fts3Table *p, /* FTS3 table handle */ + int iIndex, /* Index to search (from 0 to p->nIndex-1) */ int iLevel, /* Level of segments to scan */ const char *zTerm, /* Term to query for */ int nTerm, /* Size of zTerm in bytes */ int isPrefix, /* True for a prefix search */ int isScan, /* True to scan from zTerm to EOF */ - Fts3SegReaderCursor *pCsr /* Cursor object to populate */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ ){ int rc = SQLITE_OK; int rc2; - int iAge = 0; sqlite3_stmt *pStmt = 0; - Fts3SegReader *pPending = 0; - assert( iLevel==FTS3_SEGCURSOR_ALL - || iLevel==FTS3_SEGCURSOR_PENDING - || iLevel>=0 - ); - assert( FTS3_SEGCURSOR_PENDING<0 ); - assert( FTS3_SEGCURSOR_ALL<0 ); - assert( iLevel==FTS3_SEGCURSOR_ALL || (zTerm==0 && isPrefix==1) ); - assert( isPrefix==0 || isScan==0 ); - - - memset(pCsr, 0, sizeof(Fts3SegReaderCursor)); - - /* If iLevel is less than 0, include a seg-reader for the pending-terms. */ - assert( isScan==0 || fts3HashCount(&p->pendingTerms)==0 ); - if( iLevel<0 && isScan==0 ){ - rc = sqlite3Fts3SegReaderPending(p, zTerm, nTerm, isPrefix, &pPending); - if( rc==SQLITE_OK && pPending ){ - int nByte = (sizeof(Fts3SegReader *) * 16); - pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(nByte); - if( pCsr->apSegment==0 ){ - rc = SQLITE_NOMEM; - }else{ - pCsr->apSegment[0] = pPending; - pCsr->nSegment = 1; - pPending = 0; - } + /* If iLevel is less than 0 and this is not a scan, include a seg-reader + ** for the pending-terms. If this is a scan, then this call must be being + ** made by an fts4aux module, not an FTS table. In this case calling + ** Fts3SegReaderPending might segfault, as the data structures used by + ** fts4aux are not completely populated. So it's easiest to filter these + ** calls out here. */ + if( iLevel<0 && p->aIndex ){ + Fts3SegReader *pSeg = 0; + rc = sqlite3Fts3SegReaderPending(p, iIndex, zTerm, nTerm, isPrefix, &pSeg); + if( rc==SQLITE_OK && pSeg ){ + rc = fts3SegReaderCursorAppend(pCsr, pSeg); } } if( iLevel!=FTS3_SEGCURSOR_PENDING ){ if( rc==SQLITE_OK ){ - rc = sqlite3Fts3AllSegdirs(p, iLevel, &pStmt); + rc = sqlite3Fts3AllSegdirs(p, iIndex, iLevel, &pStmt); } + while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ + Fts3SegReader *pSeg = 0; /* Read the values returned by the SELECT into local variables. */ sqlite3_int64 iStartBlock = sqlite3_column_int64(pStmt, 1); @@ -2196,18 +2241,6 @@ int sqlite3Fts3SegReaderCursor( int nRoot = sqlite3_column_bytes(pStmt, 4); char const *zRoot = sqlite3_column_blob(pStmt, 4); - /* If nSegment is a multiple of 16 the array needs to be extended. */ - if( (pCsr->nSegment%16)==0 ){ - Fts3SegReader **apNew; - int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*); - apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte); - if( !apNew ){ - rc = SQLITE_NOMEM; - goto finished; - } - pCsr->apSegment = apNew; - } - /* If zTerm is not NULL, and this segment is not stored entirely on its ** root node, the range of leaves scanned can be reduced. Do this. */ if( iStartBlock && zTerm ){ @@ -2217,53 +2250,117 @@ int sqlite3Fts3SegReaderCursor( if( isPrefix==0 && isScan==0 ) iLeavesEndBlock = iStartBlock; } - rc = sqlite3Fts3SegReaderNew(iAge, iStartBlock, iLeavesEndBlock, - iEndBlock, zRoot, nRoot, &pCsr->apSegment[pCsr->nSegment] + rc = sqlite3Fts3SegReaderNew(pCsr->nSegment+1, + iStartBlock, iLeavesEndBlock, iEndBlock, zRoot, nRoot, &pSeg ); if( rc!=SQLITE_OK ) goto finished; - pCsr->nSegment++; - iAge++; + rc = fts3SegReaderCursorAppend(pCsr, pSeg); } } finished: rc2 = sqlite3_reset(pStmt); if( rc==SQLITE_DONE ) rc = rc2; - sqlite3Fts3SegReaderFree(pPending); return rc; } +/* +** Set up a cursor object for iterating through a full-text index or a +** single level therein. +*/ +int sqlite3Fts3SegReaderCursor( + Fts3Table *p, /* FTS3 table handle */ + int iIndex, /* Index to search (from 0 to p->nIndex-1) */ + int iLevel, /* Level of segments to scan */ + const char *zTerm, /* Term to query for */ + int nTerm, /* Size of zTerm in bytes */ + int isPrefix, /* True for a prefix search */ + int isScan, /* True to scan from zTerm to EOF */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ +){ + assert( iIndex>=0 && iIndexnIndex ); + assert( iLevel==FTS3_SEGCURSOR_ALL + || iLevel==FTS3_SEGCURSOR_PENDING + || iLevel>=0 + ); + assert( iLevelaIndex==0 ); + + memset(pCsr, 0, sizeof(Fts3MultiSegReader)); + + return fts3SegReaderCursor( + p, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr + ); +} + +static int fts3SegReaderCursorAddZero( + Fts3Table *p, + const char *zTerm, + int nTerm, + Fts3MultiSegReader *pCsr +){ + return fts3SegReaderCursor(p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0,pCsr); +} + + +int sqlite3Fts3TermSegReaderCursor( Fts3Cursor *pCsr, /* Virtual table cursor handle */ const char *zTerm, /* Term to query for */ int nTerm, /* Size of zTerm in bytes */ int isPrefix, /* True for a prefix search */ - Fts3SegReaderCursor **ppSegcsr /* OUT: Allocated seg-reader cursor */ + Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */ ){ - Fts3SegReaderCursor *pSegcsr; /* Object to allocate and return */ + Fts3MultiSegReader *pSegcsr; /* Object to allocate and return */ int rc = SQLITE_NOMEM; /* Return code */ - pSegcsr = sqlite3_malloc(sizeof(Fts3SegReaderCursor)); + pSegcsr = sqlite3_malloc(sizeof(Fts3MultiSegReader)); if( pSegcsr ){ - Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; int i; - int nCost = 0; - rc = sqlite3Fts3SegReaderCursor( - p, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr); - - for(i=0; rc==SQLITE_OK && inSegment; i++){ - rc = sqlite3Fts3SegReaderCost(pCsr, pSegcsr->apSegment[i], &nCost); + int bFound = 0; /* True once an index has been found */ + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + + if( isPrefix ){ + for(i=1; bFound==0 && inIndex; i++){ + if( p->aIndex[i].nPrefix==nTerm ){ + bFound = 1; + rc = sqlite3Fts3SegReaderCursor( + p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr); + pSegcsr->bLookup = 1; + } + } + + for(i=1; bFound==0 && inIndex; i++){ + if( p->aIndex[i].nPrefix==nTerm+1 ){ + bFound = 1; + rc = sqlite3Fts3SegReaderCursor( + p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 1, 0, pSegcsr + ); + if( rc==SQLITE_OK ){ + rc = fts3SegReaderCursorAddZero(p, zTerm, nTerm, pSegcsr); + } + } + } + } + + if( bFound==0 ){ + rc = sqlite3Fts3SegReaderCursor( + p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr + ); + pSegcsr->bLookup = !isPrefix; } - pSegcsr->nCost = nCost; } *ppSegcsr = pSegcsr; return rc; } -static void fts3SegReaderCursorFree(Fts3SegReaderCursor *pSegcsr){ +static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){ sqlite3Fts3SegReaderFinish(pSegcsr); sqlite3_free(pSegcsr); } @@ -2288,7 +2385,7 @@ static int fts3TermSelect( char **ppOut /* OUT: Malloced result buffer */ ){ int rc; /* Return code */ - Fts3SegReaderCursor *pSegcsr; /* Seg-reader cursor for this term */ + Fts3MultiSegReader *pSegcsr; /* Seg-reader cursor for this term */ TermSelect tsc; /* Context object for fts3TermSelectCb() */ Fts3SegFilter filter; /* Segment term filter configuration */ @@ -2314,7 +2411,7 @@ static int fts3TermSelect( } if( rc==SQLITE_OK ){ - rc = fts3TermSelectMerge(&tsc); + rc = fts3TermSelectMerge(p, &tsc); } if( rc==SQLITE_OK ){ *ppOut = tsc.aaOutput[0]; @@ -2364,660 +2461,6 @@ static int fts3DoclistCountDocids(int isPoslist, char *aList, int nList){ return nDoc; } -/* -** Call sqlite3Fts3DeferToken() for each token in the expression pExpr. -*/ -static int fts3DeferExpression(Fts3Cursor *pCsr, Fts3Expr *pExpr){ - int rc = SQLITE_OK; - if( pExpr ){ - rc = fts3DeferExpression(pCsr, pExpr->pLeft); - if( rc==SQLITE_OK ){ - rc = fts3DeferExpression(pCsr, pExpr->pRight); - } - if( pExpr->eType==FTSQUERY_PHRASE ){ - int iCol = pExpr->pPhrase->iColumn; - int i; - for(i=0; rc==SQLITE_OK && ipPhrase->nToken; i++){ - Fts3PhraseToken *pToken = &pExpr->pPhrase->aToken[i]; - if( pToken->pDeferred==0 ){ - rc = sqlite3Fts3DeferToken(pCsr, pToken, iCol); - } - } - } - } - return rc; -} - -/* -** This function removes the position information from a doclist. When -** called, buffer aList (size *pnList bytes) contains a doclist that includes -** position information. This function removes the position information so -** that aList contains only docids, and adjusts *pnList to reflect the new -** (possibly reduced) size of the doclist. -*/ -static void fts3DoclistStripPositions( - char *aList, /* IN/OUT: Buffer containing doclist */ - int *pnList /* IN/OUT: Size of doclist in bytes */ -){ - if( aList ){ - char *aEnd = &aList[*pnList]; /* Pointer to one byte after EOF */ - char *p = aList; /* Input cursor */ - char *pOut = aList; /* Output cursor */ - - while( piColumn; - int isTermPos = (pPhrase->nToken>1 || isReqPos); - Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; - int isFirst = 1; - - int iPrevTok = 0; - int nDoc = 0; - - /* If this is an xFilter() evaluation, create a segment-reader for each - ** phrase token. Or, if this is an xNext() or snippet/offsets/matchinfo - ** evaluation, only create segment-readers if there are no Fts3DeferredToken - ** objects attached to the phrase-tokens. - */ - for(ii=0; iinToken; ii++){ - Fts3PhraseToken *pTok = &pPhrase->aToken[ii]; - if( pTok->pSegcsr==0 ){ - if( (pCsr->eEvalmode==FTS3_EVAL_FILTER) - || (pCsr->eEvalmode==FTS3_EVAL_NEXT && pCsr->pDeferred==0) - || (pCsr->eEvalmode==FTS3_EVAL_MATCHINFO && pTok->bFulltext) - ){ - rc = fts3TermSegReaderCursor( - pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pSegcsr - ); - if( rc!=SQLITE_OK ) return rc; - } - } - } - - for(ii=0; iinToken; ii++){ - Fts3PhraseToken *pTok; /* Token to find doclist for */ - int iTok = 0; /* The token being queried this iteration */ - char *pList = 0; /* Pointer to token doclist */ - int nList = 0; /* Size of buffer at pList */ - - /* Select a token to process. If this is an xFilter() call, then tokens - ** are processed in order from least to most costly. Otherwise, tokens - ** are processed in the order in which they occur in the phrase. - */ - if( pCsr->eEvalmode==FTS3_EVAL_MATCHINFO ){ - assert( isReqPos ); - iTok = ii; - pTok = &pPhrase->aToken[iTok]; - if( pTok->bFulltext==0 ) continue; - }else if( pCsr->eEvalmode==FTS3_EVAL_NEXT || isReqPos ){ - iTok = ii; - pTok = &pPhrase->aToken[iTok]; - }else{ - int nMinCost = 0x7FFFFFFF; - int jj; - - /* Find the remaining token with the lowest cost. */ - for(jj=0; jjnToken; jj++){ - Fts3SegReaderCursor *pSegcsr = pPhrase->aToken[jj].pSegcsr; - if( pSegcsr && pSegcsr->nCostnCost; - } - } - pTok = &pPhrase->aToken[iTok]; - - /* This branch is taken if it is determined that loading the doclist - ** for the next token would require more IO than loading all documents - ** currently identified by doclist pOut/nOut. No further doclists will - ** be loaded from the full-text index for this phrase. - */ - if( nMinCost>nDoc && ii>0 ){ - rc = fts3DeferExpression(pCsr, pCsr->pExpr); - break; - } - } - - if( pCsr->eEvalmode==FTS3_EVAL_NEXT && pTok->pDeferred ){ - rc = fts3DeferredTermSelect(pTok->pDeferred, isTermPos, &nList, &pList); - }else{ - if( pTok->pSegcsr ){ - rc = fts3TermSelect(p, pTok, iCol, isTermPos, &nList, &pList); - } - pTok->bFulltext = 1; - } - assert( rc!=SQLITE_OK || pCsr->eEvalmode || pTok->pSegcsr==0 ); - if( rc!=SQLITE_OK ) break; - - if( isFirst ){ - pOut = pList; - nOut = nList; - if( pCsr->eEvalmode==FTS3_EVAL_FILTER && pPhrase->nToken>1 ){ - nDoc = fts3DoclistCountDocids(1, pOut, nOut); - } - isFirst = 0; - iPrevTok = iTok; - }else{ - /* Merge the new term list and the current output. */ - char *aLeft, *aRight; - int nLeft, nRight; - int nDist; - int mt; - - /* If this is the final token of the phrase, and positions were not - ** requested by the caller, use MERGE_PHRASE instead of POS_PHRASE. - ** This drops the position information from the output list. - */ - mt = MERGE_POS_PHRASE; - if( ii==pPhrase->nToken-1 && !isReqPos ) mt = MERGE_PHRASE; - - assert( iPrevTok!=iTok ); - if( iPrevToknToken ){ - assert( pCsr->eEvalmode==FTS3_EVAL_FILTER && isReqPos==0 ); - fts3DoclistStripPositions(pOut, &nOut); - } - *paOut = pOut; - *pnOut = nOut; - }else{ - sqlite3_free(pOut); - } - return rc; -} - -/* -** This function merges two doclists according to the requirements of a -** NEAR operator. -** -** Both input doclists must include position information. The output doclist -** includes position information if the first argument to this function -** is MERGE_POS_NEAR, or does not if it is MERGE_NEAR. -*/ -static int fts3NearMerge( - int mergetype, /* MERGE_POS_NEAR or MERGE_NEAR */ - int nNear, /* Parameter to NEAR operator */ - int nTokenLeft, /* Number of tokens in LHS phrase arg */ - char *aLeft, /* Doclist for LHS (incl. positions) */ - int nLeft, /* Size of LHS doclist in bytes */ - int nTokenRight, /* As nTokenLeft */ - char *aRight, /* As aLeft */ - int nRight, /* As nRight */ - char **paOut, /* OUT: Results of merge (malloced) */ - int *pnOut /* OUT: Sized of output buffer */ -){ - char *aOut; /* Buffer to write output doclist to */ - int rc; /* Return code */ - - assert( mergetype==MERGE_POS_NEAR || MERGE_NEAR ); - - aOut = sqlite3_malloc(nLeft+nRight+1); - if( aOut==0 ){ - rc = SQLITE_NOMEM; - }else{ - rc = fts3DoclistMerge(mergetype, nNear+nTokenRight, nNear+nTokenLeft, - aOut, pnOut, aLeft, nLeft, aRight, nRight, 0 - ); - if( rc!=SQLITE_OK ){ - sqlite3_free(aOut); - aOut = 0; - } - } - - *paOut = aOut; - return rc; -} - -/* -** This function is used as part of the processing for the snippet() and -** offsets() functions. -** -** Both pLeft and pRight are expression nodes of type FTSQUERY_PHRASE. Both -** have their respective doclists (including position information) loaded -** in Fts3Expr.aDoclist/nDoclist. This function removes all entries from -** each doclist that are not within nNear tokens of a corresponding entry -** in the other doclist. -*/ -int sqlite3Fts3ExprNearTrim(Fts3Expr *pLeft, Fts3Expr *pRight, int nNear){ - int rc; /* Return code */ - - assert( pLeft->eType==FTSQUERY_PHRASE ); - assert( pRight->eType==FTSQUERY_PHRASE ); - assert( pLeft->isLoaded && pRight->isLoaded ); - - if( pLeft->aDoclist==0 || pRight->aDoclist==0 ){ - sqlite3_free(pLeft->aDoclist); - sqlite3_free(pRight->aDoclist); - pRight->aDoclist = 0; - pLeft->aDoclist = 0; - rc = SQLITE_OK; - }else{ - char *aOut; /* Buffer in which to assemble new doclist */ - int nOut; /* Size of buffer aOut in bytes */ - - rc = fts3NearMerge(MERGE_POS_NEAR, nNear, - pLeft->pPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist, - pRight->pPhrase->nToken, pRight->aDoclist, pRight->nDoclist, - &aOut, &nOut - ); - if( rc!=SQLITE_OK ) return rc; - sqlite3_free(pRight->aDoclist); - pRight->aDoclist = aOut; - pRight->nDoclist = nOut; - - rc = fts3NearMerge(MERGE_POS_NEAR, nNear, - pRight->pPhrase->nToken, pRight->aDoclist, pRight->nDoclist, - pLeft->pPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist, - &aOut, &nOut - ); - sqlite3_free(pLeft->aDoclist); - pLeft->aDoclist = aOut; - pLeft->nDoclist = nOut; - } - return rc; -} - - -/* -** Allocate an Fts3SegReaderArray for each token in the expression pExpr. -** The allocated objects are stored in the Fts3PhraseToken.pArray member -** variables of each token structure. -*/ -static int fts3ExprAllocateSegReaders( - Fts3Cursor *pCsr, /* FTS3 table */ - Fts3Expr *pExpr, /* Expression to create seg-readers for */ - int *pnExpr /* OUT: Number of AND'd expressions */ -){ - int rc = SQLITE_OK; /* Return code */ - - assert( pCsr->eEvalmode==FTS3_EVAL_FILTER ); - if( pnExpr && pExpr->eType!=FTSQUERY_AND ){ - (*pnExpr)++; - pnExpr = 0; - } - - if( pExpr->eType==FTSQUERY_PHRASE ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - int ii; - - for(ii=0; rc==SQLITE_OK && iinToken; ii++){ - Fts3PhraseToken *pTok = &pPhrase->aToken[ii]; - if( pTok->pSegcsr==0 ){ - rc = fts3TermSegReaderCursor( - pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pSegcsr - ); - } - } - }else{ - rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pLeft, pnExpr); - if( rc==SQLITE_OK ){ - rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pRight, pnExpr); - } - } - return rc; -} - -/* -** Free the Fts3SegReaderArray objects associated with each token in the -** expression pExpr. In other words, this function frees the resources -** allocated by fts3ExprAllocateSegReaders(). -*/ -static void fts3ExprFreeSegReaders(Fts3Expr *pExpr){ - if( pExpr ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - if( pPhrase ){ - int kk; - for(kk=0; kknToken; kk++){ - fts3SegReaderCursorFree(pPhrase->aToken[kk].pSegcsr); - pPhrase->aToken[kk].pSegcsr = 0; - } - } - fts3ExprFreeSegReaders(pExpr->pLeft); - fts3ExprFreeSegReaders(pExpr->pRight); - } -} - -/* -** Return the sum of the costs of all tokens in the expression pExpr. This -** function must be called after Fts3SegReaderArrays have been allocated -** for all tokens using fts3ExprAllocateSegReaders(). -*/ -static int fts3ExprCost(Fts3Expr *pExpr){ - int nCost; /* Return value */ - if( pExpr->eType==FTSQUERY_PHRASE ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - int ii; - nCost = 0; - for(ii=0; iinToken; ii++){ - Fts3SegReaderCursor *pSegcsr = pPhrase->aToken[ii].pSegcsr; - if( pSegcsr ) nCost += pSegcsr->nCost; - } - }else{ - nCost = fts3ExprCost(pExpr->pLeft) + fts3ExprCost(pExpr->pRight); - } - return nCost; -} - -/* -** The following is a helper function (and type) for fts3EvalExpr(). It -** must be called after Fts3SegReaders have been allocated for every token -** in the expression. See the context it is called from in fts3EvalExpr() -** for further explanation. -*/ -typedef struct ExprAndCost ExprAndCost; -struct ExprAndCost { - Fts3Expr *pExpr; - int nCost; -}; -static void fts3ExprAssignCosts( - Fts3Expr *pExpr, /* Expression to create seg-readers for */ - ExprAndCost **ppExprCost /* OUT: Write to *ppExprCost */ -){ - if( pExpr->eType==FTSQUERY_AND ){ - fts3ExprAssignCosts(pExpr->pLeft, ppExprCost); - fts3ExprAssignCosts(pExpr->pRight, ppExprCost); - }else{ - (*ppExprCost)->pExpr = pExpr; - (*ppExprCost)->nCost = fts3ExprCost(pExpr); - (*ppExprCost)++; - } -} - -/* -** Evaluate the full-text expression pExpr against FTS3 table pTab. Store -** the resulting doclist in *paOut and *pnOut. This routine mallocs for -** the space needed to store the output. The caller is responsible for -** freeing the space when it has finished. -** -** This function is called in two distinct contexts: -** -** * From within the virtual table xFilter() method. In this case, the -** output doclist contains entries for all rows in the table, based on -** data read from the full-text index. -** -** In this case, if the query expression contains one or more tokens that -** are very common, then the returned doclist may contain a superset of -** the documents that actually match the expression. -** -** * From within the virtual table xNext() method. This call is only made -** if the call from within xFilter() found that there were very common -** tokens in the query expression and did return a superset of the -** matching documents. In this case the returned doclist contains only -** entries that correspond to the current row of the table. Instead of -** reading the data for each token from the full-text index, the data is -** already available in-memory in the Fts3PhraseToken.pDeferred structures. -** See fts3EvalDeferred() for how it gets there. -** -** In the first case above, Fts3Cursor.doDeferred==0. In the second (if it is -** required) Fts3Cursor.doDeferred==1. -** -** If the SQLite invokes the snippet(), offsets() or matchinfo() function -** as part of a SELECT on an FTS3 table, this function is called on each -** individual phrase expression in the query. If there were very common tokens -** found in the xFilter() call, then this function is called once for phrase -** for each row visited, and the returned doclist contains entries for the -** current row only. Otherwise, if there were no very common tokens, then this -** function is called once only for each phrase in the query and the returned -** doclist contains entries for all rows of the table. -** -** Fts3Cursor.doDeferred==1 when this function is called on phrases as a -** result of a snippet(), offsets() or matchinfo() invocation. -*/ -static int fts3EvalExpr( - Fts3Cursor *p, /* Virtual table cursor handle */ - Fts3Expr *pExpr, /* Parsed fts3 expression */ - char **paOut, /* OUT: Pointer to malloc'd result buffer */ - int *pnOut, /* OUT: Size of buffer at *paOut */ - int isReqPos /* Require positions in output buffer */ -){ - int rc = SQLITE_OK; /* Return code */ - - /* Zero the output parameters. */ - *paOut = 0; - *pnOut = 0; - - if( pExpr ){ - assert( pExpr->eType==FTSQUERY_NEAR || pExpr->eType==FTSQUERY_OR - || pExpr->eType==FTSQUERY_AND || pExpr->eType==FTSQUERY_NOT - || pExpr->eType==FTSQUERY_PHRASE - ); - assert( pExpr->eType==FTSQUERY_PHRASE || isReqPos==0 ); - - if( pExpr->eType==FTSQUERY_PHRASE ){ - rc = fts3PhraseSelect(p, pExpr->pPhrase, - isReqPos || (pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR), - paOut, pnOut - ); - fts3ExprFreeSegReaders(pExpr); - }else if( p->eEvalmode==FTS3_EVAL_FILTER && pExpr->eType==FTSQUERY_AND ){ - ExprAndCost *aExpr = 0; /* Array of AND'd expressions and costs */ - int nExpr = 0; /* Size of aExpr[] */ - char *aRet = 0; /* Doclist to return to caller */ - int nRet = 0; /* Length of aRet[] in bytes */ - int nDoc = 0x7FFFFFFF; - - assert( !isReqPos ); - - rc = fts3ExprAllocateSegReaders(p, pExpr, &nExpr); - if( rc==SQLITE_OK ){ - assert( nExpr>1 ); - aExpr = sqlite3_malloc(sizeof(ExprAndCost) * nExpr); - if( !aExpr ) rc = SQLITE_NOMEM; - } - if( rc==SQLITE_OK ){ - int ii; /* Used to iterate through expressions */ - - fts3ExprAssignCosts(pExpr, &aExpr); - aExpr -= nExpr; - for(ii=0; iipExpr && (pBest==0 || pCand->nCostnCost) ){ - pBest = pCand; - } - } - - if( pBest->nCost>nDoc ){ - rc = fts3DeferExpression(p, p->pExpr); - break; - }else{ - rc = fts3EvalExpr(p, pBest->pExpr, &aNew, &nNew, 0); - if( rc!=SQLITE_OK ) break; - pBest->pExpr = 0; - if( ii==0 ){ - aRet = aNew; - nRet = nNew; - nDoc = fts3DoclistCountDocids(0, aRet, nRet); - }else{ - fts3DoclistMerge( - MERGE_AND, 0, 0, aRet, &nRet, aRet, nRet, aNew, nNew, &nDoc - ); - sqlite3_free(aNew); - } - } - } - } - - if( rc==SQLITE_OK ){ - *paOut = aRet; - *pnOut = nRet; - }else{ - assert( *paOut==0 ); - sqlite3_free(aRet); - } - sqlite3_free(aExpr); - fts3ExprFreeSegReaders(pExpr); - - }else{ - char *aLeft; - char *aRight; - int nLeft; - int nRight; - - assert( pExpr->eType==FTSQUERY_NEAR - || pExpr->eType==FTSQUERY_OR - || pExpr->eType==FTSQUERY_NOT - || (pExpr->eType==FTSQUERY_AND && p->eEvalmode==FTS3_EVAL_NEXT) - ); - - if( 0==(rc = fts3EvalExpr(p, pExpr->pRight, &aRight, &nRight, isReqPos)) - && 0==(rc = fts3EvalExpr(p, pExpr->pLeft, &aLeft, &nLeft, isReqPos)) - ){ - switch( pExpr->eType ){ - case FTSQUERY_NEAR: { - Fts3Expr *pLeft; - Fts3Expr *pRight; - int mergetype = MERGE_NEAR; - if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){ - mergetype = MERGE_POS_NEAR; - } - pLeft = pExpr->pLeft; - while( pLeft->eType==FTSQUERY_NEAR ){ - pLeft=pLeft->pRight; - } - pRight = pExpr->pRight; - assert( pRight->eType==FTSQUERY_PHRASE ); - assert( pLeft->eType==FTSQUERY_PHRASE ); - - rc = fts3NearMerge(mergetype, pExpr->nNear, - pLeft->pPhrase->nToken, aLeft, nLeft, - pRight->pPhrase->nToken, aRight, nRight, - paOut, pnOut - ); - sqlite3_free(aLeft); - break; - } - - case FTSQUERY_OR: { - /* Allocate a buffer for the output. The maximum size is the - ** sum of the sizes of the two input buffers. The +1 term is - ** so that a buffer of zero bytes is never allocated - this can - ** cause fts3DoclistMerge() to incorrectly return SQLITE_NOMEM. - */ - char *aBuffer = sqlite3_malloc(nRight+nLeft+1); - rc = fts3DoclistMerge(MERGE_OR, 0, 0, aBuffer, pnOut, - aLeft, nLeft, aRight, nRight, 0 - ); - *paOut = aBuffer; - sqlite3_free(aLeft); - break; - } - - default: { - assert( FTSQUERY_NOT==MERGE_NOT && FTSQUERY_AND==MERGE_AND ); - fts3DoclistMerge(pExpr->eType, 0, 0, aLeft, pnOut, - aLeft, nLeft, aRight, nRight, 0 - ); - *paOut = aLeft; - break; - } - } - } - sqlite3_free(aRight); - } - } - - assert( rc==SQLITE_OK || *paOut==0 ); - return rc; -} - -/* -** This function is called from within xNext() for each row visited by -** an FTS3 query. If evaluating the FTS3 query expression within xFilter() -** was able to determine the exact set of matching rows, this function sets -** *pbRes to true and returns SQLITE_IO immediately. -** -** Otherwise, if evaluating the query expression within xFilter() returned a -** superset of the matching documents instead of an exact set (this happens -** when the query includes very common tokens and it is deemed too expensive to -** load their doclists from disk), this function tests if the current row -** really does match the FTS3 query. -** -** If an error occurs, an SQLite error code is returned. Otherwise, SQLITE_OK -** is returned and *pbRes is set to true if the current row matches the -** FTS3 query (and should be included in the results returned to SQLite), or -** false otherwise. -*/ -static int fts3EvalDeferred( - Fts3Cursor *pCsr, /* FTS3 cursor pointing at row to test */ - int *pbRes /* OUT: Set to true if row is a match */ -){ - int rc = SQLITE_OK; - if( pCsr->pDeferred==0 ){ - *pbRes = 1; - }else{ - rc = fts3CursorSeek(0, pCsr); - if( rc==SQLITE_OK ){ - sqlite3Fts3FreeDeferredDoclists(pCsr); - rc = sqlite3Fts3CacheDeferredDoclists(pCsr); - } - if( rc==SQLITE_OK ){ - char *a = 0; - int n = 0; - rc = fts3EvalExpr(pCsr, pCsr->pExpr, &a, &n, 0); - assert( n>=0 ); - *pbRes = (n>0); - sqlite3_free(a); - } - } - return rc; -} - /* ** Advance the cursor to the next row in the %_content table that ** matches the search criteria. For a MATCH search, this will be @@ -3030,39 +2473,20 @@ static int fts3EvalDeferred( ** subsequently to determine whether or not an EOF was hit. */ static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){ - int res; - int rc = SQLITE_OK; /* Return code */ + int rc; Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; - - pCsr->eEvalmode = FTS3_EVAL_NEXT; - do { - if( pCsr->aDoclist==0 ){ - if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){ - pCsr->isEof = 1; - rc = sqlite3_reset(pCsr->pStmt); - break; - } - pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0); + if( pCsr->eSearch==FTS3_DOCID_SEARCH || pCsr->eSearch==FTS3_FULLSCAN_SEARCH ){ + if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){ + pCsr->isEof = 1; + rc = sqlite3_reset(pCsr->pStmt); }else{ - if( pCsr->desc==0 ){ - if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){ - pCsr->isEof = 1; - break; - } - fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId); - }else{ - fts3GetReverseDeltaVarint(&pCsr->pNextId,pCsr->aDoclist,&pCsr->iPrevId); - if( pCsr->pNextId<=pCsr->aDoclist ){ - pCsr->isEof = 1; - break; - } - } - sqlite3_reset(pCsr->pStmt); - pCsr->isRequireSeek = 1; - pCsr->isMatchinfoNeeded = 1; + pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0); + rc = SQLITE_OK; } - }while( SQLITE_OK==(rc = fts3EvalDeferred(pCsr, &res)) && res==0 ); - + }else{ + rc = sqlite3Fts3EvalNext((Fts3Cursor *)pCursor); + } + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); return rc; } @@ -3089,11 +2513,7 @@ static int fts3FilterMethod( int nVal, /* Number of elements in apVal */ sqlite3_value **apVal /* Arguments for the indexing scheme */ ){ - const char *azSql[] = { - "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?", /* non-full-scan */ - "SELECT %s FROM %Q.'%q_content' AS x ORDER BY docid %s", /* full-scan */ - }; - int rc; /* Return code */ + int rc; char *zSql; /* SQL statement used to access %_content */ Fts3Table *p = (Fts3Table *)pCursor->pVtab; Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; @@ -3112,6 +2532,13 @@ static int fts3FilterMethod( sqlite3Fts3ExprFree(pCsr->pExpr); memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); + if( idxStr ){ + pCsr->bDesc = (idxStr[0]=='D'); + }else{ + pCsr->bDesc = p->bDescIdx; + } + pCsr->eSearch = (i16)idxNum; + if( idxNum!=FTS3_DOCID_SEARCH && idxNum!=FTS3_FULLSCAN_SEARCH ){ int iCol = idxNum-FTS3_FULLTEXT_SEARCH; const char *zQuery = (const char *)sqlite3_value_text(apVal[0]); @@ -3125,8 +2552,8 @@ static int fts3FilterMethod( ); if( rc!=SQLITE_OK ){ if( rc==SQLITE_ERROR ){ - p->base.zErrMsg = sqlite3_mprintf("malformed MATCH expression: [%s]", - zQuery); + static const char *zErr = "malformed MATCH expression: [%s]"; + p->base.zErrMsg = sqlite3_mprintf(zErr, zQuery); } return rc; } @@ -3134,7 +2561,8 @@ static int fts3FilterMethod( rc = sqlite3Fts3ReadLock(p); if( rc!=SQLITE_OK ) return rc; - rc = fts3EvalExpr(pCsr, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist, 0); + rc = sqlite3Fts3EvalStart(pCsr, pCsr->pExpr, 1); + sqlite3Fts3SegmentsClose(p); if( rc!=SQLITE_OK ) return rc; pCsr->pNextId = pCsr->aDoclist; @@ -3146,37 +2574,24 @@ static int fts3FilterMethod( ** full-text query or docid lookup, the statement retrieves a single ** row by docid. */ - zSql = (char *)azSql[idxNum==FTS3_FULLSCAN_SEARCH]; - zSql = sqlite3_mprintf( - zSql, p->zReadExprlist, p->zDb, p->zName, (idxStr ? idxStr : "ASC") - ); - if( !zSql ){ - rc = SQLITE_NOMEM; + if( idxNum==FTS3_FULLSCAN_SEARCH ){ + const char *zSort = (pCsr->bDesc ? "DESC" : "ASC"); + const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x ORDER BY docid %s"; + zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName, zSort); }else{ - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); - sqlite3_free(zSql); + const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?"; + zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName); } - if( rc==SQLITE_OK && idxNum==FTS3_DOCID_SEARCH ){ - rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); - } - pCsr->eSearch = (i16)idxNum; - - assert( pCsr->desc==0 ); + if( !zSql ) return SQLITE_NOMEM; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); if( rc!=SQLITE_OK ) return rc; - if( rc==SQLITE_OK && pCsr->nDoclist>0 && idxStr && idxStr[0]=='D' ){ - sqlite3_int64 iDocid = 0; - char *csr = pCsr->aDoclist; - while( csr<&pCsr->aDoclist[pCsr->nDoclist] ){ - fts3GetDeltaVarint(&csr, &iDocid); - } - pCsr->pNextId = csr; - pCsr->iPrevId = iDocid; - pCsr->desc = 1; - pCsr->isRequireSeek = 1; - pCsr->isMatchinfoNeeded = 1; - pCsr->eEvalmode = FTS3_EVAL_NEXT; - return SQLITE_OK; + + if( idxNum==FTS3_DOCID_SEARCH ){ + rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); + if( rc!=SQLITE_OK ) return rc; } + return fts3NextMethod(pCursor); } @@ -3196,16 +2611,7 @@ static int fts3EofMethod(sqlite3_vtab_cursor *pCursor){ */ static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; - if( pCsr->aDoclist ){ - *pRowid = pCsr->iPrevId; - }else{ - /* This branch runs if the query is implemented using a full-table scan - ** (not using the full-text index). In this case grab the rowid from the - ** SELECT statement. - */ - assert( pCsr->isRequireSeek==0 ); - *pRowid = sqlite3_column_int64(pCsr->pStmt, 0); - } + *pRowid = pCsr->iPrevId; return SQLITE_OK; } @@ -3218,7 +2624,7 @@ static int fts3ColumnMethod( sqlite3_context *pContext, /* Context for sqlite3_result_xxx() calls */ int iCol /* Index of column to read value from */ ){ - int rc; /* Return Code */ + int rc = SQLITE_OK; /* Return Code */ Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; Fts3Table *p = (Fts3Table *)pCursor->pVtab; @@ -3229,21 +2635,20 @@ static int fts3ColumnMethod( /* This call is a request for the "docid" column. Since "docid" is an ** alias for "rowid", use the xRowid() method to obtain the value. */ - sqlite3_int64 iRowid; - rc = fts3RowidMethod(pCursor, &iRowid); - sqlite3_result_int64(pContext, iRowid); + sqlite3_result_int64(pContext, pCsr->iPrevId); }else if( iCol==p->nColumn ){ /* The extra column whose name is the same as the table. ** Return a blob which is a pointer to the cursor. */ sqlite3_result_blob(pContext, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT); - rc = SQLITE_OK; }else{ rc = fts3CursorSeek(0, pCsr); if( rc==SQLITE_OK ){ sqlite3_result_value(pContext, sqlite3_column_value(pCsr->pStmt, iCol+1)); } } + + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); return rc; } @@ -3277,6 +2682,7 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){ static int fts3BeginMethod(sqlite3_vtab *pVtab){ UNUSED_PARAMETER(pVtab); TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); + assert( p->pSegments==0 ); assert( p->nPendingData==0 ); assert( p->inTransaction!=1 ); TESTONLY( p->inTransaction = 1 ); @@ -3294,6 +2700,7 @@ static int fts3CommitMethod(sqlite3_vtab *pVtab){ TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); assert( p->nPendingData==0 ); assert( p->inTransaction!=0 ); + assert( p->pSegments==0 ); TESTONLY( p->inTransaction = 0 ); TESTONLY( p->mxSavepoint = -1; ); return SQLITE_OK; @@ -3312,36 +2719,6 @@ static int fts3RollbackMethod(sqlite3_vtab *pVtab){ return SQLITE_OK; } -/* -** Load the doclist associated with expression pExpr to pExpr->aDoclist. -** The loaded doclist contains positions as well as the document ids. -** This is used by the matchinfo(), snippet() and offsets() auxillary -** functions. -*/ -int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *pCsr, Fts3Expr *pExpr){ - int rc; - assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase ); - assert( pCsr->eEvalmode==FTS3_EVAL_NEXT ); - rc = fts3EvalExpr(pCsr, pExpr, &pExpr->aDoclist, &pExpr->nDoclist, 1); - return rc; -} - -int sqlite3Fts3ExprLoadFtDoclist( - Fts3Cursor *pCsr, - Fts3Expr *pExpr, - char **paDoclist, - int *pnDoclist -){ - int rc; - assert( pCsr->eEvalmode==FTS3_EVAL_NEXT ); - assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase ); - pCsr->eEvalmode = FTS3_EVAL_MATCHINFO; - rc = fts3EvalExpr(pCsr, pExpr, paDoclist, pnDoclist, 1); - pCsr->eEvalmode = FTS3_EVAL_NEXT; - return rc; -} - - /* ** When called, *ppPoslist must point to the byte immediately following the ** end of a position-list. i.e. ( (*ppPoslist)[-1]==POS_END ). This function @@ -3349,8 +2726,10 @@ int sqlite3Fts3ExprLoadFtDoclist( ** same position list. */ static void fts3ReversePoslist(char *pStart, char **ppPoslist){ - char *p = &(*ppPoslist)[-3]; - char c = p[1]; + char *p = &(*ppPoslist)[-2]; + char c; + + while( p>pStart && (c=*p--)==0 ); while( p>pStart && (*p & 0x80) | c ){ c = *p--; } @@ -3359,80 +2738,6 @@ static void fts3ReversePoslist(char *pStart, char **ppPoslist){ *ppPoslist = p; } - -/* -** After ExprLoadDoclist() (see above) has been called, this function is -** used to iterate/search through the position lists that make up the doclist -** stored in pExpr->aDoclist. -*/ -char *sqlite3Fts3FindPositions( - Fts3Cursor *pCursor, /* Associate FTS3 cursor */ - Fts3Expr *pExpr, /* Access this expressions doclist */ - sqlite3_int64 iDocid, /* Docid associated with requested pos-list */ - int iCol /* Column of requested pos-list */ -){ - assert( pExpr->isLoaded ); - if( pExpr->aDoclist ){ - char *pEnd = &pExpr->aDoclist[pExpr->nDoclist]; - char *pCsr; - - if( pExpr->pCurrent==0 ){ - if( pCursor->desc==0 ){ - pExpr->pCurrent = pExpr->aDoclist; - pExpr->iCurrent = 0; - fts3GetDeltaVarint(&pExpr->pCurrent, &pExpr->iCurrent); - }else{ - pCsr = pExpr->aDoclist; - while( pCsriCurrent); - fts3PoslistCopy(0, &pCsr); - } - fts3ReversePoslist(pExpr->aDoclist, &pCsr); - pExpr->pCurrent = pCsr; - } - } - pCsr = pExpr->pCurrent; - assert( pCsr ); - - while( (pCursor->desc==0 && pCsrdesc && pCsr>pExpr->aDoclist) - ){ - if( pCursor->desc==0 && pExpr->iCurrentiCurrent); - } - pExpr->pCurrent = pCsr; - }else if( pCursor->desc && pExpr->iCurrent>iDocid ){ - fts3GetReverseDeltaVarint(&pCsr, pExpr->aDoclist, &pExpr->iCurrent); - fts3ReversePoslist(pExpr->aDoclist, &pCsr); - pExpr->pCurrent = pCsr; - }else{ - if( pExpr->iCurrent==iDocid ){ - int iThis = 0; - if( iCol<0 ){ - /* If iCol is negative, return a pointer to the start of the - ** position-list (instead of a pointer to the start of a list - ** of offsets associated with a specific column). - */ - return pCsr; - } - while( iThisinTransaction ); - assert( p->mxSavepoint < iSavepoint ); - TESTONLY( p->mxSavepoint = iSavepoint ); - return sqlite3Fts3PendingTermsFlush(p); + assert( ((Fts3Table *)pVtab)->inTransaction ); + assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint ); + TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint ); + return fts3SyncMethod(pVtab); } static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); @@ -3840,4 +3144,1306 @@ int sqlite3_extension_init( } #endif + +/* +** Allocate an Fts3MultiSegReader for each token in the expression headed +** by pExpr. +** +** An Fts3SegReader object is a cursor that can seek or scan a range of +** entries within a single segment b-tree. An Fts3MultiSegReader uses multiple +** Fts3SegReader objects internally to provide an interface to seek or scan +** within the union of all segments of a b-tree. Hence the name. +** +** If the allocated Fts3MultiSegReader just seeks to a single entry in a +** segment b-tree (if the term is not a prefix or it is a prefix for which +** there exists prefix b-tree of the right length) then it may be traversed +** and merged incrementally. Otherwise, it has to be merged into an in-memory +** doclist and then traversed. +*/ +static void fts3EvalAllocateReaders( + Fts3Cursor *pCsr, + Fts3Expr *pExpr, + int *pnToken, /* OUT: Total number of tokens in phrase. */ + int *pnOr, /* OUT: Total number of OR nodes in expr. */ + int *pRc +){ + if( pExpr && SQLITE_OK==*pRc ){ + if( pExpr->eType==FTSQUERY_PHRASE ){ + int i; + int nToken = pExpr->pPhrase->nToken; + *pnToken += nToken; + for(i=0; ipPhrase->aToken[i]; + int rc = sqlite3Fts3TermSegReaderCursor(pCsr, + pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr + ); + if( rc!=SQLITE_OK ){ + *pRc = rc; + return; + } + } + assert( pExpr->pPhrase->iDoclistToken==0 ); + pExpr->pPhrase->iDoclistToken = -1; + }else{ + *pnOr += (pExpr->eType==FTSQUERY_OR); + fts3EvalAllocateReaders(pCsr, pExpr->pLeft, pnToken, pnOr, pRc); + fts3EvalAllocateReaders(pCsr, pExpr->pRight, pnToken, pnOr, pRc); + } + } +} + +static void fts3EvalPhraseMergeToken( + Fts3Table *pTab, + Fts3Phrase *p, + int iToken, + char *pList, + int nList +){ + assert( iToken!=p->iDoclistToken ); + + if( pList==0 ){ + sqlite3_free(p->doclist.aAll); + p->doclist.aAll = 0; + p->doclist.nAll = 0; + } + + else if( p->iDoclistToken<0 ){ + p->doclist.aAll = pList; + p->doclist.nAll = nList; + } + + else if( p->doclist.aAll==0 ){ + sqlite3_free(pList); + } + + else { + char *pLeft; + char *pRight; + int nLeft; + int nRight; + int nDiff; + + if( p->iDoclistTokendoclist.aAll; + nLeft = p->doclist.nAll; + pRight = pList; + nRight = nList; + nDiff = iToken - p->iDoclistToken; + }else{ + pRight = p->doclist.aAll; + nRight = p->doclist.nAll; + pLeft = pList; + nLeft = nList; + nDiff = p->iDoclistToken - iToken; + } + + fts3DoclistPhraseMerge(pTab->bDescIdx, nDiff, pLeft, nLeft, pRight,&nRight); + sqlite3_free(pLeft); + p->doclist.aAll = pRight; + p->doclist.nAll = nRight; + } + + if( iToken>p->iDoclistToken ) p->iDoclistToken = iToken; +} + +static int fts3EvalPhraseLoad( + Fts3Cursor *pCsr, + Fts3Phrase *p +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int iToken; + int rc = SQLITE_OK; + + for(iToken=0; rc==SQLITE_OK && iTokennToken; iToken++){ + Fts3PhraseToken *pToken = &p->aToken[iToken]; + assert( pToken->pDeferred==0 || pToken->pSegcsr==0 ); + + if( pToken->pSegcsr ){ + int nThis = 0; + char *pThis = 0; + rc = fts3TermSelect(pTab, pToken, p->iColumn, 1, &nThis, &pThis); + if( rc==SQLITE_OK ){ + fts3EvalPhraseMergeToken(pTab, p, iToken, pThis, nThis); + } + } + assert( pToken->pSegcsr==0 ); + } + + return rc; +} + +static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ + int iToken; + int rc = SQLITE_OK; + + int nMaxUndeferred = pPhrase->iDoclistToken; + char *aPoslist = 0; + int nPoslist = 0; + int iPrev = -1; + + assert( pPhrase->doclist.bFreeList==0 ); + + for(iToken=0; rc==SQLITE_OK && iTokennToken; iToken++){ + Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; + Fts3DeferredToken *pDeferred = pToken->pDeferred; + + if( pDeferred ){ + char *pList; + int nList; + rc = sqlite3Fts3DeferredTokenList(pDeferred, &pList, &nList); + if( rc!=SQLITE_OK ) return rc; + + if( pList==0 ){ + sqlite3_free(aPoslist); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + return SQLITE_OK; + + }else if( aPoslist==0 ){ + aPoslist = pList; + nPoslist = nList; + + }else{ + char *aOut = pList; + char *p1 = aPoslist; + char *p2 = aOut; + + assert( iPrev>=0 ); + fts3PoslistPhraseMerge(&aOut, iToken-iPrev, 0, 1, &p1, &p2); + sqlite3_free(aPoslist); + aPoslist = pList; + nPoslist = aOut - aPoslist; + if( nPoslist==0 ){ + sqlite3_free(aPoslist); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + return SQLITE_OK; + } + } + iPrev = iToken; + } + } + + if( iPrev>=0 ){ + if( nMaxUndeferred<0 ){ + pPhrase->doclist.pList = aPoslist; + pPhrase->doclist.nList = nPoslist; + pPhrase->doclist.iDocid = pCsr->iPrevId; + pPhrase->doclist.bFreeList = 1; + }else{ + int nDistance; + char *p1; + char *p2; + char *aOut; + + if( nMaxUndeferred>iPrev ){ + p1 = aPoslist; + p2 = pPhrase->doclist.pList; + nDistance = nMaxUndeferred - iPrev; + }else{ + p1 = pPhrase->doclist.pList; + p2 = aPoslist; + nDistance = iPrev - nMaxUndeferred; + } + + aOut = (char *)sqlite3_malloc(nPoslist+8); + if( !aOut ){ + sqlite3_free(aPoslist); + return SQLITE_NOMEM; + } + + pPhrase->doclist.pList = aOut; + if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){ + pPhrase->doclist.bFreeList = 1; + pPhrase->doclist.nList = (aOut - pPhrase->doclist.pList); + }else{ + sqlite3_free(aOut); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + } + sqlite3_free(aPoslist); + } + } + + return SQLITE_OK; +} + +/* +** This function is called for each Fts3Phrase in a full-text query +** expression to initialize the mechanism for returning rows. Once this +** function has been called successfully on an Fts3Phrase, it may be +** used with fts3EvalPhraseNext() to iterate through the matching docids. +*/ +static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ + int rc; + Fts3PhraseToken *pFirst = &p->aToken[0]; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + + if( pCsr->bDesc==pTab->bDescIdx + && bOptOk==1 + && p->nToken==1 + && pFirst->pSegcsr + && pFirst->pSegcsr->bLookup + ){ + /* Use the incremental approach. */ + int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn); + rc = sqlite3Fts3MsrIncrStart( + pTab, pFirst->pSegcsr, iCol, pFirst->z, pFirst->n); + p->bIncr = 1; + + }else{ + /* Load the full doclist for the phrase into memory. */ + rc = fts3EvalPhraseLoad(pCsr, p); + p->bIncr = 0; + } + + assert( rc!=SQLITE_OK || p->nToken<1 || p->aToken[0].pSegcsr==0 || p->bIncr ); + return rc; +} + +/* +** This function is used to iterate backwards (from the end to start) +** through doclists. +*/ +void sqlite3Fts3DoclistPrev( + int bDescIdx, /* True if the doclist is desc */ + char *aDoclist, /* Pointer to entire doclist */ + int nDoclist, /* Length of aDoclist in bytes */ + char **ppIter, /* IN/OUT: Iterator pointer */ + sqlite3_int64 *piDocid, /* IN/OUT: Docid pointer */ + int *pnList, /* IN/OUT: List length pointer */ + u8 *pbEof /* OUT: End-of-file flag */ +){ + char *p = *ppIter; + + assert( nDoclist>0 ); + assert( *pbEof==0 ); + assert( p || *piDocid==0 ); + assert( !p || (p>aDoclist && p<&aDoclist[nDoclist]) ); + + if( p==0 ){ + sqlite3_int64 iDocid = 0; + char *pNext = 0; + char *pDocid = aDoclist; + char *pEnd = &aDoclist[nDoclist]; + int iMul = 1; + + while( pDociddoclist; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + + if( p->bIncr ){ + assert( p->nToken==1 ); + assert( pDL->pNextDocid==0 ); + rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, + &pDL->iDocid, &pDL->pList, &pDL->nList + ); + if( rc==SQLITE_OK && !pDL->pList ){ + *pbEof = 1; + } + }else if( pCsr->bDesc!=pTab->bDescIdx && pDL->nAll ){ + sqlite3Fts3DoclistPrev(pTab->bDescIdx, pDL->aAll, pDL->nAll, + &pDL->pNextDocid, &pDL->iDocid, &pDL->nList, pbEof + ); + pDL->pList = pDL->pNextDocid; + }else{ + char *pIter; /* Used to iterate through aAll */ + char *pEnd = &pDL->aAll[pDL->nAll]; /* 1 byte past end of aAll */ + if( pDL->pNextDocid ){ + pIter = pDL->pNextDocid; + }else{ + pIter = pDL->aAll; + } + + if( pIter>=pEnd ){ + /* We have already reached the end of this doclist. EOF. */ + *pbEof = 1; + }else{ + sqlite3_int64 iDelta; + pIter += sqlite3Fts3GetVarint(pIter, &iDelta); + if( pTab->bDescIdx==0 || pDL->pNextDocid==0 ){ + pDL->iDocid += iDelta; + }else{ + pDL->iDocid -= iDelta; + } + pDL->pList = pIter; + fts3PoslistCopy(0, &pIter); + pDL->nList = (pIter - pDL->pList); + + /* pIter now points just past the 0x00 that terminates the position- + ** list for document pDL->iDocid. However, if this position-list was + ** edited in place by fts3EvalNearTrim2(), then pIter may not actually + ** point to the start of the next docid value. The following line deals + ** with this case by advancing pIter past the zero-padding added by + ** fts3EvalNearTrim2(). */ + while( pIterpNextDocid = pIter; + assert( pIter>=&pDL->aAll[pDL->nAll] || *pIter ); + *pbEof = 0; + } + } + + return rc; +} + +static void fts3EvalStartReaders( + Fts3Cursor *pCsr, + Fts3Expr *pExpr, + int bOptOk, + int *pRc +){ + if( pExpr && SQLITE_OK==*pRc ){ + if( pExpr->eType==FTSQUERY_PHRASE ){ + int i; + int nToken = pExpr->pPhrase->nToken; + for(i=0; ipPhrase->aToken[i].pDeferred==0 ) break; + } + pExpr->bDeferred = (i==nToken); + *pRc = fts3EvalPhraseStart(pCsr, bOptOk, pExpr->pPhrase); + }else{ + fts3EvalStartReaders(pCsr, pExpr->pLeft, bOptOk, pRc); + fts3EvalStartReaders(pCsr, pExpr->pRight, bOptOk, pRc); + pExpr->bDeferred = (pExpr->pLeft->bDeferred && pExpr->pRight->bDeferred); + } + } +} + +typedef struct Fts3TokenAndCost Fts3TokenAndCost; +struct Fts3TokenAndCost { + Fts3Phrase *pPhrase; /* The phrase the token belongs to */ + int iToken; /* Position of token in phrase */ + Fts3PhraseToken *pToken; /* The token itself */ + Fts3Expr *pRoot; + int nOvfl; + int iCol; /* The column the token must match */ +}; + +static void fts3EvalTokenCosts( + Fts3Cursor *pCsr, + Fts3Expr *pRoot, + Fts3Expr *pExpr, + Fts3TokenAndCost **ppTC, + Fts3Expr ***ppOr, + int *pRc +){ + if( *pRc==SQLITE_OK && pExpr ){ + if( pExpr->eType==FTSQUERY_PHRASE ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + int i; + for(i=0; *pRc==SQLITE_OK && inToken; i++){ + Fts3TokenAndCost *pTC = (*ppTC)++; + pTC->pPhrase = pPhrase; + pTC->iToken = i; + pTC->pRoot = pRoot; + pTC->pToken = &pPhrase->aToken[i]; + pTC->iCol = pPhrase->iColumn; + *pRc = sqlite3Fts3MsrOvfl(pCsr, pTC->pToken->pSegcsr, &pTC->nOvfl); + } + }else if( pExpr->eType!=FTSQUERY_NOT ){ + if( pExpr->eType==FTSQUERY_OR ){ + pRoot = pExpr->pLeft; + **ppOr = pRoot; + (*ppOr)++; + } + fts3EvalTokenCosts(pCsr, pRoot, pExpr->pLeft, ppTC, ppOr, pRc); + if( pExpr->eType==FTSQUERY_OR ){ + pRoot = pExpr->pRight; + **ppOr = pRoot; + (*ppOr)++; + } + fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc); + } + } +} + +static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){ + if( pCsr->nRowAvg==0 ){ + /* The average document size, which is required to calculate the cost + ** of each doclist, has not yet been determined. Read the required + ** data from the %_stat table to calculate it. + ** + ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 + ** varints, where nCol is the number of columns in the FTS3 table. + ** The first varint is the number of documents currently stored in + ** the table. The following nCol varints contain the total amount of + ** data stored in all rows of each column of the table, from left + ** to right. + */ + int rc; + Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; + sqlite3_stmt *pStmt; + sqlite3_int64 nDoc = 0; + sqlite3_int64 nByte = 0; + const char *pEnd; + const char *a; + + rc = sqlite3Fts3SelectDoctotal(p, &pStmt); + if( rc!=SQLITE_OK ) return rc; + a = sqlite3_column_blob(pStmt, 0); + assert( a ); + + pEnd = &a[sqlite3_column_bytes(pStmt, 0)]; + a += sqlite3Fts3GetVarint(a, &nDoc); + while( anDoc = nDoc; + pCsr->nRowAvg = (int)(((nByte / nDoc) + p->nPgsz) / p->nPgsz); + assert( pCsr->nRowAvg>0 ); + rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ) return rc; + } + + *pnPage = pCsr->nRowAvg; + return SQLITE_OK; +} + +static int fts3EvalSelectDeferred( + Fts3Cursor *pCsr, + Fts3Expr *pRoot, + Fts3TokenAndCost *aTC, + int nTC +){ + int nDocSize = 0; + int nDocEst = 0; + int rc = SQLITE_OK; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int ii; + + int nOvfl = 0; + int nTerm = 0; + + for(ii=0; iinOvfl) + ){ + pTC = &aTC[jj]; + } + } + assert( pTC ); + + /* At this point pTC points to the cheapest remaining token. */ + if( ii==0 ){ + if( pTC->nOvfl ){ + nDocEst = (pTC->nOvfl * pTab->nPgsz + pTab->nPgsz) / 10; + }else{ + Fts3PhraseToken *pToken = pTC->pToken; + int nList = 0; + char *pList = 0; + rc = fts3TermSelect(pTab, pToken, pTC->iCol, 1, &nList, &pList); + assert( rc==SQLITE_OK || pList==0 ); + + if( rc==SQLITE_OK ){ + nDocEst = fts3DoclistCountDocids(1, pList, nList); + fts3EvalPhraseMergeToken(pTab, pTC->pPhrase, pTC->iToken,pList,nList); + } + } + }else{ + if( pTC->nOvfl>=(nDocEst*nDocSize) ){ + Fts3PhraseToken *pToken = pTC->pToken; + rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol); + fts3SegReaderCursorFree(pToken->pSegcsr); + pToken->pSegcsr = 0; + } + nDocEst = 1 + (nDocEst/4); + } + pTC->pToken = 0; + } + + return rc; +} + +int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int nToken = 0; + int nOr = 0; + + /* Allocate a MultiSegReader for each token in the expression. */ + fts3EvalAllocateReaders(pCsr, pExpr, &nToken, &nOr, &rc); + + /* Call fts3EvalPhraseStart() on all phrases in the expression. TODO: + ** This call will eventually also be responsible for determining which + ** tokens are 'deferred' until the document text is loaded into memory. + ** + ** Each token in each phrase is dealt with using one of the following + ** three strategies: + ** + ** 1. Entire doclist loaded into memory as part of the + ** fts3EvalStartReaders() call. + ** + ** 2. Doclist loaded into memory incrementally, as part of each + ** sqlite3Fts3EvalNext() call. + ** + ** 3. Token doclist is never loaded. Instead, documents are loaded into + ** memory and scanned for the token as part of the sqlite3Fts3EvalNext() + ** call. This is known as a "deferred" token. + */ + + /* If bOptOk is true, check if there are any tokens that should be deferred. + */ + if( rc==SQLITE_OK && bOptOk && nToken>1 && pTab->bHasStat ){ + Fts3TokenAndCost *aTC; + Fts3Expr **apOr; + aTC = (Fts3TokenAndCost *)sqlite3_malloc( + sizeof(Fts3TokenAndCost) * nToken + + sizeof(Fts3Expr *) * nOr * 2 + ); + apOr = (Fts3Expr **)&aTC[nToken]; + + if( !aTC ){ + rc = SQLITE_NOMEM; + }else{ + int ii; + Fts3TokenAndCost *pTC = aTC; + Fts3Expr **ppOr = apOr; + + fts3EvalTokenCosts(pCsr, 0, pExpr, &pTC, &ppOr, &rc); + nToken = pTC-aTC; + nOr = ppOr-apOr; + + if( rc==SQLITE_OK ){ + rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken); + for(ii=0; rc==SQLITE_OK && iidoclist.bFreeList ){ + sqlite3_free(pPhrase->doclist.pList); + } + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + pPhrase->doclist.bFreeList = 0; +} + +static int fts3EvalNearTrim2( + int nNear, + char *aTmp, /* Temporary space to use */ + char **paPoslist, /* IN/OUT: Position list */ + int *pnToken, /* IN/OUT: Tokens in phrase of *paPoslist */ + Fts3Phrase *pPhrase /* The phrase object to trim the doclist of */ +){ + int nParam1 = nNear + pPhrase->nToken; + int nParam2 = nNear + *pnToken; + int nNew; + char *p2; + char *pOut; + int res; + + assert( pPhrase->doclist.pList ); + + p2 = pOut = pPhrase->doclist.pList; + res = fts3PoslistNearMerge( + &pOut, aTmp, nParam1, nParam2, paPoslist, &p2 + ); + if( res ){ + nNew = (pOut - pPhrase->doclist.pList) - 1; + assert( pPhrase->doclist.pList[nNew]=='\0' ); + assert( nNew<=pPhrase->doclist.nList && nNew>0 ); + memset(&pPhrase->doclist.pList[nNew], 0, pPhrase->doclist.nList - nNew); + pPhrase->doclist.nList = nNew; + *paPoslist = pPhrase->doclist.pList; + *pnToken = pPhrase->nToken; + } + + return res; +} + +static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ + int res = 1; + + /* The following block runs if pExpr is the root of a NEAR query. + ** For example, the query: + ** + ** "w" NEAR "x" NEAR "y" NEAR "z" + ** + ** which is represented in tree form as: + ** + ** | + ** +--NEAR--+ <-- root of NEAR query + ** | | + ** +--NEAR--+ "z" + ** | | + ** +--NEAR--+ "y" + ** | | + ** "w" "x" + ** + ** The right-hand child of a NEAR node is always a phrase. The + ** left-hand child may be either a phrase or a NEAR node. There are + ** no exceptions to this. + */ + if( *pRc==SQLITE_OK + && pExpr->eType==FTSQUERY_NEAR + && pExpr->bEof==0 + && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) + ){ + Fts3Expr *p; + int nTmp = 0; /* Bytes of temp space */ + char *aTmp; /* Temp space for PoslistNearMerge() */ + + /* Allocate temporary working space. */ + for(p=pExpr; p->pLeft; p=p->pLeft){ + nTmp += p->pRight->pPhrase->doclist.nList; + } + nTmp += p->pPhrase->doclist.nList; + aTmp = sqlite3_malloc(nTmp*2); + if( !aTmp ){ + *pRc = SQLITE_NOMEM; + res = 0; + }else{ + char *aPoslist = p->pPhrase->doclist.pList; + int nToken = p->pPhrase->nToken; + + for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){ + Fts3Phrase *pPhrase = p->pRight->pPhrase; + int nNear = p->nNear; + res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase); + } + + aPoslist = pExpr->pRight->pPhrase->doclist.pList; + nToken = pExpr->pRight->pPhrase->nToken; + for(p=pExpr->pLeft; p && res; p=p->pLeft){ + int nNear = p->pParent->nNear; + Fts3Phrase *pPhrase = ( + p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase + ); + res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase); + } + } + + sqlite3_free(aTmp); + } + + return res; +} + +/* +** This macro is used by the fts3EvalNext() function. The two arguments are +** 64-bit docid values. If the current query is "ORDER BY docid ASC", then +** the macro returns (i1 - i2). Or if it is "ORDER BY docid DESC", then +** it returns (i2 - i1). This allows the same code to be used for merging +** doclists in ascending or descending order. +*/ +#define DOCID_CMP(i1, i2) ((pCsr->bDesc?-1:1) * (i1-i2)) + +static void fts3EvalNext( + Fts3Cursor *pCsr, + Fts3Expr *pExpr, + int *pRc +){ + if( *pRc==SQLITE_OK ){ + assert( pExpr->bEof==0 ); + pExpr->bStart = 1; + + switch( pExpr->eType ){ + case FTSQUERY_NEAR: + case FTSQUERY_AND: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + assert( !pLeft->bDeferred || !pRight->bDeferred ); + if( pLeft->bDeferred ){ + fts3EvalNext(pCsr, pRight, pRc); + pExpr->iDocid = pRight->iDocid; + pExpr->bEof = pRight->bEof; + }else if( pRight->bDeferred ){ + fts3EvalNext(pCsr, pLeft, pRc); + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = pLeft->bEof; + }else{ + fts3EvalNext(pCsr, pLeft, pRc); + fts3EvalNext(pCsr, pRight, pRc); + + while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){ + sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + if( iDiff==0 ) break; + if( iDiff<0 ){ + fts3EvalNext(pCsr, pLeft, pRc); + }else{ + fts3EvalNext(pCsr, pRight, pRc); + } + } + + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = (pLeft->bEof || pRight->bEof); + } + break; + } + + case FTSQUERY_OR: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + + assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid ); + assert( pRight->bStart || pLeft->iDocid==pRight->iDocid ); + + if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ + fts3EvalNext(pCsr, pLeft, pRc); + }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){ + fts3EvalNext(pCsr, pRight, pRc); + }else{ + fts3EvalNext(pCsr, pLeft, pRc); + fts3EvalNext(pCsr, pRight, pRc); + } + + pExpr->bEof = (pLeft->bEof && pRight->bEof); + iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ + pExpr->iDocid = pLeft->iDocid; + }else{ + pExpr->iDocid = pRight->iDocid; + } + + break; + } + + case FTSQUERY_NOT: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + + if( pRight->bStart==0 ){ + fts3EvalNext(pCsr, pRight, pRc); + assert( *pRc!=SQLITE_OK || pRight->bStart ); + } + + fts3EvalNext(pCsr, pLeft, pRc); + if( pLeft->bEof==0 ){ + while( !*pRc + && !pRight->bEof + && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 + ){ + fts3EvalNext(pCsr, pRight, pRc); + } + } + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = pLeft->bEof; + break; + } + + default: { + Fts3Phrase *pPhrase = pExpr->pPhrase; + fts3EvalZeroPoslist(pPhrase); + *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof); + pExpr->iDocid = pPhrase->doclist.iDocid; + break; + } + } + } +} + +static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ + int bHit = 1; + if( *pRc==SQLITE_OK ){ + switch( pExpr->eType ){ + case FTSQUERY_NEAR: + case FTSQUERY_AND: + bHit = ( + fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc) + && fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc) + && fts3EvalNearTest(pExpr, pRc) + ); + + /* If the NEAR expression does not match any rows, zero the doclist for + ** all phrases involved in the NEAR. This is because the snippet(), + ** offsets() and matchinfo() functions are not supposed to recognize + ** any instances of phrases that are part of unmatched NEAR queries. + ** For example if this expression: + ** + ** ... MATCH 'a OR (b NEAR c)' + ** + ** is matched against a row containing: + ** + ** 'a b d e' + ** + ** then any snippet() should ony highlight the "a" term, not the "b" + ** (as "b" is part of a non-matching NEAR clause). + */ + if( bHit==0 + && pExpr->eType==FTSQUERY_NEAR + && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) + ){ + Fts3Expr *p; + for(p=pExpr; p->pPhrase==0; p=p->pLeft){ + if( p->pRight->iDocid==pCsr->iPrevId ){ + fts3EvalZeroPoslist(p->pRight->pPhrase); + } + } + if( p->iDocid==pCsr->iPrevId ){ + fts3EvalZeroPoslist(p->pPhrase); + } + } + + break; + + case FTSQUERY_OR: { + int bHit1 = fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc); + int bHit2 = fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc); + bHit = bHit1 || bHit2; + break; + } + + case FTSQUERY_NOT: + bHit = ( + fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc) + && !fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc) + ); + break; + + default: { + if( pCsr->pDeferred + && (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred) + ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 ); + if( pExpr->bDeferred ){ + fts3EvalZeroPoslist(pPhrase); + } + *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase); + bHit = (pPhrase->doclist.pList!=0); + pExpr->iDocid = pCsr->iPrevId; + }else{ + bHit = (pExpr->bEof==0 && pExpr->iDocid==pCsr->iPrevId); + } + break; + } + } + } + return bHit; +} + +/* +** Return 1 if both of the following are true: +** +** 1. *pRc is SQLITE_OK when this function returns, and +** +** 2. After scanning the current FTS table row for the deferred tokens, +** it is determined that the row does not match the query. +** +** Or, if no error occurs and it seems the current row does match the FTS +** query, return 0. +*/ +static int fts3EvalLoadDeferred(Fts3Cursor *pCsr, int *pRc){ + int rc = *pRc; + int bMiss = 0; + if( rc==SQLITE_OK ){ + if( pCsr->pDeferred ){ + rc = fts3CursorSeek(0, pCsr); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3CacheDeferredDoclists(pCsr); + } + } + bMiss = (0==fts3EvalDeferredTest(pCsr, pCsr->pExpr, &rc)); + sqlite3Fts3FreeDeferredDoclists(pCsr); + *pRc = rc; + } + return (rc==SQLITE_OK && bMiss); +} + +/* +** Advance to the next document that matches the FTS expression in +** Fts3Cursor.pExpr. +*/ +int sqlite3Fts3EvalNext(Fts3Cursor *pCsr){ + int rc = SQLITE_OK; /* Return Code */ + Fts3Expr *pExpr = pCsr->pExpr; + assert( pCsr->isEof==0 ); + if( pExpr==0 ){ + pCsr->isEof = 1; + }else{ + do { + if( pCsr->isRequireSeek==0 ){ + sqlite3_reset(pCsr->pStmt); + } + assert( sqlite3_data_count(pCsr->pStmt)==0 ); + fts3EvalNext(pCsr, pExpr, &rc); + pCsr->isEof = pExpr->bEof; + pCsr->isRequireSeek = 1; + pCsr->isMatchinfoNeeded = 1; + pCsr->iPrevId = pExpr->iDocid; + }while( pCsr->isEof==0 && fts3EvalLoadDeferred(pCsr, &rc) ); + } + return rc; +} + +/* +** Restart interation for expression pExpr so that the next call to +** sqlite3Fts3EvalNext() visits the first row. Do not allow incremental +** loading or merging of phrase doclists for this iteration. +** +** If *pRc is other than SQLITE_OK when this function is called, it is +** a no-op. If an error occurs within this function, *pRc is set to an +** SQLite error code before returning. +*/ +static void fts3EvalRestart( + Fts3Cursor *pCsr, + Fts3Expr *pExpr, + int *pRc +){ + if( pExpr && *pRc==SQLITE_OK ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + + if( pPhrase ){ + fts3EvalZeroPoslist(pPhrase); + if( pPhrase->bIncr ){ + assert( pPhrase->nToken==1 ); + assert( pPhrase->aToken[0].pSegcsr ); + sqlite3Fts3MsrIncrRestart(pPhrase->aToken[0].pSegcsr); + *pRc = fts3EvalPhraseStart(pCsr, 0, pPhrase); + } + + pPhrase->doclist.pNextDocid = 0; + pPhrase->doclist.iDocid = 0; + } + + pExpr->iDocid = 0; + pExpr->bEof = 0; + pExpr->bStart = 0; + + fts3EvalRestart(pCsr, pExpr->pLeft, pRc); + fts3EvalRestart(pCsr, pExpr->pRight, pRc); + } +} + +/* +** After allocating the Fts3Expr.aMI[] array for each phrase in the +** expression rooted at pExpr, the cursor iterates through all rows matched +** by pExpr, calling this function for each row. This function increments +** the values in Fts3Expr.aMI[] according to the position-list currently +** found in Fts3Expr.pPhrase->doclist.pList for each of the phrase +** expression nodes. +*/ +static void fts3EvalUpdateCounts(Fts3Expr *pExpr){ + if( pExpr ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + if( pPhrase && pPhrase->doclist.pList ){ + int iCol = 0; + char *p = pPhrase->doclist.pList; + + assert( *p ); + while( 1 ){ + u8 c = 0; + int iCnt = 0; + while( 0xFE & (*p | c) ){ + if( (c&0x80)==0 ) iCnt++; + c = *p++ & 0x80; + } + + /* aMI[iCol*3 + 1] = Number of occurrences + ** aMI[iCol*3 + 2] = Number of rows containing at least one instance + */ + pExpr->aMI[iCol*3 + 1] += iCnt; + pExpr->aMI[iCol*3 + 2] += (iCnt>0); + if( *p==0x00 ) break; + p++; + p += sqlite3Fts3GetVarint32(p, &iCol); + } + } + + fts3EvalUpdateCounts(pExpr->pLeft); + fts3EvalUpdateCounts(pExpr->pRight); + } +} + +/* +** Expression pExpr must be of type FTSQUERY_PHRASE. +** +** If it is not already allocated and populated, this function allocates and +** populates the Fts3Expr.aMI[] array for expression pExpr. If pExpr is part +** of a NEAR expression, then it also allocates and populates the same array +** for all other phrases that are part of the NEAR expression. +** +** SQLITE_OK is returned if the aMI[] array is successfully allocated and +** populated. Otherwise, if an error occurs, an SQLite error code is returned. +*/ +static int fts3EvalGatherStats( + Fts3Cursor *pCsr, /* Cursor object */ + Fts3Expr *pExpr /* FTSQUERY_PHRASE expression */ +){ + int rc = SQLITE_OK; /* Return code */ + + assert( pExpr->eType==FTSQUERY_PHRASE ); + if( pExpr->aMI==0 ){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + Fts3Expr *pRoot; /* Root of NEAR expression */ + Fts3Expr *p; /* Iterator used for several purposes */ + + sqlite3_int64 iPrevId = pCsr->iPrevId; + sqlite3_int64 iDocid; + u8 bEof; + + /* Find the root of the NEAR expression */ + pRoot = pExpr; + while( pRoot->pParent && pRoot->pParent->eType==FTSQUERY_NEAR ){ + pRoot = pRoot->pParent; + } + iDocid = pRoot->iDocid; + bEof = pRoot->bEof; + assert( pRoot->bStart ); + + /* Allocate space for the aMSI[] array of each FTSQUERY_PHRASE node */ + for(p=pRoot; p; p=p->pLeft){ + Fts3Expr *pE = (p->eType==FTSQUERY_PHRASE?p:p->pRight); + assert( pE->aMI==0 ); + pE->aMI = (u32 *)sqlite3_malloc(pTab->nColumn * 3 * sizeof(u32)); + if( !pE->aMI ) return SQLITE_NOMEM; + memset(pE->aMI, 0, pTab->nColumn * 3 * sizeof(u32)); + } + + fts3EvalRestart(pCsr, pRoot, &rc); + + while( pCsr->isEof==0 && rc==SQLITE_OK ){ + + do { + /* Ensure the %_content statement is reset. */ + if( pCsr->isRequireSeek==0 ) sqlite3_reset(pCsr->pStmt); + assert( sqlite3_data_count(pCsr->pStmt)==0 ); + + /* Advance to the next document */ + fts3EvalNext(pCsr, pRoot, &rc); + pCsr->isEof = pRoot->bEof; + pCsr->isRequireSeek = 1; + pCsr->isMatchinfoNeeded = 1; + pCsr->iPrevId = pRoot->iDocid; + }while( pCsr->isEof==0 + && pRoot->eType==FTSQUERY_NEAR + && fts3EvalLoadDeferred(pCsr, &rc) + ); + + if( rc==SQLITE_OK && pCsr->isEof==0 ){ + fts3EvalUpdateCounts(pRoot); + } + } + + pCsr->isEof = 0; + pCsr->iPrevId = iPrevId; + + if( bEof ){ + pRoot->bEof = bEof; + }else{ + /* Caution: pRoot may iterate through docids in ascending or descending + ** order. For this reason, even though it seems more defensive, the + ** do loop can not be written: + ** + ** do {...} while( pRoot->iDocidbEof==0 ); + }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK ); + fts3EvalLoadDeferred(pCsr, &rc); + } + } + return rc; +} + +/* +** This function is used by the matchinfo() module to query a phrase +** expression node for the following information: +** +** 1. The total number of occurrences of the phrase in each column of +** the FTS table (considering all rows), and +** +** 2. For each column, the number of rows in the table for which the +** column contains at least one instance of the phrase. +** +** If no error occurs, SQLITE_OK is returned and the values for each column +** written into the array aiOut as follows: +** +** aiOut[iCol*3 + 1] = Number of occurrences +** aiOut[iCol*3 + 2] = Number of rows containing at least one instance +** +** Caveats: +** +** * If a phrase consists entirely of deferred tokens, then all output +** values are set to the number of documents in the table. In other +** words we assume that very common tokens occur exactly once in each +** column of each row of the table. +** +** * If a phrase contains some deferred tokens (and some non-deferred +** tokens), count the potential occurrence identified by considering +** the non-deferred tokens instead of actual phrase occurrences. +** +** * If the phrase is part of a NEAR expression, then only phrase instances +** that meet the NEAR constraint are included in the counts. +*/ +int sqlite3Fts3EvalPhraseStats( + Fts3Cursor *pCsr, /* FTS cursor handle */ + Fts3Expr *pExpr, /* Phrase expression */ + u32 *aiOut /* Array to write results into (see above) */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int iCol; + + if( pExpr->bDeferred && pExpr->pParent->eType!=FTSQUERY_NEAR ){ + assert( pCsr->nDoc>0 ); + for(iCol=0; iColnColumn; iCol++){ + aiOut[iCol*3 + 1] = pCsr->nDoc; + aiOut[iCol*3 + 2] = pCsr->nDoc; + } + }else{ + rc = fts3EvalGatherStats(pCsr, pExpr); + if( rc==SQLITE_OK ){ + assert( pExpr->aMI ); + for(iCol=0; iColnColumn; iCol++){ + aiOut[iCol*3 + 1] = pExpr->aMI[iCol*3 + 1]; + aiOut[iCol*3 + 2] = pExpr->aMI[iCol*3 + 2]; + } + } + } + + return rc; +} + +/* +** The expression pExpr passed as the second argument to this function +** must be of type FTSQUERY_PHRASE. +** +** The returned value is either NULL or a pointer to a buffer containing +** a position-list indicating the occurrences of the phrase in column iCol +** of the current row. +** +** More specifically, the returned buffer contains 1 varint for each +** occurence of the phrase in the column, stored using the normal (delta+2) +** compression and is terminated by either an 0x01 or 0x00 byte. For example, +** if the requested column contains "a b X c d X X" and the position-list +** for 'X' is requested, the buffer returned may contain: +** +** 0x04 0x05 0x03 0x01 or 0x04 0x05 0x03 0x00 +** +** This function works regardless of whether or not the phrase is deferred, +** incremental, or neither. +*/ +char *sqlite3Fts3EvalPhrasePoslist( + Fts3Cursor *pCsr, /* FTS3 cursor object */ + Fts3Expr *pExpr, /* Phrase to return doclist for */ + int iCol /* Column to return position list for */ +){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + char *pIter = pPhrase->doclist.pList; + int iThis; + + assert( iCol>=0 && iColnColumn ); + if( !pIter + || pExpr->bEof + || pExpr->iDocid!=pCsr->iPrevId + || (pPhrase->iColumnnColumn && pPhrase->iColumn!=iCol) + ){ + return 0; + } + + assert( pPhrase->doclist.nList>0 ); + if( *pIter==0x01 ){ + pIter++; + pIter += sqlite3Fts3GetVarint32(pIter, &iThis); + }else{ + iThis = 0; + } + while( iThisdoclist, and +** * any Fts3MultiSegReader objects held by phrase tokens. +*/ +void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){ + if( pPhrase ){ + int i; + sqlite3_free(pPhrase->doclist.aAll); + fts3EvalZeroPoslist(pPhrase); + memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); + for(i=0; inToken; i++){ + fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr); + pPhrase->aToken[i].pSegcsr = 0; + } + } +} + #endif diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index bb21165ffc..acdaf3d5bb 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -11,7 +11,6 @@ ****************************************************************************** ** */ - #ifndef _FTSINT_H #define _FTSINT_H @@ -19,6 +18,16 @@ # define NDEBUG 1 #endif +/* +** FTS4 is really an extension for FTS3. It is enabled using the +** SQLITE_ENABLE_FTS3 macro. But to avoid confusion we also all +** the SQLITE_ENABLE_FTS4 macro to serve as an alisse for SQLITE_ENABLE_FTS3. +*/ +#if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3) +# define SQLITE_ENABLE_FTS3 +#endif + +#ifdef SQLITE_ENABLE_FTS3 #include "sqlite3.h" #include "fts3_tokenizer.h" #include "fts3_hash.h" @@ -47,12 +56,35 @@ */ #define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0]))) + +#ifndef MIN +# define MIN(x,y) ((x)<(y)?(x):(y)) +#endif + /* ** Maximum length of a varint encoded integer. The varint format is different ** from that used by SQLite, so the maximum length is 10, not 9. */ #define FTS3_VARINT_MAX 10 +/* +** FTS4 virtual tables may maintain multiple indexes - one index of all terms +** in the document set and zero or more prefix indexes. All indexes are stored +** as one or more b+-trees in the %_segments and %_segdir tables. +** +** It is possible to determine which index a b+-tree belongs to based on the +** value stored in the "%_segdir.level" column. Given this value L, the index +** that the b+-tree belongs to is (L<<10). In other words, all b+-trees with +** level values between 0 and 1023 (inclusive) belong to index 0, all levels +** between 1024 and 2047 to index 1, and so on. +** +** It is considered impossible for an index to use more than 1024 levels. In +** theory though this may happen, but only after at least +** (FTS3_MERGE_COUNT^1024) separate flushes of the pending-terms tables. +*/ +#define FTS3_SEGDIR_MAXLEVEL 1024 +#define FTS3_SEGDIR_MAXLEVEL_STR "1024" + /* ** The testcase() macro is only used by the amalgamation. If undefined, ** make it a no-op. @@ -124,10 +156,11 @@ typedef struct Fts3Expr Fts3Expr; typedef struct Fts3Phrase Fts3Phrase; typedef struct Fts3PhraseToken Fts3PhraseToken; +typedef struct Fts3Doclist Fts3Doclist; typedef struct Fts3SegFilter Fts3SegFilter; typedef struct Fts3DeferredToken Fts3DeferredToken; typedef struct Fts3SegReader Fts3SegReader; -typedef struct Fts3SegReaderCursor Fts3SegReaderCursor; +typedef struct Fts3MultiSegReader Fts3MultiSegReader; /* ** A connection to a fulltext index is an instance of the following @@ -148,7 +181,7 @@ struct Fts3Table { /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. */ - sqlite3_stmt *aStmt[24]; + sqlite3_stmt *aStmt[27]; char *zReadExprlist; char *zWriteExprlist; @@ -156,21 +189,33 @@ struct Fts3Table { int nNodeSize; /* Soft limit for node size */ u8 bHasStat; /* True if %_stat table exists */ u8 bHasDocsize; /* True if %_docsize table exists */ + u8 bDescIdx; /* True if doclists are in reverse order */ int nPgsz; /* Page size for host database */ char *zSegmentsTbl; /* Name of %_segments table */ sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ - /* The following hash table is used to buffer pending index updates during + /* TODO: Fix the first paragraph of this comment. + ** + ** The following hash table is used to buffer pending index updates during ** transactions. Variable nPendingData estimates the memory size of the ** pending data, including hash table overhead, but not malloc overhead. ** When nPendingData exceeds nMaxPendingData, the buffer is flushed ** automatically. Variable iPrevDocid is the docid of the most recently ** inserted record. + ** + ** A single FTS4 table may have multiple full-text indexes. For each index + ** there is an entry in the aIndex[] array. Index 0 is an index of all the + ** terms that appear in the document set. Each subsequent index in aIndex[] + ** is an index of prefixes of a specific length. */ - int nMaxPendingData; - int nPendingData; - sqlite_int64 iPrevDocid; - Fts3Hash pendingTerms; + int nIndex; /* Size of aIndex[] */ + struct Fts3Index { + int nPrefix; /* Prefix length (0 for main terms index) */ + Fts3Hash hPending; /* Pending terms table for this index */ + } *aIndex; + int nMaxPendingData; /* Max pending data before flush to disk */ + int nPendingData; /* Current bytes of pending data */ + sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */ #if defined(SQLITE_DEBUG) /* State variables used for validating that the transaction control @@ -201,9 +246,10 @@ struct Fts3Cursor { char *pNextId; /* Pointer into the body of aDoclist */ char *aDoclist; /* List of docids for full-text queries */ int nDoclist; /* Size of buffer at aDoclist */ - int desc; /* True to sort in descending order */ + u8 bDesc; /* True to sort in descending order */ int eEvalmode; /* An FTS3_EVAL_XX constant */ int nRowAvg; /* Average size of database rows, in pages */ + int nDoc; /* Documents in table */ int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */ u32 *aMatchinfo; /* Information about most recent match */ @@ -234,47 +280,70 @@ struct Fts3Cursor { #define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */ #define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */ + +struct Fts3Doclist { + char *aAll; /* Array containing doclist (or NULL) */ + int nAll; /* Size of a[] in bytes */ + char *pNextDocid; /* Pointer to next docid */ + + sqlite3_int64 iDocid; /* Current docid (if pList!=0) */ + int bFreeList; /* True if pList should be sqlite3_free()d */ + char *pList; /* Pointer to position list following iDocid */ + int nList; /* Length of position list */ +} doclist; + /* ** A "phrase" is a sequence of one or more tokens that must match in ** sequence. A single token is the base case and the most common case. ** For a sequence of tokens contained in double-quotes (i.e. "one two three") ** nToken will be the number of tokens in the string. -** -** The nDocMatch and nMatch variables contain data that may be used by the -** matchinfo() function. They are populated when the full-text index is -** queried for hits on the phrase. If one or more tokens in the phrase -** are deferred, the nDocMatch and nMatch variables are populated based -** on the assumption that the */ struct Fts3PhraseToken { char *z; /* Text of the token */ int n; /* Number of bytes in buffer z */ int isPrefix; /* True if token ends with a "*" character */ - int bFulltext; /* True if full-text index was used */ - Fts3SegReaderCursor *pSegcsr; /* Segment-reader for this token */ + + /* Variables above this point are populated when the expression is + ** parsed (by code in fts3_expr.c). Below this point the variables are + ** used when evaluating the expression. */ Fts3DeferredToken *pDeferred; /* Deferred token object for this token */ + Fts3MultiSegReader *pSegcsr; /* Segment-reader for this token */ }; struct Fts3Phrase { - /* Variables populated by fts3_expr.c when parsing a MATCH expression */ + /* Cache of doclist for this phrase. */ + Fts3Doclist doclist; + int bIncr; /* True if doclist is loaded incrementally */ + int iDoclistToken; + + /* Variables below this point are populated by fts3_expr.c when parsing + ** a MATCH expression. Everything above is part of the evaluation phase. + */ int nToken; /* Number of tokens in the phrase */ int iColumn; /* Index of column this phrase must match */ - int isNot; /* Phrase prefixed by unary not (-) operator */ Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */ }; /* ** A tree of these objects forms the RHS of a MATCH operator. ** -** If Fts3Expr.eType is either FTSQUERY_NEAR or FTSQUERY_PHRASE and isLoaded -** is true, then aDoclist points to a malloced buffer, size nDoclist bytes, -** containing the results of the NEAR or phrase query in FTS3 doclist -** format. As usual, the initial "Length" field found in doclists stored -** on disk is omitted from this buffer. +** If Fts3Expr.eType is FTSQUERY_PHRASE and isLoaded is true, then aDoclist +** points to a malloced buffer, size nDoclist bytes, containing the results +** of this phrase query in FTS3 doclist format. As usual, the initial +** "Length" field found in doclists stored on disk is omitted from this +** buffer. ** -** Variable pCurrent always points to the start of a docid field within -** aDoclist. Since the doclist is usually scanned in docid order, this can -** be used to accelerate seeking to the required docid within the doclist. +** Variable aMI is used only for FTSQUERY_NEAR nodes to store the global +** matchinfo data. If it is not NULL, it points to an array of size nCol*3, +** where nCol is the number of columns in the queried FTS table. The array +** is populated as follows: +** +** aMI[iCol*3 + 0] = Undefined +** aMI[iCol*3 + 1] = Number of occurrences +** aMI[iCol*3 + 2] = Number of rows containing at least one instance +** +** The aMI array is allocated using sqlite3_malloc(). It should be freed +** when the expression node is. */ struct Fts3Expr { int eType; /* One of the FTSQUERY_XXX values defined below */ @@ -284,12 +353,13 @@ struct Fts3Expr { Fts3Expr *pRight; /* Right operand */ Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */ - int isLoaded; /* True if aDoclist/nDoclist are initialized. */ - char *aDoclist; /* Buffer containing doclist */ - int nDoclist; /* Size of aDoclist in bytes */ + /* The following are used by the fts3_eval.c module. */ + sqlite3_int64 iDocid; /* Current docid */ + u8 bEof; /* True this expression is at EOF already */ + u8 bStart; /* True if iDocid is valid */ + u8 bDeferred; /* True if this expression is entirely deferred */ - sqlite3_int64 iCurrent; - char *pCurrent; + u32 *aMI; }; /* @@ -317,12 +387,12 @@ void sqlite3Fts3PendingTermsClear(Fts3Table *); int sqlite3Fts3Optimize(Fts3Table *); int sqlite3Fts3SegReaderNew(int, sqlite3_int64, sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); -int sqlite3Fts3SegReaderPending(Fts3Table*,const char*,int,int,Fts3SegReader**); +int sqlite3Fts3SegReaderPending( + Fts3Table*,int,const char*,int,int,Fts3SegReader**); void sqlite3Fts3SegReaderFree(Fts3SegReader *); -int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *); -int sqlite3Fts3AllSegdirs(Fts3Table*, int, sqlite3_stmt **); +int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, sqlite3_stmt **); int sqlite3Fts3ReadLock(Fts3Table *); -int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*); +int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*); int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **); int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **); @@ -331,17 +401,18 @@ void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *); int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); -char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *, int *); void sqlite3Fts3SegmentsClose(Fts3Table *); -#define FTS3_SEGCURSOR_PENDING -1 -#define FTS3_SEGCURSOR_ALL -2 +/* Special values interpreted by sqlite3SegReaderCursor() */ +#define FTS3_SEGCURSOR_PENDING -1 +#define FTS3_SEGCURSOR_ALL -2 + +int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Fts3SegFilter*); +int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *); +void sqlite3Fts3SegReaderFinish(Fts3MultiSegReader *); -int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3SegReaderCursor*, Fts3SegFilter*); -int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3SegReaderCursor *); -void sqlite3Fts3SegReaderFinish(Fts3SegReaderCursor *); int sqlite3Fts3SegReaderCursor( - Fts3Table *, int, const char *, int, int, int, Fts3SegReaderCursor *); + Fts3Table *, int, int, const char *, int, int, int, Fts3MultiSegReader *); /* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ #define FTS3_SEGMENT_REQUIRE_POS 0x00000001 @@ -358,7 +429,7 @@ struct Fts3SegFilter { int flags; }; -struct Fts3SegReaderCursor { +struct Fts3MultiSegReader { /* Used internally by sqlite3Fts3SegReaderXXX() calls */ Fts3SegReader **apSegment; /* Array of Fts3SegReader objects */ int nSegment; /* Size of apSegment array */ @@ -367,8 +438,12 @@ struct Fts3SegReaderCursor { char *aBuffer; /* Buffer to merge doclists in */ int nBuffer; /* Allocated size of aBuffer[] in bytes */ - /* Cost of running this iterator. Used by fts3.c only. */ - int nCost; + int iColFilter; /* If >=0, filter for this column */ + int bRestart; + + /* Used by fts3.c only. */ + int nCost; /* Cost of running iterator */ + int bLookup; /* True if a lookup of a single entry. */ /* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */ char *zTerm; /* Pointer to term buffer */ @@ -383,11 +458,9 @@ int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); int sqlite3Fts3GetVarint32(const char *, int *); int sqlite3Fts3VarintLen(sqlite3_uint64); void sqlite3Fts3Dequote(char *); +void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*); -char *sqlite3Fts3FindPositions(Fts3Cursor *, Fts3Expr *, sqlite3_int64, int); -int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *, Fts3Expr *); -int sqlite3Fts3ExprLoadFtDoclist(Fts3Cursor *, Fts3Expr *, char **, int *); -int sqlite3Fts3ExprNearTrim(Fts3Expr *, Fts3Expr *, int); +int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *); /* fts3_tokenizer.c */ const char *sqlite3Fts3NextToken(const char *, int *); @@ -417,4 +490,28 @@ int sqlite3Fts3InitTerm(sqlite3 *db); /* fts3_aux.c */ int sqlite3Fts3InitAux(sqlite3 *db); +int sqlite3Fts3TermSegReaderCursor( + Fts3Cursor *pCsr, /* Virtual table cursor handle */ + const char *zTerm, /* Term to query for */ + int nTerm, /* Size of zTerm in bytes */ + int isPrefix, /* True for a prefix search */ + Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */ +); + +void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *); + +int sqlite3Fts3EvalStart(Fts3Cursor *, Fts3Expr *, int); +int sqlite3Fts3EvalNext(Fts3Cursor *pCsr); + +int sqlite3Fts3MsrIncrStart( + Fts3Table*, Fts3MultiSegReader*, int, const char*, int); +int sqlite3Fts3MsrIncrNext( + Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *); +char *sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol); +int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); +int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); + +int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *); + +#endif /* SQLITE_ENABLE_FTS3 */ #endif /* _FTSINT_H */ diff --git a/ext/fts3/fts3_aux.c b/ext/fts3/fts3_aux.c index cd035d2a30..ada85d796b 100644 --- a/ext/fts3/fts3_aux.c +++ b/ext/fts3/fts3_aux.c @@ -11,10 +11,9 @@ ****************************************************************************** ** */ - +#include "fts3Int.h" #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) -#include "fts3Int.h" #include #include @@ -28,7 +27,7 @@ struct Fts3auxTable { struct Fts3auxCursor { sqlite3_vtab_cursor base; /* Base class used by SQLite core */ - Fts3SegReaderCursor csr; /* Must be right after "base" */ + Fts3MultiSegReader csr; /* Must be right after "base" */ Fts3SegFilter filter; char *zStop; int nStop; /* Byte-length of string zStop */ @@ -96,6 +95,7 @@ static int fts3auxConnectMethod( p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1]; p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1]; p->pFts3Tab->db = db; + p->pFts3Tab->nIndex = 1; memcpy((char *)p->pFts3Tab->zDb, zDb, nDb); memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3); @@ -376,7 +376,7 @@ static int fts3auxFilterMethod( if( pCsr->zStop==0 ) return SQLITE_NOMEM; } - rc = sqlite3Fts3SegReaderCursor(pFts3, FTS3_SEGCURSOR_ALL, + rc = sqlite3Fts3SegReaderCursor(pFts3, 0, FTS3_SEGCURSOR_ALL, pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr ); if( rc==SQLITE_OK ){ diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index 43f6d84a84..7eb2962d44 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -15,6 +15,7 @@ ** syntax is relatively simple, the whole tokenizer/parser system is ** hand-coded. */ +#include "fts3Int.h" #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) /* @@ -77,16 +78,24 @@ int sqlite3_fts3_enable_parentheses = 0; */ #define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10 -#include "fts3Int.h" #include #include +/* +** isNot: +** This variable is used by function getNextNode(). When getNextNode() is +** called, it sets ParseContext.isNot to true if the 'next node' is a +** FTSQUERY_PHRASE with a unary "-" attached to it. i.e. "mysql" in the +** FTS3 query "sqlite -mysql". Otherwise, ParseContext.isNot is set to +** zero. +*/ typedef struct ParseContext ParseContext; struct ParseContext { sqlite3_tokenizer *pTokenizer; /* Tokenizer module */ const char **azCol; /* Array of column names for fts3 table */ int nCol; /* Number of entries in azCol[] */ int iDefaultCol; /* Default column to query */ + int isNot; /* True if getNextNode() sees a unary - */ sqlite3_context *pCtx; /* Write error message here */ int nNest; /* Number of nested brackets */ }; @@ -172,7 +181,7 @@ static int getNextToken( iEnd++; } if( !sqlite3_fts3_enable_parentheses && iStart>0 && z[iStart-1]=='-' ){ - pRet->pPhrase->isNot = 1; + pParse->isNot = 1; } } nConsumed = iEnd; @@ -224,36 +233,55 @@ static int getNextString( char *zTemp = 0; int nTemp = 0; + const int nSpace = sizeof(Fts3Expr) + sizeof(Fts3Phrase); + int nToken = 0; + + /* The final Fts3Expr data structure, including the Fts3Phrase, + ** Fts3PhraseToken structures token buffers are all stored as a single + ** allocation so that the expression can be freed with a single call to + ** sqlite3_free(). Setting this up requires a two pass approach. + ** + ** The first pass, in the block below, uses a tokenizer cursor to iterate + ** through the tokens in the expression. This pass uses fts3ReallocOrFree() + ** to assemble data in two dynamic buffers: + ** + ** Buffer p: Points to the Fts3Expr structure, followed by the Fts3Phrase + ** structure, followed by the array of Fts3PhraseToken + ** structures. This pass only populates the Fts3PhraseToken array. + ** + ** Buffer zTemp: Contains copies of all tokens. + ** + ** The second pass, in the block that begins "if( rc==SQLITE_DONE )" below, + ** appends buffer zTemp to buffer p, and fills in the Fts3Expr and Fts3Phrase + ** structures. + */ rc = pModule->xOpen(pTokenizer, zInput, nInput, &pCursor); if( rc==SQLITE_OK ){ int ii; pCursor->pTokenizer = pTokenizer; for(ii=0; rc==SQLITE_OK; ii++){ - const char *zToken; - int nToken, iBegin, iEnd, iPos; - rc = pModule->xNext(pCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos); + const char *zByte; + int nByte, iBegin, iEnd, iPos; + rc = pModule->xNext(pCursor, &zByte, &nByte, &iBegin, &iEnd, &iPos); if( rc==SQLITE_OK ){ - int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase); - p = fts3ReallocOrFree(p, nByte+ii*sizeof(Fts3PhraseToken)); - zTemp = fts3ReallocOrFree(zTemp, nTemp + nToken); - if( !p || !zTemp ){ - goto no_mem; - } - if( ii==0 ){ - memset(p, 0, nByte); - p->pPhrase = (Fts3Phrase *)&p[1]; - } - p->pPhrase = (Fts3Phrase *)&p[1]; - memset(&p->pPhrase->aToken[ii], 0, sizeof(Fts3PhraseToken)); - p->pPhrase->nToken = ii+1; - p->pPhrase->aToken[ii].n = nToken; - memcpy(&zTemp[nTemp], zToken, nToken); - nTemp += nToken; - if( iEndpPhrase->aToken[ii].isPrefix = 1; - }else{ - p->pPhrase->aToken[ii].isPrefix = 0; - } + Fts3PhraseToken *pToken; + + p = fts3ReallocOrFree(p, nSpace + ii*sizeof(Fts3PhraseToken)); + if( !p ) goto no_mem; + + zTemp = fts3ReallocOrFree(zTemp, nTemp + nByte); + if( !zTemp ) goto no_mem; + + assert( nToken==ii ); + pToken = &((Fts3Phrase *)(&p[1]))->aToken[ii]; + memset(pToken, 0, sizeof(Fts3PhraseToken)); + + memcpy(&zTemp[nTemp], zByte, nByte); + nTemp += nByte; + + pToken->n = nByte; + pToken->isPrefix = (iEndpPhrase->nToken-1):0) * sizeof(Fts3PhraseToken); - p = fts3ReallocOrFree(p, nByte + nTemp); - if( !p ){ - goto no_mem; - } - if( zTemp ){ - zNew = &(((char *)p)[nByte]); - memcpy(zNew, zTemp, nTemp); - }else{ - memset(p, 0, nByte+nTemp); - } - p->pPhrase = (Fts3Phrase *)&p[1]; - for(jj=0; jjpPhrase->nToken; jj++){ - p->pPhrase->aToken[jj].z = &zNew[nNew]; - nNew += p->pPhrase->aToken[jj].n; - } - sqlite3_free(zTemp); + char *zBuf = 0; + + p = fts3ReallocOrFree(p, nSpace + nToken*sizeof(Fts3PhraseToken) + nTemp); + if( !p ) goto no_mem; + memset(p, 0, (char *)&(((Fts3Phrase *)&p[1])->aToken[0])-(char *)p); p->eType = FTSQUERY_PHRASE; + p->pPhrase = (Fts3Phrase *)&p[1]; p->pPhrase->iColumn = pParse->iDefaultCol; + p->pPhrase->nToken = nToken; + + zBuf = (char *)&p->pPhrase->aToken[nToken]; + memcpy(zBuf, zTemp, nTemp); + sqlite3_free(zTemp); + + for(jj=0; jjpPhrase->nToken; jj++){ + p->pPhrase->aToken[jj].z = zBuf; + zBuf += p->pPhrase->aToken[jj].n; + } rc = SQLITE_OK; } @@ -341,6 +365,8 @@ static int getNextNode( const char *zInput = z; int nInput = n; + pParse->isNot = 0; + /* Skip over any whitespace before checking for a keyword, an open or ** close bracket, or a quoted string. */ @@ -559,7 +585,7 @@ static int fts3ExprParse( int isPhrase; if( !sqlite3_fts3_enable_parentheses - && p->eType==FTSQUERY_PHRASE && p->pPhrase->isNot + && p->eType==FTSQUERY_PHRASE && pParse->isNot ){ /* Create an implicit NOT operator. */ Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr)); @@ -577,7 +603,6 @@ static int fts3ExprParse( p = pPrev; }else{ int eType = p->eType; - assert( eType!=FTSQUERY_PHRASE || !p->pPhrase->isNot ); isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft); /* The isRequirePhrase variable is set to true if a phrase or @@ -740,9 +765,11 @@ int sqlite3Fts3ExprParse( */ void sqlite3Fts3ExprFree(Fts3Expr *p){ if( p ){ + assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 ); sqlite3Fts3ExprFree(p->pLeft); sqlite3Fts3ExprFree(p->pRight); - sqlite3_free(p->aDoclist); + sqlite3Fts3EvalPhraseCleanup(p->pPhrase); + sqlite3_free(p->aMI); sqlite3_free(p); } } @@ -800,7 +827,7 @@ static char *exprToString(Fts3Expr *pExpr, char *zBuf){ Fts3Phrase *pPhrase = pExpr->pPhrase; int i; zBuf = sqlite3_mprintf( - "%zPHRASE %d %d", zBuf, pPhrase->iColumn, pPhrase->isNot); + "%zPHRASE %d 0", zBuf, pPhrase->iColumn); for(i=0; zBuf && inToken; i++){ zBuf = sqlite3_mprintf("%z %.*s%s", zBuf, pPhrase->aToken[i].n, pPhrase->aToken[i].z, diff --git a/ext/fts3/fts3_hash.c b/ext/fts3/fts3_hash.c index 98be529605..b7c3e8b5a7 100644 --- a/ext/fts3/fts3_hash.c +++ b/ext/fts3/fts3_hash.c @@ -23,6 +23,7 @@ ** * The FTS3 module is being built into the core of ** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). */ +#include "fts3Int.h" #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) #include diff --git a/ext/fts3/fts3_icu.c b/ext/fts3/fts3_icu.c index 85390d3b06..a10a55d67b 100644 --- a/ext/fts3/fts3_icu.c +++ b/ext/fts3/fts3_icu.c @@ -10,10 +10,8 @@ ** ************************************************************************* ** This file implements a tokenizer for fts3 based on the ICU library. -** -** $Id: fts3_icu.c,v 1.3 2008/09/01 18:34:20 danielk1977 Exp $ */ - +#include "fts3Int.h" #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) #ifdef SQLITE_ENABLE_ICU diff --git a/ext/fts3/fts3_porter.c b/ext/fts3/fts3_porter.c index 27f9cf39d7..148c570088 100644 --- a/ext/fts3/fts3_porter.c +++ b/ext/fts3/fts3_porter.c @@ -22,9 +22,8 @@ ** * The FTS3 module is being built into the core of ** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) - #include "fts3Int.h" +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) #include #include diff --git a/ext/fts3/fts3_snippet.c b/ext/fts3/fts3_snippet.c index 8bf9ed2d5a..0da370c7ea 100644 --- a/ext/fts3/fts3_snippet.c +++ b/ext/fts3/fts3_snippet.c @@ -11,9 +11,9 @@ ****************************************************************************** */ +#include "fts3Int.h" #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) -#include "fts3Int.h" #include #include @@ -176,51 +176,6 @@ static int fts3ExprIterate( return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx); } -/* -** The argument to this function is always a phrase node. Its doclist -** (Fts3Expr.aDoclist[]) and the doclists associated with all phrase nodes -** to the left of this one in the query tree have already been loaded. -** -** If this phrase node is part of a series of phrase nodes joined by -** NEAR operators (and is not the left-most of said series), then elements are -** removed from the phrases doclist consistent with the NEAR restriction. If -** required, elements may be removed from the doclists of phrases to the -** left of this one that are part of the same series of NEAR operator -** connected phrases. -** -** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK. -*/ -static int fts3ExprNearTrim(Fts3Expr *pExpr){ - int rc = SQLITE_OK; - Fts3Expr *pParent = pExpr->pParent; - - assert( pExpr->eType==FTSQUERY_PHRASE ); - while( rc==SQLITE_OK - && pParent - && pParent->eType==FTSQUERY_NEAR - && pParent->pRight==pExpr - ){ - /* This expression (pExpr) is the right-hand-side of a NEAR operator. - ** Find the expression to the left of the same operator. - */ - int nNear = pParent->nNear; - Fts3Expr *pLeft = pParent->pLeft; - - if( pLeft->eType!=FTSQUERY_PHRASE ){ - assert( pLeft->eType==FTSQUERY_NEAR ); - assert( pLeft->pRight->eType==FTSQUERY_PHRASE ); - pLeft = pLeft->pRight; - } - - rc = sqlite3Fts3ExprNearTrim(pLeft, pExpr, nNear); - - pExpr = pLeft; - pParent = pExpr->pParent; - } - - return rc; -} - /* ** This is an fts3ExprIterate() callback used while loading the doclists ** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also @@ -228,20 +183,13 @@ static int fts3ExprNearTrim(Fts3Expr *pExpr){ */ static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ int rc = SQLITE_OK; + Fts3Phrase *pPhrase = pExpr->pPhrase; LoadDoclistCtx *p = (LoadDoclistCtx *)ctx; UNUSED_PARAMETER(iPhrase); p->nPhrase++; - p->nToken += pExpr->pPhrase->nToken; - - if( pExpr->isLoaded==0 ){ - rc = sqlite3Fts3ExprLoadDoclist(p->pCsr, pExpr); - pExpr->isLoaded = 1; - if( rc==SQLITE_OK ){ - rc = fts3ExprNearTrim(pExpr); - } - } + p->nToken += pPhrase->nToken; return rc; } @@ -415,7 +363,7 @@ static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){ pPhrase->nToken = pExpr->pPhrase->nToken; - pCsr = sqlite3Fts3FindPositions(p->pCsr, pExpr, p->pCsr->iPrevId, p->iCol); + pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol); if( pCsr ){ int iFirst = 0; pPhrase->pList = pCsr; @@ -772,26 +720,6 @@ static int fts3ColumnlistCount(char **ppCollist){ return nEntry; } -static void fts3LoadColumnlistCounts(char **pp, u32 *aOut, int isGlobal){ - char *pCsr = *pp; - while( *pCsr ){ - int nHit; - sqlite3_int64 iCol = 0; - if( *pCsr==0x01 ){ - pCsr++; - pCsr += sqlite3Fts3GetVarint(pCsr, &iCol); - } - nHit = fts3ColumnlistCount(&pCsr); - assert( nHit>0 ); - if( isGlobal ){ - aOut[iCol*3+1]++; - } - aOut[iCol*3] += nHit; - } - pCsr++; - *pp = pCsr; -} - /* ** fts3ExprIterate() callback used to collect the "global" matchinfo stats ** for a single query. @@ -825,48 +753,9 @@ static int fts3ExprGlobalHitsCb( void *pCtx /* Pointer to MatchInfo structure */ ){ MatchInfo *p = (MatchInfo *)pCtx; - Fts3Cursor *pCsr = p->pCursor; - char *pIter; - char *pEnd; - char *pFree = 0; - u32 *aOut = &p->aMatchinfo[3*iPhrase*p->nCol]; - - assert( pExpr->isLoaded ); - assert( pExpr->eType==FTSQUERY_PHRASE ); - - if( pCsr->pDeferred ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - int ii; - for(ii=0; iinToken; ii++){ - if( pPhrase->aToken[ii].bFulltext ) break; - } - if( iinToken ){ - int nFree = 0; - int rc = sqlite3Fts3ExprLoadFtDoclist(pCsr, pExpr, &pFree, &nFree); - if( rc!=SQLITE_OK ) return rc; - pIter = pFree; - pEnd = &pFree[nFree]; - }else{ - int iCol; /* Column index */ - for(iCol=0; iColnCol; iCol++){ - aOut[iCol*3 + 1] = (u32)p->nDoc; - aOut[iCol*3 + 2] = (u32)p->nDoc; - } - return SQLITE_OK; - } - }else{ - pIter = pExpr->aDoclist; - pEnd = &pExpr->aDoclist[pExpr->nDoclist]; - } - - /* Fill in the global hit count matrix row for this phrase. */ - while( pIterpCursor, pExpr, &p->aMatchinfo[3*iPhrase*p->nCol] + ); } /* @@ -883,14 +772,13 @@ static int fts3ExprLocalHitsCb( int iStart = iPhrase * p->nCol * 3; int i; - for(i=0; inCol; i++) p->aMatchinfo[iStart+i*3] = 0; - - if( pExpr->aDoclist ){ + for(i=0; inCol; i++){ char *pCsr; - - pCsr = sqlite3Fts3FindPositions(p->pCursor, pExpr, p->pCursor->iPrevId, -1); + pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, i); if( pCsr ){ - fts3LoadColumnlistCounts(&pCsr, &p->aMatchinfo[iStart], 0); + p->aMatchinfo[iStart+i*3] = fts3ColumnlistCount(&pCsr); + }else{ + p->aMatchinfo[iStart+i*3] = 0; } } @@ -976,9 +864,8 @@ static int fts3MatchinfoSelectDoctotal( typedef struct LcsIterator LcsIterator; struct LcsIterator { Fts3Expr *pExpr; /* Pointer to phrase expression */ - char *pRead; /* Cursor used to iterate through aDoclist */ int iPosOffset; /* Tokens count up to end of this phrase */ - int iCol; /* Current column number */ + char *pRead; /* Cursor used to iterate through aDoclist */ int iPos; /* Current position */ }; @@ -1009,17 +896,10 @@ static int fts3LcsIteratorAdvance(LcsIterator *pIter){ int rc = 0; pRead += sqlite3Fts3GetVarint(pRead, &iRead); - if( iRead==0 ){ - pIter->iCol = LCS_ITERATOR_FINISHED; + if( iRead==0 || iRead==1 ){ + pRead = 0; rc = 1; }else{ - if( iRead==1 ){ - pRead += sqlite3Fts3GetVarint(pRead, &iRead); - pIter->iCol = (int)iRead; - pIter->iPos = pIter->iPosOffset; - pRead += sqlite3Fts3GetVarint(pRead, &iRead); - rc = 1; - } pIter->iPos += (int)(iRead-2); } @@ -1051,42 +931,34 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){ if( !aIter ) return SQLITE_NOMEM; memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase); (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter); + for(i=0; inPhrase; i++){ LcsIterator *pIter = &aIter[i]; nToken -= pIter->pExpr->pPhrase->nToken; pIter->iPosOffset = nToken; - pIter->pRead = sqlite3Fts3FindPositions(pCsr,pIter->pExpr,pCsr->iPrevId,-1); - if( pIter->pRead ){ - pIter->iPos = pIter->iPosOffset; - fts3LcsIteratorAdvance(&aIter[i]); - }else{ - pIter->iCol = LCS_ITERATOR_FINISHED; - } } for(iCol=0; iColnCol; iCol++){ int nLcs = 0; /* LCS value for this column */ int nLive = 0; /* Number of iterators in aIter not at EOF */ - /* Loop through the iterators in aIter[]. Set nLive to the number of - ** iterators that point to a position-list corresponding to column iCol. - */ for(i=0; inPhrase; i++){ - assert( aIter[i].iCol>=iCol ); - if( aIter[i].iCol==iCol ) nLive++; + LcsIterator *pIt = &aIter[i]; + pIt->pRead = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iCol); + if( pIt->pRead ){ + pIt->iPos = pIt->iPosOffset; + fts3LcsIteratorAdvance(&aIter[i]); + nLive++; + } } - /* The following loop runs until all iterators in aIter[] have finished - ** iterating through positions in column iCol. Exactly one of the - ** iterators is advanced each time the body of the loop is run. - */ while( nLive>0 ){ LcsIterator *pAdv = 0; /* The iterator to advance by one position */ int nThisLcs = 0; /* LCS for the current iterator positions */ for(i=0; inPhrase; i++){ LcsIterator *pIter = &aIter[i]; - if( iCol!=pIter->iCol ){ + if( pIter->pRead==0 ){ /* This iterator is already at EOF for this column. */ nThisLcs = 0; }else{ @@ -1426,7 +1298,7 @@ static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){ int iPos = 0; /* First position in position-list */ UNUSED_PARAMETER(iPhrase); - pList = sqlite3Fts3FindPositions(p->pCsr, pExpr, p->iDocid, p->iCol); + pList = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol); nTerm = pExpr->pPhrase->nToken; if( pList ){ fts3GetDeltaPosition(&pList, &iPos); diff --git a/ext/fts3/fts3_term.c b/ext/fts3/fts3_term.c index e207ff870d..da9bf4543b 100644 --- a/ext/fts3/fts3_term.c +++ b/ext/fts3/fts3_term.c @@ -15,10 +15,10 @@ ** access to the full-text index of an FTS table. */ +#include "fts3Int.h" #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) #ifdef SQLITE_TEST -#include "fts3Int.h" #include #include @@ -27,12 +27,13 @@ typedef struct Fts3termCursor Fts3termCursor; struct Fts3termTable { sqlite3_vtab base; /* Base class used by SQLite core */ + int iIndex; /* Index for Fts3Table.aIndex[] */ Fts3Table *pFts3Tab; }; struct Fts3termCursor { sqlite3_vtab_cursor base; /* Base class used by SQLite core */ - Fts3SegReaderCursor csr; /* Must be right after "base" */ + Fts3MultiSegReader csr; /* Must be right after "base" */ Fts3SegFilter filter; int isEof; /* True if cursor is at EOF */ @@ -56,7 +57,7 @@ struct Fts3termCursor { */ static int fts3termConnectMethod( sqlite3 *db, /* Database connection */ - void *pUnused, /* Unused */ + void *pCtx, /* Non-zero for an fts4prefix table */ int argc, /* Number of elements in argv array */ const char * const *argv, /* xCreate/xConnect argument array */ sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ @@ -69,8 +70,12 @@ static int fts3termConnectMethod( int nByte; /* Bytes of space to allocate here */ int rc; /* value returned by declare_vtab() */ Fts3termTable *p; /* Virtual table object to return */ + int iIndex = 0; - UNUSED_PARAMETER(pUnused); + if( argc==5 ){ + iIndex = atoi(argv[4]); + argc--; + } /* The user should specify a single argument - the name of an fts3 table. */ if( argc!=4 ){ @@ -97,6 +102,8 @@ static int fts3termConnectMethod( p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1]; p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1]; p->pFts3Tab->db = db; + p->pFts3Tab->nIndex = iIndex+1; + p->iIndex = iIndex; memcpy((char *)p->pFts3Tab->zDb, zDb, nDb); memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3); @@ -244,7 +251,8 @@ static int fts3termFilterMethod( sqlite3_value **apVal /* Arguments for the indexing scheme */ ){ Fts3termCursor *pCsr = (Fts3termCursor *)pCursor; - Fts3Table *pFts3 = ((Fts3termTable *)pCursor->pVtab)->pFts3Tab; + Fts3termTable *p = (Fts3termTable *)pCursor->pVtab; + Fts3Table *pFts3 = p->pFts3Tab; int rc; UNUSED_PARAMETER(nVal); @@ -262,7 +270,7 @@ static int fts3termFilterMethod( pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; pCsr->filter.flags |= FTS3_SEGMENT_SCAN; - rc = sqlite3Fts3SegReaderCursor(pFts3, FTS3_SEGCURSOR_ALL, + rc = sqlite3Fts3SegReaderCursor(pFts3, p->iIndex, FTS3_SEGCURSOR_ALL, pCsr->filter.zTerm, pCsr->filter.nTerm, 0, 1, &pCsr->csr ); if( rc==SQLITE_OK ){ diff --git a/ext/fts3/fts3_test.c b/ext/fts3/fts3_test.c new file mode 100644 index 0000000000..704c7502af --- /dev/null +++ b/ext/fts3/fts3_test.c @@ -0,0 +1,320 @@ +/* +** 2011 Jun 13 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file is not part of the production FTS code. It is only used for +** testing. It contains a Tcl command that can be used to test if a document +** matches an FTS NEAR expression. +*/ + +#include +#include +#include + +#define NM_MAX_TOKEN 12 + +typedef struct NearPhrase NearPhrase; +typedef struct NearDocument NearDocument; +typedef struct NearToken NearToken; + +struct NearDocument { + int nToken; /* Length of token in bytes */ + NearToken *aToken; /* Token array */ +}; + +struct NearToken { + int n; /* Length of token in bytes */ + const char *z; /* Pointer to token string */ +}; + +struct NearPhrase { + int nNear; /* Preceding NEAR value */ + int nToken; /* Number of tokens in this phrase */ + NearToken aToken[NM_MAX_TOKEN]; /* Array of tokens in this phrase */ +}; + +static int nm_phrase_match( + NearPhrase *p, + NearToken *aToken +){ + int ii; + + for(ii=0; iinToken; ii++){ + NearToken *pToken = &p->aToken[ii]; + if( pToken->n>0 && pToken->z[pToken->n-1]=='*' ){ + if( aToken[ii].n<(pToken->n-1) ) return 0; + if( memcmp(aToken[ii].z, pToken->z, pToken->n-1) ) return 0; + }else{ + if( aToken[ii].n!=pToken->n ) return 0; + if( memcmp(aToken[ii].z, pToken->z, pToken->n) ) return 0; + } + } + + return 1; +} + +static int nm_near_chain( + int iDir, /* Direction to iterate through aPhrase[] */ + NearDocument *pDoc, /* Document to match against */ + int iPos, /* Position at which iPhrase was found */ + int nPhrase, /* Size of phrase array */ + NearPhrase *aPhrase, /* Phrase array */ + int iPhrase /* Index of phrase found */ +){ + int iStart; + int iStop; + int ii; + int nNear; + int iPhrase2; + NearPhrase *p; + NearPhrase *pPrev; + + assert( iDir==1 || iDir==-1 ); + + if( iDir==1 ){ + if( (iPhrase+1)==nPhrase ) return 1; + nNear = aPhrase[iPhrase+1].nNear; + }else{ + if( iPhrase==0 ) return 1; + nNear = aPhrase[iPhrase].nNear; + } + pPrev = &aPhrase[iPhrase]; + iPhrase2 = iPhrase+iDir; + p = &aPhrase[iPhrase2]; + + iStart = iPos - nNear - p->nToken; + iStop = iPos + nNear + pPrev->nToken; + + if( iStart<0 ) iStart = 0; + if( iStop > pDoc->nToken - p->nToken ) iStop = pDoc->nToken - p->nToken; + + for(ii=iStart; ii<=iStop; ii++){ + if( nm_phrase_match(p, &pDoc->aToken[ii]) ){ + if( nm_near_chain(iDir, pDoc, ii, nPhrase, aPhrase, iPhrase2) ) return 1; + } + } + + return 0; +} + +static int nm_match_count( + NearDocument *pDoc, /* Document to match against */ + int nPhrase, /* Size of phrase array */ + NearPhrase *aPhrase, /* Phrase array */ + int iPhrase /* Index of phrase to count matches for */ +){ + int nOcc = 0; + int ii; + NearPhrase *p = &aPhrase[iPhrase]; + + for(ii=0; ii<(pDoc->nToken + 1 - p->nToken); ii++){ + if( nm_phrase_match(p, &pDoc->aToken[ii]) ){ + /* Test forward NEAR chain (i>iPhrase) */ + if( 0==nm_near_chain(1, pDoc, ii, nPhrase, aPhrase, iPhrase) ) continue; + + /* Test reverse NEAR chain (iNM_MAX_TOKEN ){ + Tcl_AppendResult(interp, "Too many tokens in phrase", 0); + rc = TCL_ERROR; + goto near_match_out; + } + for(jj=0; jjz = Tcl_GetStringFromObj(apToken[jj], &pT->n); + } + aPhrase[ii].nToken = nToken; + } + for(ii=1; ii0)); + + near_match_out: + ckfree((char *)aPhrase); + ckfree((char *)doc.aToken); + return rc; +} + +/* +** Tclcmd: fts3_configure_incr_load ?CHUNKSIZE THRESHOLD? +** +** Normally, FTS uses hard-coded values to determine the minimum doclist +** size eligible for incremental loading, and the size of the chunks loaded +** when a doclist is incrementally loaded. This command allows the built-in +** values to be overridden for testing purposes. +** +** If present, the first argument is the chunksize in bytes to load doclists +** in. The second argument is the minimum doclist size in bytes to use +** incremental loading with. +** +** Whether or not the arguments are present, this command returns a list of +** two integers - the initial chunksize and threshold when the command is +** invoked. This can be used to restore the default behaviour after running +** tests. For example: +** +** # Override incr-load settings for testing: +** set cfg [fts3_configure_incr_load $new_chunksize $new_threshold] +** +** .... run tests .... +** +** # Restore initial incr-load settings: +** eval fts3_configure_incr_load $cfg +*/ +static int fts3_configure_incr_load_cmd( + ClientData clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ +#ifdef SQLITE_ENABLE_FTS3 + extern int test_fts3_node_chunksize; + extern int test_fts3_node_chunk_threshold; + int iArg1; + int iArg2; + Tcl_Obj *pRet; + + if( objc!=1 && objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "?CHUNKSIZE THRESHOLD?"); + return TCL_ERROR; + } + + pRet = Tcl_NewObj(); + Tcl_IncrRefCount(pRet); + Tcl_ListObjAppendElement( + interp, pRet, Tcl_NewIntObj(test_fts3_node_chunksize)); + Tcl_ListObjAppendElement( + interp, pRet, Tcl_NewIntObj(test_fts3_node_chunk_threshold)); + + if( objc==3 ){ + int iArg1; + int iArg2; + if( Tcl_GetIntFromObj(interp, objv[1], &iArg1) + || Tcl_GetIntFromObj(interp, objv[2], &iArg2) + ){ + Tcl_DecrRefCount(pRet); + return TCL_ERROR; + } + test_fts3_node_chunksize = iArg1; + test_fts3_node_chunk_threshold = iArg2; + } + + Tcl_SetObjResult(interp, pRet); + Tcl_DecrRefCount(pRet); +#endif + return TCL_OK; +} + +int Sqlitetestfts3_Init(Tcl_Interp *interp){ + Tcl_CreateObjCommand(interp, "fts3_near_match", fts3_near_match_cmd, 0, 0); + Tcl_CreateObjCommand(interp, + "fts3_configure_incr_load", fts3_configure_incr_load_cmd, 0, 0 + ); + return TCL_OK; +} diff --git a/ext/fts3/fts3_tokenizer.c b/ext/fts3/fts3_tokenizer.c index aa28933439..87a9a3d430 100644 --- a/ext/fts3/fts3_tokenizer.c +++ b/ext/fts3/fts3_tokenizer.c @@ -23,14 +23,14 @@ ** * The FTS3 module is being built into the core of ** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) - #include "sqlite3ext.h" #ifndef SQLITE_CORE SQLITE_EXTENSION_INIT1 #endif - #include "fts3Int.h" + +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + #include #include diff --git a/ext/fts3/fts3_tokenizer1.c b/ext/fts3/fts3_tokenizer1.c index 432c35d1a2..d11a499766 100644 --- a/ext/fts3/fts3_tokenizer1.c +++ b/ext/fts3/fts3_tokenizer1.c @@ -22,9 +22,8 @@ ** * The FTS3 module is being built into the core of ** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) - #include "fts3Int.h" +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) #include #include diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index d00632a75d..da84d6ca61 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -17,9 +17,9 @@ ** code in fts3.c. */ +#include "fts3Int.h" #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) -#include "fts3Int.h" #include #include #include @@ -36,14 +36,40 @@ */ #define FTS3_NODE_PADDING (FTS3_VARINT_MAX*2) +/* +** Under certain circumstances, b-tree nodes (doclists) can be loaded into +** memory incrementally instead of all at once. This can be a big performance +** win (reduced IO and CPU) if SQLite stops calling the virtual table xNext() +** method before retrieving all query results (as may happen, for example, +** if a query has a LIMIT clause). +** +** Incremental loading is used for b-tree nodes FTS3_NODE_CHUNK_THRESHOLD +** bytes and larger. Nodes are loaded in chunks of FTS3_NODE_CHUNKSIZE bytes. +** The code is written so that the hard lower-limit for each of these values +** is 1. Clearly such small values would be inefficient, but can be useful +** for testing purposes. +** +** If this module is built with SQLITE_TEST defined, these constants may +** be overridden at runtime for testing purposes. File fts3_test.c contains +** a Tcl interface to read and write the values. +*/ +#ifdef SQLITE_TEST +int test_fts3_node_chunksize = (4*1024); +int test_fts3_node_chunk_threshold = (4*1024)*4; +# define FTS3_NODE_CHUNKSIZE test_fts3_node_chunksize +# define FTS3_NODE_CHUNK_THRESHOLD test_fts3_node_chunk_threshold +#else +# define FTS3_NODE_CHUNKSIZE (4*1024) +# define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4) +#endif + typedef struct PendingList PendingList; typedef struct SegmentNode SegmentNode; typedef struct SegmentWriter SegmentWriter; /* -** Data structure used while accumulating terms in the pending-terms hash -** table. The hash table entry maps from term (a string) to a malloc'd -** instance of this structure. +** An instance of the following data structure is used to build doclists +** incrementally. See function fts3PendingListAppend() for details. */ struct PendingList { int nData; @@ -74,7 +100,6 @@ struct Fts3DeferredToken { ** ** sqlite3Fts3SegReaderNew() ** sqlite3Fts3SegReaderFree() -** sqlite3Fts3SegReaderCost() ** sqlite3Fts3SegReaderIterate() ** ** Methods used to manipulate Fts3SegReader structures: @@ -93,6 +118,9 @@ struct Fts3SegReader { char *aNode; /* Pointer to node data (or NULL) */ int nNode; /* Size of buffer at aNode (or 0) */ + int nPopulate; /* If >0, bytes of buffer aNode[] loaded */ + sqlite3_blob *pBlob; /* If not NULL, blob handle to read node */ + Fts3HashElem **ppNextElem; /* Variables set by fts3SegReaderNext(). These may be read directly @@ -106,8 +134,11 @@ struct Fts3SegReader { char *aDoclist; /* Pointer to doclist of current entry */ int nDoclist; /* Size of doclist in current entry */ - /* The following variables are used to iterate through the current doclist */ + /* The following variables are used by fts3SegReaderNextDocid() to iterate + ** through the current doclist (aDoclist/nDoclist). + */ char *pOffsetList; + int nOffsetList; /* For descending pending seg-readers only */ sqlite3_int64 iDocid; }; @@ -145,6 +176,14 @@ struct SegmentWriter { ** fts3NodeAddTerm() ** fts3NodeWrite() ** fts3NodeFree() +** +** When a b+tree is written to the database (either as a result of a merge +** or the pending-terms table being flushed), leaves are written into the +** database file as soon as they are completely populated. The interior of +** the tree is assembled in memory and written out only once all leaves have +** been populated and stored. This is Ok, as the b+-tree fanout is usually +** very large, meaning that the interior of the tree consumes relatively +** little memory. */ struct SegmentNode { SegmentNode *pParent; /* Parent node (or NULL for root node) */ @@ -175,10 +214,10 @@ struct SegmentNode { #define SQL_NEXT_SEGMENTS_ID 10 #define SQL_INSERT_SEGDIR 11 #define SQL_SELECT_LEVEL 12 -#define SQL_SELECT_ALL_LEVEL 13 +#define SQL_SELECT_LEVEL_RANGE 13 #define SQL_SELECT_LEVEL_COUNT 14 -#define SQL_SELECT_SEGDIR_COUNT_MAX 15 -#define SQL_DELETE_SEGDIR_BY_LEVEL 16 +#define SQL_SELECT_SEGDIR_MAX_LEVEL 15 +#define SQL_DELETE_SEGDIR_LEVEL 16 #define SQL_DELETE_SEGMENTS_RANGE 17 #define SQL_CONTENT_INSERT 18 #define SQL_DELETE_DOCSIZE 19 @@ -187,6 +226,11 @@ struct SegmentNode { #define SQL_SELECT_DOCTOTAL 22 #define SQL_REPLACE_DOCTOTAL 23 +#define SQL_SELECT_ALL_PREFIX_LEVEL 24 +#define SQL_DELETE_ALL_TERMS_SEGDIR 25 + +#define SQL_DELETE_SEGDIR_RANGE 26 + /* ** This function is used to obtain an SQLite prepared statement handle ** for the statement identified by the second argument. If successful, @@ -222,10 +266,11 @@ static int fts3SqlStmt( /* 12 */ "SELECT idx, start_block, leaves_end_block, end_block, root " "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC", /* 13 */ "SELECT idx, start_block, leaves_end_block, end_block, root " - "FROM %Q.'%q_segdir' ORDER BY level DESC, idx ASC", + "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?" + "ORDER BY level DESC, idx ASC", /* 14 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?", -/* 15 */ "SELECT count(*), max(level) FROM %Q.'%q_segdir'", +/* 15 */ "SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", /* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?", /* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?", @@ -235,6 +280,11 @@ static int fts3SqlStmt( /* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", /* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0", /* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)", +/* 24 */ "", +/* 25 */ "", + +/* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", + }; int rc = SQLITE_OK; sqlite3_stmt *pStmt; @@ -390,14 +440,32 @@ int sqlite3Fts3ReadLock(Fts3Table *p){ ** 3: end_block ** 4: root */ -int sqlite3Fts3AllSegdirs(Fts3Table *p, int iLevel, sqlite3_stmt **ppStmt){ +int sqlite3Fts3AllSegdirs( + Fts3Table *p, /* FTS3 table */ + int iIndex, /* Index for p->aIndex[] */ + int iLevel, /* Level to select */ + sqlite3_stmt **ppStmt /* OUT: Compiled statement */ +){ int rc; sqlite3_stmt *pStmt = 0; + + assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel>=0 ); + assert( iLevel=0 && iIndexnIndex ); + if( iLevel<0 ){ - rc = fts3SqlStmt(p, SQL_SELECT_ALL_LEVEL, &pStmt, 0); + /* "SELECT * FROM %_segdir WHERE level BETWEEN ? AND ? ORDER BY ..." */ + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL); + sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL-1); + } }else{ + /* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); - if( rc==SQLITE_OK ) sqlite3_bind_int(pStmt, 1, iLevel); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pStmt, 1, iLevel+iIndex*FTS3_SEGDIR_MAXLEVEL); + } } *ppStmt = pStmt; return rc; @@ -512,6 +580,47 @@ static int fts3PendingListAppend( return 0; } +/* +** Free a PendingList object allocated by fts3PendingListAppend(). +*/ +static void fts3PendingListDelete(PendingList *pList){ + sqlite3_free(pList); +} + +/* +** Add an entry to one of the pending-terms hash tables. +*/ +static int fts3PendingTermsAddOne( + Fts3Table *p, + int iCol, + int iPos, + Fts3Hash *pHash, /* Pending terms hash table to add entry to */ + const char *zToken, + int nToken +){ + PendingList *pList; + int rc = SQLITE_OK; + + pList = (PendingList *)fts3HashFind(pHash, zToken, nToken); + if( pList ){ + p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem)); + } + if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){ + if( pList==fts3HashInsert(pHash, zToken, nToken, pList) ){ + /* Malloc failed while inserting the new entry. This can only + ** happen if there was no previous entry for this token. + */ + assert( 0==fts3HashFind(pHash, zToken, nToken) ); + sqlite3_free(pList); + rc = SQLITE_NOMEM; + } + } + if( rc==SQLITE_OK ){ + p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem)); + } + return rc; +} + /* ** Tokenize the nul-terminated string zText and add all tokens to the ** pending-terms hash-table. The docid used is that currently stored in @@ -560,8 +669,7 @@ static int fts3PendingTermsAdd( while( SQLITE_OK==rc && SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos)) ){ - PendingList *pList; - + int i; if( iPos>=nWord ) nWord = iPos+1; /* Positions cannot be negative; we use -1 as a terminator internally. @@ -572,22 +680,19 @@ static int fts3PendingTermsAdd( break; } - pList = (PendingList *)fts3HashFind(&p->pendingTerms, zToken, nToken); - if( pList ){ - p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem)); - } - if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){ - if( pList==fts3HashInsert(&p->pendingTerms, zToken, nToken, pList) ){ - /* Malloc failed while inserting the new entry. This can only - ** happen if there was no previous entry for this token. - */ - assert( 0==fts3HashFind(&p->pendingTerms, zToken, nToken) ); - sqlite3_free(pList); - rc = SQLITE_NOMEM; - } - } - if( rc==SQLITE_OK ){ - p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem)); + /* Add the term to the terms index */ + rc = fts3PendingTermsAddOne( + p, iCol, iPos, &p->aIndex[0].hPending, zToken, nToken + ); + + /* Add the term to each of the prefix indexes that it is not too + ** short for. */ + for(i=1; rc==SQLITE_OK && inIndex; i++){ + struct Fts3Index *pIndex = &p->aIndex[i]; + if( nTokennPrefix ) continue; + rc = fts3PendingTermsAddOne( + p, iCol, iPos, &pIndex->hPending, zToken, pIndex->nPrefix + ); } } @@ -617,14 +722,19 @@ static int fts3PendingTermsDocid(Fts3Table *p, sqlite_int64 iDocid){ } /* -** Discard the contents of the pending-terms hash table. +** Discard the contents of the pending-terms hash tables. */ void sqlite3Fts3PendingTermsClear(Fts3Table *p){ - Fts3HashElem *pElem; - for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){ - sqlite3_free(fts3HashData(pElem)); + int i; + for(i=0; inIndex; i++){ + Fts3HashElem *pElem; + Fts3Hash *pHash = &p->aIndex[i].hPending; + for(pElem=fts3HashFirst(pHash); pElem; pElem=fts3HashNext(pElem)){ + PendingList *pList = (PendingList *)fts3HashData(pElem); + fts3PendingListDelete(pList); + } + fts3HashClear(pHash); } - fts3HashClear(&p->pendingTerms); p->nPendingData = 0; } @@ -780,7 +890,7 @@ static void fts3DeleteTerms( ** Forward declaration to account for the circular dependency between ** functions fts3SegmentMerge() and fts3AllocateSegdirIdx(). */ -static int fts3SegmentMerge(Fts3Table *, int); +static int fts3SegmentMerge(Fts3Table *, int, int); /* ** This function allocates a new level iLevel index in the segdir table. @@ -797,7 +907,12 @@ static int fts3SegmentMerge(Fts3Table *, int); ** If successful, *piIdx is set to the allocated index slot and SQLITE_OK ** returned. Otherwise, an SQLite error code is returned. */ -static int fts3AllocateSegdirIdx(Fts3Table *p, int iLevel, int *piIdx){ +static int fts3AllocateSegdirIdx( + Fts3Table *p, + int iIndex, /* Index for p->aIndex */ + int iLevel, + int *piIdx +){ int rc; /* Return Code */ sqlite3_stmt *pNextIdx; /* Query for next idx at level iLevel */ int iNext = 0; /* Result of query pNextIdx */ @@ -805,7 +920,7 @@ static int fts3AllocateSegdirIdx(Fts3Table *p, int iLevel, int *piIdx){ /* Set variable iNext to the next available segdir index at level iLevel. */ rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pNextIdx, 1, iLevel); + sqlite3_bind_int(pNextIdx, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel); if( SQLITE_ROW==sqlite3_step(pNextIdx) ){ iNext = sqlite3_column_int(pNextIdx, 0); } @@ -819,7 +934,7 @@ static int fts3AllocateSegdirIdx(Fts3Table *p, int iLevel, int *piIdx){ ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext. */ if( iNext>=FTS3_MERGE_COUNT ){ - rc = fts3SegmentMerge(p, iLevel); + rc = fts3SegmentMerge(p, iIndex, iLevel); *piIdx = 0; }else{ *piIdx = iNext; @@ -860,7 +975,8 @@ int sqlite3Fts3ReadBlock( Fts3Table *p, /* FTS3 table handle */ sqlite3_int64 iBlockid, /* Access the row with blockid=$iBlockid */ char **paBlob, /* OUT: Blob data in malloc'd buffer */ - int *pnBlob /* OUT: Size of blob data */ + int *pnBlob, /* OUT: Size of blob data */ + int *pnLoad /* OUT: Bytes actually loaded */ ){ int rc; /* Return code */ @@ -881,11 +997,16 @@ int sqlite3Fts3ReadBlock( if( rc==SQLITE_OK ){ int nByte = sqlite3_blob_bytes(p->pSegments); + *pnBlob = nByte; if( paBlob ){ char *aByte = sqlite3_malloc(nByte + FTS3_NODE_PADDING); if( !aByte ){ rc = SQLITE_NOMEM; }else{ + if( pnLoad && nByte>(FTS3_NODE_CHUNK_THRESHOLD) ){ + nByte = FTS3_NODE_CHUNKSIZE; + *pnLoad = nByte; + } rc = sqlite3_blob_read(p->pSegments, aByte, nByte, 0); memset(&aByte[nByte], 0, FTS3_NODE_PADDING); if( rc!=SQLITE_OK ){ @@ -895,7 +1016,6 @@ int sqlite3Fts3ReadBlock( } *paBlob = aByte; } - *pnBlob = nByte; } return rc; @@ -909,13 +1029,55 @@ void sqlite3Fts3SegmentsClose(Fts3Table *p){ sqlite3_blob_close(p->pSegments); p->pSegments = 0; } + +static int fts3SegReaderIncrRead(Fts3SegReader *pReader){ + int nRead; /* Number of bytes to read */ + int rc; /* Return code */ + + nRead = MIN(pReader->nNode - pReader->nPopulate, FTS3_NODE_CHUNKSIZE); + rc = sqlite3_blob_read( + pReader->pBlob, + &pReader->aNode[pReader->nPopulate], + nRead, + pReader->nPopulate + ); + + if( rc==SQLITE_OK ){ + pReader->nPopulate += nRead; + memset(&pReader->aNode[pReader->nPopulate], 0, FTS3_NODE_PADDING); + if( pReader->nPopulate==pReader->nNode ){ + sqlite3_blob_close(pReader->pBlob); + pReader->pBlob = 0; + pReader->nPopulate = 0; + } + } + return rc; +} + +static int fts3SegReaderRequire(Fts3SegReader *pReader, char *pFrom, int nByte){ + int rc = SQLITE_OK; + assert( !pReader->pBlob + || (pFrom>=pReader->aNode && pFrom<&pReader->aNode[pReader->nNode]) + ); + while( pReader->pBlob && rc==SQLITE_OK + && (pFrom - pReader->aNode + nByte)>pReader->nPopulate + ){ + rc = fts3SegReaderIncrRead(pReader); + } + return rc; +} /* ** Move the iterator passed as the first argument to the next term in the ** segment. If successful, SQLITE_OK is returned. If there is no next term, ** SQLITE_DONE. Otherwise, an SQLite error code. */ -static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ +static int fts3SegReaderNext( + Fts3Table *p, + Fts3SegReader *pReader, + int bIncr +){ + int rc; /* Return code of various sub-routines */ char *pNext; /* Cursor variable */ int nPrefix; /* Number of bytes in term prefix */ int nSuffix; /* Number of bytes in term suffix */ @@ -927,7 +1089,6 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ } if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){ - int rc; /* Return code from Fts3ReadBlock() */ if( fts3SegReaderIsPending(pReader) ){ Fts3HashElem *pElem = *(pReader->ppNextElem); @@ -947,6 +1108,8 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ if( !fts3SegReaderIsRootOnly(pReader) ){ sqlite3_free(pReader->aNode); + sqlite3_blob_close(pReader->pBlob); + pReader->pBlob = 0; } pReader->aNode = 0; @@ -958,15 +1121,25 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ } rc = sqlite3Fts3ReadBlock( - p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode + p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode, + (bIncr ? &pReader->nPopulate : 0) ); if( rc!=SQLITE_OK ) return rc; + assert( pReader->pBlob==0 ); + if( bIncr && pReader->nPopulatenNode ){ + pReader->pBlob = p->pSegments; + p->pSegments = 0; + } pNext = pReader->aNode; } + + assert( !fts3SegReaderIsPending(pReader) ); + + rc = fts3SegReaderRequire(pReader, pNext, FTS3_VARINT_MAX*2); + if( rc!=SQLITE_OK ) return rc; /* Because of the FTS3_NODE_PADDING bytes of padding, the following is - ** safe (no risk of overread) even if the node data is corrupted. - */ + ** safe (no risk of overread) even if the node data is corrupted. */ pNext += sqlite3Fts3GetVarint32(pNext, &nPrefix); pNext += sqlite3Fts3GetVarint32(pNext, &nSuffix); if( nPrefix<0 || nSuffix<=0 @@ -984,6 +1157,10 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ pReader->zTerm = zNew; pReader->nTermAlloc = nNew; } + + rc = fts3SegReaderRequire(pReader, pNext, nSuffix+FTS3_VARINT_MAX); + if( rc!=SQLITE_OK ) return rc; + memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix); pReader->nTerm = nPrefix+nSuffix; pNext += nSuffix; @@ -996,7 +1173,7 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ ** of these statements is untrue, then the data structure is corrupt. */ if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode] - || pReader->aDoclist[pReader->nDoclist-1] + || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1]) ){ return SQLITE_CORRUPT_VTAB; } @@ -1007,12 +1184,26 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ ** Set the SegReader to point to the first docid in the doclist associated ** with the current term. */ -static void fts3SegReaderFirstDocid(Fts3SegReader *pReader){ - int n; +static int fts3SegReaderFirstDocid(Fts3Table *pTab, Fts3SegReader *pReader){ + int rc = SQLITE_OK; assert( pReader->aDoclist ); assert( !pReader->pOffsetList ); - n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid); - pReader->pOffsetList = &pReader->aDoclist[n]; + if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){ + u8 bEof = 0; + pReader->iDocid = 0; + pReader->nOffsetList = 0; + sqlite3Fts3DoclistPrev(0, + pReader->aDoclist, pReader->nDoclist, &pReader->pOffsetList, + &pReader->iDocid, &pReader->nOffsetList, &bEof + ); + }else{ + rc = fts3SegReaderRequire(pReader, pReader->aDoclist, FTS3_VARINT_MAX); + if( rc==SQLITE_OK ){ + int n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid); + pReader->pOffsetList = &pReader->aDoclist[n]; + } + } + return rc; } /* @@ -1025,128 +1216,125 @@ static void fts3SegReaderFirstDocid(Fts3SegReader *pReader){ ** *pnOffsetList is set to the length of the set of column-offset ** lists, not including the nul-terminator byte. For example: */ -static void fts3SegReaderNextDocid( - Fts3SegReader *pReader, - char **ppOffsetList, - int *pnOffsetList +static int fts3SegReaderNextDocid( + Fts3Table *pTab, + Fts3SegReader *pReader, /* Reader to advance to next docid */ + char **ppOffsetList, /* OUT: Pointer to current position-list */ + int *pnOffsetList /* OUT: Length of *ppOffsetList in bytes */ ){ + int rc = SQLITE_OK; char *p = pReader->pOffsetList; char c = 0; - /* Pointer p currently points at the first byte of an offset list. The - ** following two lines advance it to point one byte past the end of - ** the same offset list. - */ - while( *p | c ) c = *p++ & 0x80; - p++; + assert( p ); - /* If required, populate the output variables with a pointer to and the - ** size of the previous offset-list. - */ - if( ppOffsetList ){ - *ppOffsetList = pReader->pOffsetList; - *pnOffsetList = (int)(p - pReader->pOffsetList - 1); - } - - /* If there are no more entries in the doclist, set pOffsetList to - ** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and - ** Fts3SegReader.pOffsetList to point to the next offset list before - ** returning. - */ - if( p>=&pReader->aDoclist[pReader->nDoclist] ){ - pReader->pOffsetList = 0; + if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){ + /* A pending-terms seg-reader for an FTS4 table that uses order=desc. + ** Pending-terms doclists are always built up in ascending order, so + ** we have to iterate through them backwards here. */ + u8 bEof = 0; + if( ppOffsetList ){ + *ppOffsetList = pReader->pOffsetList; + *pnOffsetList = pReader->nOffsetList - 1; + } + sqlite3Fts3DoclistPrev(0, + pReader->aDoclist, pReader->nDoclist, &p, &pReader->iDocid, + &pReader->nOffsetList, &bEof + ); + if( bEof ){ + pReader->pOffsetList = 0; + }else{ + pReader->pOffsetList = p; + } }else{ - sqlite3_int64 iDelta; - pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta); - pReader->iDocid += iDelta; + char *pEnd = &pReader->aDoclist[pReader->nDoclist]; + + /* Pointer p currently points at the first byte of an offset list. The + ** following block advances it to point one byte past the end of + ** the same offset list. */ + while( 1 ){ + + /* The following line of code (and the "p++" below the while() loop) is + ** normally all that is required to move pointer p to the desired + ** position. The exception is if this node is being loaded from disk + ** incrementally and pointer "p" now points to the first byte passed + ** the populated part of pReader->aNode[]. + */ + while( *p | c ) c = *p++ & 0x80; + assert( *p==0 ); + + if( pReader->pBlob==0 || p<&pReader->aNode[pReader->nPopulate] ) break; + rc = fts3SegReaderIncrRead(pReader); + if( rc!=SQLITE_OK ) return rc; + } + p++; + + /* If required, populate the output variables with a pointer to and the + ** size of the previous offset-list. + */ + if( ppOffsetList ){ + *ppOffsetList = pReader->pOffsetList; + *pnOffsetList = (int)(p - pReader->pOffsetList - 1); + } + + while( p=pEnd ){ + pReader->pOffsetList = 0; + }else{ + rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX); + if( rc==SQLITE_OK ){ + sqlite3_int64 iDelta; + pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta); + if( pTab->bDescIdx ){ + pReader->iDocid -= iDelta; + }else{ + pReader->iDocid += iDelta; + } + } + } } + + return SQLITE_OK; } -/* -** This function is called to estimate the amount of data that will be -** loaded from the disk If SegReaderIterate() is called on this seg-reader, -** in units of average document size. -** -** This can be used as follows: If the caller has a small doclist that -** contains references to N documents, and is considering merging it with -** a large doclist (size X "average documents"), it may opt not to load -** the large doclist if X>N. -*/ -int sqlite3Fts3SegReaderCost( - Fts3Cursor *pCsr, /* FTS3 cursor handle */ - Fts3SegReader *pReader, /* Segment-reader handle */ - int *pnCost /* IN/OUT: Number of bytes read */ + +int sqlite3Fts3MsrOvfl( + Fts3Cursor *pCsr, + Fts3MultiSegReader *pMsr, + int *pnOvfl ){ Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; - int rc = SQLITE_OK; /* Return code */ - int nCost = 0; /* Cost in bytes to return */ - int pgsz = p->nPgsz; /* Database page size */ + int nOvfl = 0; + int ii; + int rc = SQLITE_OK; + int pgsz = p->nPgsz; - /* If this seg-reader is reading the pending-terms table, or if all data - ** for the segment is stored on the root page of the b-tree, then the cost - ** is zero. In this case all required data is already in main memory. - */ - if( p->bHasStat - && !fts3SegReaderIsPending(pReader) - && !fts3SegReaderIsRootOnly(pReader) - ){ - int nBlob = 0; - sqlite3_int64 iBlock; + assert( p->bHasStat ); + assert( pgsz>0 ); - if( pCsr->nRowAvg==0 ){ - /* The average document size, which is required to calculate the cost - ** of each doclist, has not yet been determined. Read the required - ** data from the %_stat table to calculate it. - ** - ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 - ** varints, where nCol is the number of columns in the FTS3 table. - ** The first varint is the number of documents currently stored in - ** the table. The following nCol varints contain the total amount of - ** data stored in all rows of each column of the table, from left - ** to right. - */ - sqlite3_stmt *pStmt; - sqlite3_int64 nDoc = 0; - sqlite3_int64 nByte = 0; - const char *pEnd; - const char *a; - - rc = sqlite3Fts3SelectDoctotal(p, &pStmt); - if( rc!=SQLITE_OK ) return rc; - a = sqlite3_column_blob(pStmt, 0); - assert( a ); - - pEnd = &a[sqlite3_column_bytes(pStmt, 0)]; - a += sqlite3Fts3GetVarint(a, &nDoc); - while( anRowAvg = (int)(((nByte / nDoc) + pgsz) / pgsz); - assert( pCsr->nRowAvg>0 ); - rc = sqlite3_reset(pStmt); - if( rc!=SQLITE_OK ) return rc; - } - - /* Assume that a blob flows over onto overflow pages if it is larger - ** than (pgsz-35) bytes in size (the file-format documentation - ** confirms this). - */ - for(iBlock=pReader->iStartBlock; iBlock<=pReader->iLeafEndBlock; iBlock++){ - rc = sqlite3Fts3ReadBlock(p, iBlock, 0, &nBlob); - if( rc!=SQLITE_OK ) break; - if( (nBlob+35)>pgsz ){ - int nOvfl = (nBlob + 34)/pgsz; - nCost += ((nOvfl + pCsr->nRowAvg - 1)/pCsr->nRowAvg); + for(ii=0; rc==SQLITE_OK && iinSegment; ii++){ + Fts3SegReader *pReader = pMsr->apSegment[ii]; + if( !fts3SegReaderIsPending(pReader) + && !fts3SegReaderIsRootOnly(pReader) + ){ + int jj; + for(jj=pReader->iStartBlock; jj<=pReader->iLeafEndBlock; jj++){ + int nBlob; + rc = sqlite3Fts3ReadBlock(p, jj, 0, &nBlob, 0); + if( rc!=SQLITE_OK ) break; + if( (nBlob+35)>pgsz ){ + nOvfl += (nBlob + 34)/pgsz; + } } } } - - *pnCost += nCost; + *pnOvfl = nOvfl; return rc; } @@ -1159,6 +1347,7 @@ void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){ sqlite3_free(pReader->zTerm); if( !fts3SegReaderIsRootOnly(pReader) ){ sqlite3_free(pReader->aNode); + sqlite3_blob_close(pReader->pBlob); } } sqlite3_free(pReader); @@ -1235,24 +1424,42 @@ static int fts3CompareElemByTerm(const void *lhs, const void *rhs){ /* ** This function is used to allocate an Fts3SegReader that iterates through ** a subset of the terms stored in the Fts3Table.pendingTerms array. +** +** If the isPrefixIter parameter is zero, then the returned SegReader iterates +** through each term in the pending-terms table. Or, if isPrefixIter is +** non-zero, it iterates through each term and its prefixes. For example, if +** the pending terms hash table contains the terms "sqlite", "mysql" and +** "firebird", then the iterator visits the following 'terms' (in the order +** shown): +** +** f fi fir fire fireb firebi firebir firebird +** m my mys mysq mysql +** s sq sql sqli sqlit sqlite +** +** Whereas if isPrefixIter is zero, the terms visited are: +** +** firebird mysql sqlite */ int sqlite3Fts3SegReaderPending( Fts3Table *p, /* Virtual table handle */ + int iIndex, /* Index for p->aIndex */ const char *zTerm, /* Term to search for */ int nTerm, /* Size of buffer zTerm */ - int isPrefix, /* True for a term-prefix query */ + int bPrefix, /* True for a prefix iterator */ Fts3SegReader **ppReader /* OUT: SegReader for pending-terms */ ){ Fts3SegReader *pReader = 0; /* Fts3SegReader object to return */ Fts3HashElem **aElem = 0; /* Array of term hash entries to scan */ int nElem = 0; /* Size of array at aElem */ int rc = SQLITE_OK; /* Return Code */ + Fts3Hash *pHash; - if( isPrefix ){ + pHash = &p->aIndex[iIndex].hPending; + if( bPrefix ){ int nAlloc = 0; /* Size of allocated array at aElem */ Fts3HashElem *pE = 0; /* Iterator variable */ - for(pE=fts3HashFirst(&p->pendingTerms); pE; pE=fts3HashNext(pE)){ + for(pE=fts3HashFirst(pHash); pE; pE=fts3HashNext(pE)){ char *zKey = (char *)fts3HashKey(pE); int nKey = fts3HashKeysize(pE); if( nTerm==0 || (nKey>=nTerm && 0==memcmp(zKey, zTerm, nTerm)) ){ @@ -1269,6 +1476,7 @@ int sqlite3Fts3SegReaderPending( } aElem = aElem2; } + aElem[nElem++] = pE; } } @@ -1282,7 +1490,9 @@ int sqlite3Fts3SegReaderPending( } }else{ - Fts3HashElem *pE = fts3HashFindElem(&p->pendingTerms, zTerm, nTerm); + /* The query is a simple term lookup that matches at most one term in + ** the index. All that is required is a straight hash-lookup. */ + Fts3HashElem *pE = fts3HashFindElem(pHash, zTerm, nTerm); if( pE ){ aElem = &pE; nElem = 1; @@ -1302,7 +1512,7 @@ int sqlite3Fts3SegReaderPending( } } - if( isPrefix ){ + if( bPrefix ){ sqlite3_free(aElem); } *ppReader = pReader; @@ -1366,6 +1576,18 @@ static int fts3SegReaderDoclistCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ assert( pLhs->aNode && pRhs->aNode ); return rc; } +static int fts3SegReaderDoclistCmpRev(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ + int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0); + if( rc==0 ){ + if( pLhs->iDocid==pRhs->iDocid ){ + rc = pRhs->iIdx - pLhs->iIdx; + }else{ + rc = (pLhs->iDocid < pRhs->iDocid) ? 1 : -1; + } + } + assert( pLhs->aNode && pRhs->aNode ); + return rc; +} /* ** Compare the term that the Fts3SegReader object passed as the first argument @@ -1914,21 +2136,30 @@ static int fts3IsEmpty(Fts3Table *p, sqlite3_value *pRowid, int *pisEmpty){ } /* -** Set *pnSegment to the total number of segments in the database. Set -** *pnMax to the largest segment level in the database (segment levels -** are stored in the 'level' column of the %_segdir table). +** Set *pnMax to the largest segment level in the database for the index +** iIndex. +** +** Segment levels are stored in the 'level' column of the %_segdir table. ** ** Return SQLITE_OK if successful, or an SQLite error code if not. */ -static int fts3SegmentCountMax(Fts3Table *p, int *pnSegment, int *pnMax){ +static int fts3SegmentMaxLevel(Fts3Table *p, int iIndex, int *pnMax){ sqlite3_stmt *pStmt; int rc; + assert( iIndex>=0 && iIndexnIndex ); - rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_COUNT_MAX, &pStmt, 0); + /* Set pStmt to the compiled version of: + ** + ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? + ** + ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). + */ + rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL); + sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL - 1); if( SQLITE_ROW==sqlite3_step(pStmt) ){ - *pnSegment = sqlite3_column_int(pStmt, 0); - *pnMax = sqlite3_column_int(pStmt, 1); + *pnMax = sqlite3_column_int(pStmt, 0); } return sqlite3_reset(pStmt); } @@ -1949,6 +2180,7 @@ static int fts3SegmentCountMax(Fts3Table *p, int *pnSegment, int *pnMax){ */ static int fts3DeleteSegdir( Fts3Table *p, /* Virtual table handle */ + int iIndex, /* Index for p->aIndex */ int iLevel, /* Level of %_segdir entries to delete */ Fts3SegReader **apSegment, /* Array of SegReader objects */ int nReader /* Size of array apSegment */ @@ -1971,18 +2203,23 @@ static int fts3DeleteSegdir( return rc; } + assert( iLevel>=0 || iLevel==FTS3_SEGCURSOR_ALL ); if( iLevel==FTS3_SEGCURSOR_ALL ){ - fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0); - }else if( iLevel==FTS3_SEGCURSOR_PENDING ){ - sqlite3Fts3PendingTermsClear(p); - }else{ - assert( iLevel>=0 ); - rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_BY_LEVEL, &pDelete, 0); + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_RANGE, &pDelete, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pDelete, 1, iLevel); - sqlite3_step(pDelete); - rc = sqlite3_reset(pDelete); + sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL); + sqlite3_bind_int(pDelete, 2, (iIndex+1) * FTS3_SEGDIR_MAXLEVEL - 1); } + }else{ + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel); + } + } + + if( rc==SQLITE_OK ){ + sqlite3_step(pDelete); + rc = sqlite3_reset(pDelete); } return rc; @@ -2031,15 +2268,106 @@ static void fts3ColumnFilter( *pnList = nList; } -int sqlite3Fts3SegReaderStart( +/* +** Cache data in the Fts3MultiSegReader.aBuffer[] buffer (overwriting any +** existing data). Grow the buffer if required. +** +** If successful, return SQLITE_OK. Otherwise, if an OOM error is encountered +** trying to resize the buffer, return SQLITE_NOMEM. +*/ +static int fts3MsrBufferData( + Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */ + char *pList, + int nList +){ + if( nList>pMsr->nBuffer ){ + char *pNew; + pMsr->nBuffer = nList*2; + pNew = (char *)sqlite3_realloc(pMsr->aBuffer, pMsr->nBuffer); + if( !pNew ) return SQLITE_NOMEM; + pMsr->aBuffer = pNew; + } + + memcpy(pMsr->aBuffer, pList, nList); + return SQLITE_OK; +} + +int sqlite3Fts3MsrIncrNext( Fts3Table *p, /* Virtual table handle */ - Fts3SegReaderCursor *pCsr, /* Cursor object */ - Fts3SegFilter *pFilter /* Restrictions on range of iteration */ + Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */ + sqlite3_int64 *piDocid, /* OUT: Docid value */ + char **paPoslist, /* OUT: Pointer to position list */ + int *pnPoslist /* OUT: Size of position list in bytes */ +){ + int nMerge = pMsr->nAdvance; + Fts3SegReader **apSegment = pMsr->apSegment; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); + + if( nMerge==0 ){ + *paPoslist = 0; + return SQLITE_OK; + } + + while( 1 ){ + Fts3SegReader *pSeg; + pSeg = pMsr->apSegment[0]; + + if( pSeg->pOffsetList==0 ){ + *paPoslist = 0; + break; + }else{ + int rc; + char *pList; + int nList; + int j; + sqlite3_int64 iDocid = apSegment[0]->iDocid; + + rc = fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList); + j = 1; + while( rc==SQLITE_OK + && jpOffsetList + && apSegment[j]->iDocid==iDocid + ){ + rc = fts3SegReaderNextDocid(p, apSegment[j], 0, 0); + j++; + } + if( rc!=SQLITE_OK ) return rc; + fts3SegReaderSort(pMsr->apSegment, nMerge, j, xCmp); + + if( pMsr->iColFilter>=0 ){ + fts3ColumnFilter(pMsr->iColFilter, &pList, &nList); + } + + if( nList>0 ){ + if( fts3SegReaderIsPending(apSegment[0]) ){ + rc = fts3MsrBufferData(pMsr, pList, nList+1); + if( rc!=SQLITE_OK ) return rc; + *paPoslist = pMsr->aBuffer; + assert( (pMsr->aBuffer[nList] & 0xFE)==0x00 ); + }else{ + *paPoslist = pList; + } + *piDocid = iDocid; + *pnPoslist = nList; + break; + } + } + } + + return SQLITE_OK; +} + +static int fts3SegReaderStart( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr, /* Cursor object */ + const char *zTerm, /* Term searched for (or NULL) */ + int nTerm /* Length of zTerm in bytes */ ){ int i; - - /* Initialize the cursor object */ - pCsr->pFilter = pFilter; + int nSeg = pCsr->nSegment; /* If the Fts3SegFilter defines a specific term (or term prefix) to search ** for, then advance each segment iterator until it points to a term of @@ -2047,24 +2375,105 @@ int sqlite3Fts3SegReaderStart( ** unnecessary merge/sort operations for the case where single segment ** b-tree leaf nodes contain more than one term. */ - for(i=0; inSegment; i++){ - int nTerm = pFilter->nTerm; - const char *zTerm = pFilter->zTerm; + for(i=0; pCsr->bRestart==0 && inSegment; i++){ Fts3SegReader *pSeg = pCsr->apSegment[i]; do { - int rc = fts3SegReaderNext(p, pSeg); + int rc = fts3SegReaderNext(p, pSeg, 0); if( rc!=SQLITE_OK ) return rc; }while( zTerm && fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ); } - fts3SegReaderSort( - pCsr->apSegment, pCsr->nSegment, pCsr->nSegment, fts3SegReaderCmp); + fts3SegReaderSort(pCsr->apSegment, nSeg, nSeg, fts3SegReaderCmp); return SQLITE_OK; } +int sqlite3Fts3SegReaderStart( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr, /* Cursor object */ + Fts3SegFilter *pFilter /* Restrictions on range of iteration */ +){ + pCsr->pFilter = pFilter; + return fts3SegReaderStart(p, pCsr, pFilter->zTerm, pFilter->nTerm); +} + +int sqlite3Fts3MsrIncrStart( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr, /* Cursor object */ + int iCol, /* Column to match on. */ + const char *zTerm, /* Term to iterate through a doclist for */ + int nTerm /* Number of bytes in zTerm */ +){ + int i; + int rc; + int nSegment = pCsr->nSegment; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); + + assert( pCsr->pFilter==0 ); + assert( zTerm && nTerm>0 ); + + /* Advance each segment iterator until it points to the term zTerm/nTerm. */ + rc = fts3SegReaderStart(p, pCsr, zTerm, nTerm); + if( rc!=SQLITE_OK ) return rc; + + /* Determine how many of the segments actually point to zTerm/nTerm. */ + for(i=0; iapSegment[i]; + if( !pSeg->aNode || fts3SegReaderTermCmp(pSeg, zTerm, nTerm) ){ + break; + } + } + pCsr->nAdvance = i; + + /* Advance each of the segments to point to the first docid. */ + for(i=0; inAdvance; i++){ + rc = fts3SegReaderFirstDocid(p, pCsr->apSegment[i]); + if( rc!=SQLITE_OK ) return rc; + } + fts3SegReaderSort(pCsr->apSegment, i, i, xCmp); + + assert( iCol<0 || iColnColumn ); + pCsr->iColFilter = iCol; + + return SQLITE_OK; +} + +/* +** This function is called on a MultiSegReader that has been started using +** sqlite3Fts3MsrIncrStart(). One or more calls to MsrIncrNext() may also +** have been made. Calling this function puts the MultiSegReader in such +** a state that if the next two calls are: +** +** sqlite3Fts3SegReaderStart() +** sqlite3Fts3SegReaderStep() +** +** then the entire doclist for the term is available in +** MultiSegReader.aDoclist/nDoclist. +*/ +int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr){ + int i; /* Used to iterate through segment-readers */ + + assert( pCsr->zTerm==0 ); + assert( pCsr->nTerm==0 ); + assert( pCsr->aDoclist==0 ); + assert( pCsr->nDoclist==0 ); + + pCsr->nAdvance = 0; + pCsr->bRestart = 1; + for(i=0; inSegment; i++){ + pCsr->apSegment[i]->pOffsetList = 0; + pCsr->apSegment[i]->nOffsetList = 0; + pCsr->apSegment[i]->iDocid = 0; + } + + return SQLITE_OK; +} + + int sqlite3Fts3SegReaderStep( Fts3Table *p, /* Virtual table handle */ - Fts3SegReaderCursor *pCsr /* Cursor object */ + Fts3MultiSegReader *pCsr /* Cursor object */ ){ int rc = SQLITE_OK; @@ -2077,6 +2486,9 @@ int sqlite3Fts3SegReaderStep( Fts3SegReader **apSegment = pCsr->apSegment; int nSegment = pCsr->nSegment; Fts3SegFilter *pFilter = pCsr->pFilter; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); if( pCsr->nSegment==0 ) return SQLITE_OK; @@ -2088,7 +2500,7 @@ int sqlite3Fts3SegReaderStep( ** forward. Then sort the list in order of current term again. */ for(i=0; inAdvance; i++){ - rc = fts3SegReaderNext(p, apSegment[i]); + rc = fts3SegReaderNext(p, apSegment[i], 0); if( rc!=SQLITE_OK ) return rc; } fts3SegReaderSort(apSegment, nSegment, pCsr->nAdvance, fts3SegReaderCmp); @@ -2127,10 +2539,18 @@ int sqlite3Fts3SegReaderStep( } assert( isIgnoreEmpty || (isRequirePos && !isColFilter) ); - if( nMerge==1 && !isIgnoreEmpty ){ - pCsr->aDoclist = apSegment[0]->aDoclist; + if( nMerge==1 + && !isIgnoreEmpty + && (p->bDescIdx==0 || fts3SegReaderIsPending(apSegment[0])==0) + ){ pCsr->nDoclist = apSegment[0]->nDoclist; - rc = SQLITE_ROW; + if( fts3SegReaderIsPending(apSegment[0]) ){ + rc = fts3MsrBufferData(pCsr, apSegment[0]->aDoclist, pCsr->nDoclist); + pCsr->aDoclist = pCsr->aBuffer; + }else{ + pCsr->aDoclist = apSegment[0]->aDoclist; + } + if( rc==SQLITE_OK ) rc = SQLITE_ROW; }else{ int nDoclist = 0; /* Size of doclist */ sqlite3_int64 iPrev = 0; /* Previous docid stored in doclist */ @@ -2140,22 +2560,22 @@ int sqlite3Fts3SegReaderStep( ** and a single term returned with the merged doclist. */ for(i=0; ipOffsetList ){ int j; /* Number of segments that share a docid */ char *pList; int nList; int nByte; sqlite3_int64 iDocid = apSegment[0]->iDocid; - fts3SegReaderNextDocid(apSegment[0], &pList, &nList); + fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList); j = 1; while( jpOffsetList && apSegment[j]->iDocid==iDocid ){ - fts3SegReaderNextDocid(apSegment[j], 0, 0); + fts3SegReaderNextDocid(p, apSegment[j], 0, 0); j++; } @@ -2164,7 +2584,19 @@ int sqlite3Fts3SegReaderStep( } if( !isIgnoreEmpty || nList>0 ){ - nByte = sqlite3Fts3VarintLen(iDocid-iPrev) + (isRequirePos?nList+1:0); + + /* Calculate the 'docid' delta value to write into the merged + ** doclist. */ + sqlite3_int64 iDelta; + if( p->bDescIdx && nDoclist>0 ){ + iDelta = iPrev - iDocid; + }else{ + iDelta = iDocid - iPrev; + } + assert( iDelta>0 || (nDoclist==0 && iDelta==iDocid) ); + assert( nDoclist>0 || iDelta==iDocid ); + + nByte = sqlite3Fts3VarintLen(iDelta) + (isRequirePos?nList+1:0); if( nDoclist+nByte>pCsr->nBuffer ){ char *aNew; pCsr->nBuffer = (nDoclist+nByte)*2; @@ -2174,9 +2606,7 @@ int sqlite3Fts3SegReaderStep( } pCsr->aBuffer = aNew; } - nDoclist += sqlite3Fts3PutVarint( - &pCsr->aBuffer[nDoclist], iDocid-iPrev - ); + nDoclist += sqlite3Fts3PutVarint(&pCsr->aBuffer[nDoclist], iDelta); iPrev = iDocid; if( isRequirePos ){ memcpy(&pCsr->aBuffer[nDoclist], pList, nList); @@ -2185,7 +2615,7 @@ int sqlite3Fts3SegReaderStep( } } - fts3SegReaderSort(apSegment, nMerge, j, fts3SegReaderDoclistCmp); + fts3SegReaderSort(apSegment, nMerge, j, xCmp); } if( nDoclist>0 ){ pCsr->aDoclist = pCsr->aBuffer; @@ -2199,8 +2629,9 @@ int sqlite3Fts3SegReaderStep( return rc; } + void sqlite3Fts3SegReaderFinish( - Fts3SegReaderCursor *pCsr /* Cursor object */ + Fts3MultiSegReader *pCsr /* Cursor object */ ){ if( pCsr ){ int i; @@ -2227,43 +2658,56 @@ void sqlite3Fts3SegReaderFinish( ** Otherwise, if successful, SQLITE_OK is returned. If an error occurs, ** an SQLite error code is returned. */ -static int fts3SegmentMerge(Fts3Table *p, int iLevel){ +static int fts3SegmentMerge(Fts3Table *p, int iIndex, int iLevel){ int rc; /* Return code */ int iIdx = 0; /* Index of new segment */ - int iNewLevel = 0; /* Level to create new segment at */ + int iNewLevel = 0; /* Level/index to create new segment at */ SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */ Fts3SegFilter filter; /* Segment term filter condition */ - Fts3SegReaderCursor csr; /* Cursor to iterate through level(s) */ + Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */ + int bIgnoreEmpty = 0; /* True to ignore empty segments */ - rc = sqlite3Fts3SegReaderCursor(p, iLevel, 0, 0, 1, 0, &csr); + assert( iLevel==FTS3_SEGCURSOR_ALL + || iLevel==FTS3_SEGCURSOR_PENDING + || iLevel>=0 + ); + assert( iLevel=0 && iIndexnIndex ); + + rc = sqlite3Fts3SegReaderCursor(p, iIndex, iLevel, 0, 0, 1, 0, &csr); if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished; if( iLevel==FTS3_SEGCURSOR_ALL ){ /* This call is to merge all segments in the database to a single ** segment. The level of the new segment is equal to the the numerically - ** greatest segment level currently present in the database. The index - ** of the new segment is always 0. */ - int nDummy; /* TODO: Remove this */ + ** greatest segment level currently present in the database for this + ** index. The idx of the new segment is always 0. */ if( csr.nSegment==1 ){ rc = SQLITE_DONE; goto finished; } - rc = fts3SegmentCountMax(p, &nDummy, &iNewLevel); + rc = fts3SegmentMaxLevel(p, iIndex, &iNewLevel); + bIgnoreEmpty = 1; + + }else if( iLevel==FTS3_SEGCURSOR_PENDING ){ + iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL; + rc = fts3AllocateSegdirIdx(p, iIndex, 0, &iIdx); }else{ - /* This call is to merge all segments at level iLevel. Find the next + /* This call is to merge all segments at level iLevel. find the next ** available segment index at level iLevel+1. The call to ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to ** a single iLevel+2 segment if necessary. */ - iNewLevel = iLevel+1; - rc = fts3AllocateSegdirIdx(p, iNewLevel, &iIdx); + rc = fts3AllocateSegdirIdx(p, iIndex, iLevel+1, &iIdx); + iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL + iLevel+1; } if( rc!=SQLITE_OK ) goto finished; assert( csr.nSegment>0 ); - assert( iNewLevel>=0 ); + assert( iNewLevel>=(iIndex*FTS3_SEGDIR_MAXLEVEL) ); + assert( iNewLevel<((iIndex+1)*FTS3_SEGDIR_MAXLEVEL) ); memset(&filter, 0, sizeof(Fts3SegFilter)); filter.flags = FTS3_SEGMENT_REQUIRE_POS; - filter.flags |= (iLevel==FTS3_SEGCURSOR_ALL ? FTS3_SEGMENT_IGNORE_EMPTY : 0); + filter.flags |= (bIgnoreEmpty ? FTS3_SEGMENT_IGNORE_EMPTY : 0); rc = sqlite3Fts3SegReaderStart(p, &csr, &filter); while( SQLITE_OK==rc ){ @@ -2275,8 +2719,10 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){ if( rc!=SQLITE_OK ) goto finished; assert( pWriter ); - rc = fts3DeleteSegdir(p, iLevel, csr.apSegment, csr.nSegment); - if( rc!=SQLITE_OK ) goto finished; + if( iLevel!=FTS3_SEGCURSOR_PENDING ){ + rc = fts3DeleteSegdir(p, iIndex, iLevel, csr.apSegment, csr.nSegment); + if( rc!=SQLITE_OK ) goto finished; + } rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx); finished: @@ -2287,10 +2733,17 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){ /* -** Flush the contents of pendingTerms to a level 0 segment. +** Flush the contents of pendingTerms to level 0 segments. */ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ - return fts3SegmentMerge(p, FTS3_SEGCURSOR_PENDING); + int rc = SQLITE_OK; + int i; + for(i=0; rc==SQLITE_OK && inIndex; i++){ + rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_PENDING); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + sqlite3Fts3PendingTermsClear(p); + return rc; } /* @@ -2441,6 +2894,23 @@ static void fts3UpdateDocTotals( sqlite3_free(a); } +static int fts3DoOptimize(Fts3Table *p, int bReturnDone){ + int i; + int bSeenDone = 0; + int rc = SQLITE_OK; + for(i=0; rc==SQLITE_OK && inIndex; i++){ + rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_ALL); + if( rc==SQLITE_DONE ){ + bSeenDone = 1; + rc = SQLITE_OK; + } + } + sqlite3Fts3SegmentsClose(p); + sqlite3Fts3PendingTermsClear(p); + + return (rc==SQLITE_OK && bReturnDone && bSeenDone) ? SQLITE_DONE : rc; +} + /* ** Handle a 'special' INSERT of the form: ** @@ -2457,12 +2927,7 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ if( !zVal ){ return SQLITE_NOMEM; }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){ - rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL); - if( rc==SQLITE_DONE ){ - rc = SQLITE_OK; - }else{ - sqlite3Fts3PendingTermsClear(p); - } + rc = fts3DoOptimize(p, 0); #ifdef SQLITE_TEST }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ p->nNodeSize = atoi(&zVal[9]); @@ -2475,44 +2940,9 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ rc = SQLITE_ERROR; } - sqlite3Fts3SegmentsClose(p); return rc; } -/* -** Return the deferred doclist associated with deferred token pDeferred. -** This function assumes that sqlite3Fts3CacheDeferredDoclists() has already -** been called to allocate and populate the doclist. -*/ -char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *pDeferred, int *pnByte){ - if( pDeferred->pList ){ - *pnByte = pDeferred->pList->nData; - return pDeferred->pList->aData; - } - *pnByte = 0; - return 0; -} - -/* -** Helper fucntion for FreeDeferredDoclists(). This function removes all -** references to deferred doclists from within the tree of Fts3Expr -** structures headed by -*/ -static void fts3DeferredDoclistClear(Fts3Expr *pExpr){ - if( pExpr ){ - fts3DeferredDoclistClear(pExpr->pLeft); - fts3DeferredDoclistClear(pExpr->pRight); - if( pExpr->isLoaded ){ - sqlite3_free(pExpr->aDoclist); - pExpr->isLoaded = 0; - pExpr->aDoclist = 0; - pExpr->nDoclist = 0; - pExpr->pCurrent = 0; - pExpr->iCurrent = 0; - } - } -} - /* ** Delete all cached deferred doclists. Deferred doclists are cached ** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function. @@ -2520,12 +2950,9 @@ static void fts3DeferredDoclistClear(Fts3Expr *pExpr){ void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *pCsr){ Fts3DeferredToken *pDef; for(pDef=pCsr->pDeferred; pDef; pDef=pDef->pNext){ - sqlite3_free(pDef->pList); + fts3PendingListDelete(pDef->pList); pDef->pList = 0; } - if( pCsr->pDeferred ){ - fts3DeferredDoclistClear(pCsr->pExpr); - } } /* @@ -2537,7 +2964,7 @@ void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *pCsr){ Fts3DeferredToken *pNext; for(pDef=pCsr->pDeferred; pDef; pDef=pNext){ pNext = pDef->pNext; - sqlite3_free(pDef->pList); + fts3PendingListDelete(pDef->pList); sqlite3_free(pDef); } pCsr->pDeferred = 0; @@ -2602,6 +3029,33 @@ int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){ return rc; } +int sqlite3Fts3DeferredTokenList( + Fts3DeferredToken *p, + char **ppData, + int *pnData +){ + char *pRet; + int nSkip; + sqlite3_int64 dummy; + + *ppData = 0; + *pnData = 0; + + if( p->pList==0 ){ + return SQLITE_OK; + } + + pRet = (char *)sqlite3_malloc(p->pList->nData); + if( !pRet ) return SQLITE_NOMEM; + + nSkip = sqlite3Fts3GetVarint(p->pList->aData, &dummy); + *pnData = p->pList->nData - nSkip; + *ppData = pRet; + + memcpy(pRet, &p->pList->aData[nSkip], *pnData); + return SQLITE_OK; +} + /* ** Add an entry for token pToken to the pCsr->pDeferred list. */ @@ -2676,7 +3130,7 @@ int sqlite3Fts3UpdateMethod( int rc = SQLITE_OK; /* Return Code */ int isRemove = 0; /* True for an UPDATE or DELETE */ sqlite3_int64 iRemove = 0; /* Rowid removed by UPDATE or DELETE */ - u32 *aSzIns; /* Sizes of inserted documents */ + u32 *aSzIns = 0; /* Sizes of inserted documents */ u32 *aSzDel; /* Sizes of deleted documents */ int nChng = 0; /* Net change in number of documents */ int bInsertDone = 0; @@ -2691,12 +3145,16 @@ int sqlite3Fts3UpdateMethod( && sqlite3_value_type(apVal[0])==SQLITE_NULL && sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL ){ - return fts3SpecialInsert(p, apVal[p->nColumn+2]); + rc = fts3SpecialInsert(p, apVal[p->nColumn+2]); + goto update_out; } /* Allocate space to hold the change in document sizes */ aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*(p->nColumn+1)*2 ); - if( aSzIns==0 ) return SQLITE_NOMEM; + if( aSzIns==0 ){ + rc = SQLITE_NOMEM; + goto update_out; + } aSzDel = &aSzIns[p->nColumn+1]; memset(aSzIns, 0, sizeof(aSzIns[0])*(p->nColumn+1)*2); @@ -2746,8 +3204,7 @@ int sqlite3Fts3UpdateMethod( } } if( rc!=SQLITE_OK ){ - sqlite3_free(aSzIns); - return rc; + goto update_out; } /* If this is a DELETE or UPDATE operation, remove the old record. */ @@ -2780,6 +3237,7 @@ int sqlite3Fts3UpdateMethod( fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng); } + update_out: sqlite3_free(aSzIns); sqlite3Fts3SegmentsClose(p); return rc; @@ -2794,12 +3252,10 @@ int sqlite3Fts3Optimize(Fts3Table *p){ int rc; rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0); if( rc==SQLITE_OK ){ - rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL); - if( rc==SQLITE_OK ){ - rc = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); - if( rc==SQLITE_OK ){ - sqlite3Fts3PendingTermsClear(p); - } + rc = fts3DoOptimize(p, 1); + if( rc==SQLITE_OK || rc==SQLITE_DONE ){ + int rc2 = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); + if( rc2!=SQLITE_OK ) rc = rc2; }else{ sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0); sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); diff --git a/main.mk b/main.mk index 840a3d01aa..d1229405ab 100644 --- a/main.mk +++ b/main.mk @@ -221,6 +221,7 @@ SRC += \ # Source code to the test files. # TESTSRC = \ + $(TOP)/ext/fts3/fts3_test.c \ $(TOP)/src/test1.c \ $(TOP)/src/test2.c \ $(TOP)/src/test3.c \ diff --git a/manifest b/manifest index ac09e78587..ffa65232b3 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,9 @@ C Merge\sthe\slatest\strunk\schanges\sinto\sthe\ssessions\sbranch. -D 2011-05-30T13:39:20.980 +D 2011-06-20T10:44:10.301 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d +F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 +F Makefile.msc ce73810c83d4bd202deb59d547a5df4a5cfef7c9 F Makefile.vxworks c85ec1d8597fe2f7bc225af12ac1666e21379151 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 F VERSION 3fcdd7fbe3eb282df3978fe77288544543767961 @@ -61,21 +62,22 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c b3a10a1a320aaeb56a1dd6710bf09eb5c2370839 +F ext/fts3/fts3.c f919a7966426e539b3f39f696bc94269e3726033 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h c8c0011c5e5b3a7703376ea6cd7deb91cfb96a06 -F ext/fts3/fts3_aux.c 97c960b1b0d371c08eae6b8565dfac619eb9d979 -F ext/fts3/fts3_expr.c 5f49e0deaf723724b08100bb3ff40aab02ad0c93 -F ext/fts3/fts3_hash.c 3c8f6387a4a7f5305588b203fa7c887d753e1f1c +F ext/fts3/fts3Int.h 8ece4390eb44e7179bb05c59d40f447663f5c077 +F ext/fts3/fts3_aux.c 0ebfa7b86cf8ff6a0861605fcc63b83ec1b70691 +F ext/fts3/fts3_expr.c 23791de01b3a5d313d76e02befd2601d4096bc2b +F ext/fts3/fts3_hash.c aad95afa01cf2a5ffaa448e4b0ab043880cd1efb F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec -F ext/fts3/fts3_icu.c ac494aed69835008185299315403044664bda295 -F ext/fts3/fts3_porter.c d61cfd81fb0fd8fbcb25adcaee0ba671aefaa5c2 -F ext/fts3/fts3_snippet.c 92b40397b28422c35c4127492d7ac6da34d1966a -F ext/fts3/fts3_term.c f115f5a5f4298303d3b22fc6c524b8d565c7b950 -F ext/fts3/fts3_tokenizer.c 055f3dc7369585350b28db1ee0f3b214dca6724d +F ext/fts3/fts3_icu.c 6c8f395cdf9e1e3afa7fadb7e523dbbf381c6dfa +F ext/fts3/fts3_porter.c 8d946908f4812c005d3d33fcbe78418b1f4eb70c +F ext/fts3/fts3_snippet.c a44b38a07d39701ab6d20d7d89fcafe193bf3680 +F ext/fts3/fts3_term.c 51e384269edcc015e8b555fdad2338f053388975 +F ext/fts3/fts3_test.c 4e833729c13cea9a6bb98d3b353f6e3b8f756004 +F ext/fts3/fts3_tokenizer.c 90ba6cdd8bb1b3686ab7a3d72333131e13c8fdb2 F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 -F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d -F ext/fts3/fts3_write.c b50181e5ecf484c2f56e98d651424e4b69f96c89 +F ext/fts3/fts3_tokenizer1.c 0dde8f307b8045565cf63797ba9acfaff1c50c68 +F ext/fts3/fts3_write.c 5774a7ee9632355ebf1ec4b7a5071fc9ab9eb956 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -112,7 +114,7 @@ F ext/session/sqlite3session.h 665f5591562e3c71eb3d0da26f1a1efae26f7bcf F ext/session/test_session.c 311e5b9228374d0b5780448f289847ff1cf7d388 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F main.mk bc31e3b2cfa42337a34fc4509a1b550c0cd5b202 +F main.mk c864cbc95010ff6bdb6ec6d213c17be0bd2a86a2 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac @@ -125,26 +127,26 @@ F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad -F src/alter.c 280f5c04b11b492703a342222b3de0a999445280 +F src/alter.c ac80a0f31189f8b4a524ebf661e47e84536ee7f5 F src/analyze.c a425d62e8fa9ebcb4359ab84ff0c62c6563d2e2a F src/attach.c 12c6957996908edc31c96d7c68d4942c2474405f F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 986c15232757f2873dff35ee3b35cbf935fc573c F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 975ad691a57eb1fb60f1ec76ad0b6571eace62f9 +F src/btree.c 8c46f0ab69ad9549c75a3a91fed87abdaa743e2f F src/btree.h f5d775cd6cfc7ac32a2535b70e8d2af48ef5f2ce F src/btreeInt.h 67978c014fa4f7cc874032dd3aacadd8db656bc3 -F src/build.c 0132bc6631fa617a1d28ef805921f6dbac18a514 +F src/build.c 5a428625d21ad409514afb40ad083bee25dd957a F src/callback.c 0425c6320730e6d3981acfb9202c1bed9016ad1a F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 7deec4534f3b5a0c3b4a4cbadf809d321f64f9c4 F src/date.c 1548fdac51377e4e7833251de878b4058c148e1b -F src/delete.c ad9fa1cbf91a83ec6990d0aecb7e21cd5ff07e71 -F src/expr.c e3cf0957c6b8faaaf7386a3bc69e53c0dc9705be +F src/delete.c c84066147544e8fddb7ca60ac7fd36ed2df8487f +F src/expr.c ab46ab0f0c44979a8164ca31728d7d10ae5e8106 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb -F src/fkey.c a43ba8a005fb5efd1deeee06853e3a6120d46a91 -F src/func.c b9117e40975245b8504cf3625d7e321d8d4b63dc +F src/fkey.c 9fabba17a4d4778dc660f0cb9d781fc86d7b9d41 +F src/func.c 59bb046d7e3df1ab512ac339ccb0a6f996a17cb7 F src/global.c 29bfb85611dd816b04f10fba0ca910366e128d38 F src/hash.c 458488dcc159c301b8e7686280ab209f1fb915af F src/hash.h 2894c932d84d9f892d4b4023a75e501f83050970 @@ -154,7 +156,7 @@ F src/journal.c 552839e54d1bf76fb8f7abe51868b66acacf6a0e F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 3ae0d52da013a6326310655be6473fd472347b85 -F src/main.c 8206d7970cb858979ec84eea9a5eff2b575849a6 +F src/main.c f7e8176ec1a9cad97470801660629179570d6b1d F src/malloc.c 591aedb20ae40813f1045f2ef253438a334775d9 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 00bd8265c81abb665c48fea1e0c234eb3b922206 @@ -173,30 +175,30 @@ F src/os.c 22ac61d06e72a0dac900400147333b07b13d8e1d F src/os.h 9dbed8c2b9c1f2f2ebabc09e49829d4777c26bf9 F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 -F src/os_unix.c 6d4a58d81ad4b782406519f3790202f330e89bb7 +F src/os_unix.c fd4e9588ff0ce09720721ce739ab2682202875ae F src/os_win.c 218b899469e570d46eb8147c2383075f7c026230 F src/pager.c 120550e7ef01dafaa2cbb4a0528c0d87c8f12b41 F src/pager.h 3f8c783de1d4706b40b1ac15b64f5f896bcc78d1 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 F src/pcache.c 49e718c095810c6b3334e3a6d89970aceaddefce F src/pcache.h c683390d50f856d4cd8e24342ae62027d1bb6050 -F src/pcache1.c d548e31beafa792d1994b663a29a5303569efc4e -F src/pragma.c 9e778decc3ee9bcaf88904b4a3b0a4360aaf0eab +F src/pcache1.c 912bd5687d6df344698d8e69560f347b6e21c18a +F src/pragma.c ebcd20f1e654f5cb3aeef864ed69c4697719fbaa F src/prepare.c e64261559a3187698a3e7e6c8b001a4f4f98dab4 F src/printf.c 585a36b6a963df832cfb69505afa3a34ed5ef8a1 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 1c0f32b64f8e3f555fe1f732f9d6f501a7f05706 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c d9d440809025a58547e39f4f268c2a296bfb56ff -F src/shell.c decd04236a7ef26be5ef46d4ea963044bfad9a48 -F src/sqlite.h.in 6e087deaec87d358aa4027545ef7454e427b5619 +F src/shell.c 0e0173b3e79d956368013e759f084caa7995ecb1 +F src/sqlite.h.in 1daf26cc593fa78dd041af564c708869b1f12df3 F src/sqlite3ext.h c90bd5507099f62043832d73f6425d8d5c5da754 -F src/sqliteInt.h 38481431b379b468f7bd282d2e318aed4b56d860 +F src/sqliteInt.h 928caa40080d47e8f92e1edee83f6249d03c5862 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac64842c86cec2fc1a1d0e5c16d3beb8ad332bf F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e -F src/tclsqlite.c fe0da0eb0ebd8d21eec90683b779456e64351de6 -F src/test1.c 4a1171af201be90c21d64a872e686b1333d9a2cf +F src/tclsqlite.c 761d9d7a7b79a41d5a237e6c5c350a1c719ec983 +F src/test1.c efca486a25fb894988e7a82e84579a4e57388a02 F src/test2.c 80d323d11e909cf0eb1b6fbb4ac22276483bcf31 F src/test3.c 124ff9735fb6bb7d41de180d6bac90e7b1509432 F src/test4.c d1e5a5e904d4b444cf572391fdcb017638e36ff7 @@ -227,7 +229,7 @@ F src/test_mutex.c a6bd7b9cf6e19d989e31392b06ac8d189f0d573e F src/test_onefile.c 40cf9e212a377a6511469384a64b01e6e34b2eec F src/test_osinst.c 62b0b8ef21ce754cc94e17bb42377ed8795dba32 F src/test_pcache.c 7bf828972ac0d2403f5cfa4cd14da41f8ebe73d8 -F src/test_quota.c b5576f17d701af461effd7ca1e71f0d100071192 +F src/test_quota.c cc4f67e12558a252ea4a11720be268348f4b1595 F src/test_rtree.c 30c981837445a4e187ee850a49c4760d9642f7c3 F src/test_schema.c 8c06ef9ddb240c7a0fcd31bc221a6a2aade58bf0 F src/test_server.c 2f99eb2837dfa06a4aacf24af24c6affdf66a84f @@ -240,22 +242,22 @@ F src/test_vfs.c e7855568dfa1e0ba73668d273b65605d9f8b77e8 F src/test_vfstrace.c 0b884e06094a746da729119a2cabdc7aa790063d F src/test_wholenumber.c 6129adfbe7c7444f2e60cc785927f3aa74e12290 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/tokenize.c 604607d6813e9551cf5189d899e0a25c12681080 -F src/trigger.c 144cc18bb701f3286484aae4292a9531f09278c8 -F src/update.c f66b651c15e42875f36501ec39a968e836ee5586 -F src/utf.c d83650c3ea08f7407bd9d0839d9885241c209c60 +F src/tokenize.c c819d9f72168a035d545a5bdafe9b085b20df705 +F src/trigger.c c836a6caac16ba96611558922106858f6ca3d6bf +F src/update.c a81bda229f8c3b698f8dcf8e69485c97e1347102 +F src/utf.c c53eb7404b3eb5c1cbb5655c6a7a0e0ce6bd50f0 F src/util.c 0f33bbbdfcc4a2d8cf20c3b2a16ffc3b57c58a70 F src/vacuum.c 05513dca036a1e7848fe18d5ed1265ac0b32365e -F src/vdbe.c 619fb3fc054f7f9bf41c0e354702dfc35eebb17f -F src/vdbe.h 44fd57aeed86da0cd31206626c13cdde0e72cc0e -F src/vdbeInt.h b95de01246c15499c700ae00cfda0de25c01358a -F src/vdbeapi.c 8051038f7674c708f4515ab189fc3ea929e09a4c -F src/vdbeaux.c 8c5a643ba7d0d9882902ef2b4ab3dc2084d838ae -F src/vdbeblob.c c3ccb7c8732858c680f442932e66ad06bb036562 +F src/vdbe.c af4d8ba06efb768f405d189d4b992c81ae14d711 +F src/vdbe.h 322af148cceef120bb1ec9cff7f122e76abf94da +F src/vdbeInt.h 3de6588b36c833969aebab202e1766d586c37ec2 +F src/vdbeapi.c 3f6e988bd19391be1aa49ffd1f259654dcc8d975 +F src/vdbeaux.c db3d4eedccea5add714dfb8b10f70d0f8d692db5 +F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 F src/vdbemem.c 0498796b6ffbe45e32960d6a1f5adfb6e419883b F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114 -F src/vtab.c 9ba8c7fdb7d39260c033a402f6032d3e7bc5d336 -F src/wal.c ab1d8c5abf904fd6396f90499cbd8c54b8d6961b +F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 +F src/wal.c 0c70ad7b1cac6005fa5e2cbefd23ee05e391c290 F src/wal.h 66b40bd91bc29a5be1c88ddd1f5ade8f3f48728a F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c 55403ce19c506be6a321c7f129aff693d6103db5 @@ -390,7 +392,7 @@ F test/e_resolve.test dcce9308fb13b934ce29591105d031d3e14fbba6 F test/e_select.test bf385ae3aa0f014c4933ae66fd3e1302138493eb F test/e_select2.test 5c3d3da19c7b3e90ae444579db2b70098599ab92 F test/e_update.test 963d6876064e65f318d1c93aaed36a02b9b389bf -F test/e_uri.test 9ce11319fb9b271bf7392027f913f7830e93e7a7 +F test/e_uri.test b6da43a10f44d9aa0aff5ffa3c2f3de668361255 F test/e_vacuum.test 6c09c2af7f2f140518f371c5342100118f779dcf F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea F test/enc2.test 6d91a5286f59add0cfcbb2d0da913b76f2242398 @@ -408,7 +410,7 @@ F test/filectrl.test 97003734290887566e01dded09dc9e99cb937e9e F test/filefmt.test f178cfc29501a14565954c961b226e61877dd32c F test/fkey1.test 01c7de578e11747e720c2d9aeef27f239853c4da F test/fkey2.test 080969fe219b3b082b0e097ac18c6af2e5b0631f -F test/fkey3.test 42f88d6048d8dc079e2a8cf7baad1cc1483a7620 +F test/fkey3.test 5ec899d12b13bcf1e9ef40eff7fb692fdb91392e F test/fkey4.test c6c8f9f9be885f95c85c7bceb26f243ad906fd49 F test/fkey_malloc.test a5ede29bd2f6e56dea78c3d43fb86dd696c068c8 F test/format4.test 1f0cac8ff3895e9359ed87e41aaabee982a812eb @@ -464,6 +466,7 @@ F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8 F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18 F test/fts3ao.test b83f99f70e9eec85f27d75801a974b3f820e01f9 F test/fts3atoken.test 402ef2f7c2fb4b3d4fa0587df6441c1447e799b3 +F test/fts3auto.test b0d360b331ff68bd9fb497a6192d23dc0783637c F test/fts3aux1.test 0b02743955d56fc0d4d66236a26177bd1b726de0 F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958 @@ -473,21 +476,22 @@ F test/fts3corrupt.test 7b0f91780ca36118d73324ec803187208ad33b32 F test/fts3corrupt2.test 6d96efae2f8a6af3eeaf283aba437e6d0e5447ba F test/fts3cov.test e0fb00d8b715ddae4a94c305992dfc3ef70353d7 F test/fts3d.test 95fb3c862cbc4297c93fceb9a635543744e9ef52 -F test/fts3defer.test d6cb0db9b5997ecf863d96ff419f83f8f2c87f4f -F test/fts3defer2.test 288bef6de15557319b8c12d476ebdc83688ef96c +F test/fts3defer.test 7c8a38d5f617d7b52ae1c43ed73c536e7e895a35 +F test/fts3defer2.test 35867d33ba6db03f6c73bd6f5fc333ae14f68c81 F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851 F test/fts3expr.test 5e745b2b6348499d9ef8d59015de3182072c564c F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a F test/fts3fault.test f83e556465bb69dc8bc676339eca408dce4ca246 F test/fts3fault2.test dc96203af6ba31ce20163fc35460e1556e8edf4d F test/fts3malloc.test 9c8cc3f885bb4dfc66d0460c52f68f45e4710d1b -F test/fts3matchinfo.test cc0b009edbbf575283d5fdb53271179e0d8019ba +F test/fts3matchinfo.test 08a82d18cc08abb28aec41d412b4c2ef25ba6a5f F test/fts3near.test 2e318ee434d32babd27c167142e2b94ddbab4844 +F test/fts3prefix.test 36246609111ec1683f7ea5ed27666ce2cefb5676 F test/fts3query.test ef79d31fdb355d094baec1c1b24b60439a1fb8a2 -F test/fts3rnd.test 2b1a579be557ab8ac54a51b39caa4aa8043cc4ad +F test/fts3rnd.test 1320d8826a845e38a96e769562bf83d7a92a15d0 F test/fts3shared.test 8bb266521d7c5495c0ae522bb4d376ad5387d4a2 F test/fts3snippet.test a12f22a3ba4dd59751a57c79b031d07ab5f51ddd -F test/fts3sort.test e6f24e9cffc46484bcc9fe63d3c2ce41afcaa6c9 +F test/fts3sort.test 63d52c1812904b751f9e1ff487472e44833f5402 F test/fts4aa.test eadf85621c0a113d4c7ad3ccbf8441130e007b8f F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f @@ -498,7 +502,7 @@ F test/fuzz3.test aec64345184d1662bd30e6a17851ff659d596dc5 F test/fuzz_common.tcl a87dfbb88c2a6b08a38e9a070dabd129e617b45b F test/fuzz_malloc.test dd7001ac86d09c154a7dff064f4739c60e2b312c F test/fuzzer1.test 3105b5a89a6cb0d475f0877debec942fe4143462 -F test/hook.test 040cf2ca263f192c66b358e095138dad0a9d75bb +F test/hook.test 4fd80e9c3ffb49de0379e2a026ef2fe7b9f3e534 F test/icu.test 70df4faca133254c042d02ae342c0a141f2663f4 F test/in.test 19b642bb134308980a92249750ea4ce3f6c75c2d F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 @@ -558,11 +562,11 @@ F test/lock4.test c82268c031d39345d05efa672f80b025481b3ae5 F test/lock5.test b2abb5e711bc59b0eae00f6c97a36ec9f458fada F test/lock6.test ad5b387a3a8096afd3c68a55b9535056431b0cf5 F test/lock7.test 64006c84c1c616657e237c7ad6532b765611cf64 -F test/lock_common.tcl d279887a0ab16cdb6d935c1203e64113c5a000e9 +F test/lock_common.tcl 0c270b121d40959fa2f3add382200c27045b3d95 F test/lookaside.test 93f07bac140c5bb1d49f3892d2684decafdc7af2 F test/main.test 9d7bbfcc1b52c88ba7b2ba6554068ecf9939f252 F test/make-where7.tcl 05c16b5d4f5d6512881dfec560cb793915932ef9 -F test/malloc.test e56c9c3358da2c18385aea15a42dc970913986c2 +F test/malloc.test 76017be66cec4375a4b4ea5c71245e27a9fe2d0b F test/malloc3.test 4128b1e6ffa506103b278ad97af89174f310c7ca F test/malloc4.test 957337613002b7058a85116493a262f679f3a261 F test/malloc5.test 4d16d1bb26d2deddd7c4f480deec341f9b2d0e22 @@ -600,7 +604,7 @@ F test/misc5.test 45b2e3ed5f79af2b4f38ae362eaf4c49674575bd F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test 29032efcd3d826fbd409e2a7af873e7939f4a4e3 F test/misuse.test 30b3a458e5a70c31e74c291937b6c82204c59f33 -F test/multiplex.test 7a8a50c8ed72dfcf4db9ebae977f7a63184639d8 +F test/multiplex.test 555080c87abfc72ba68e2f3df01d4a9a7a4fdf58 F test/mutex1.test 78b2b9bb320e51d156c4efdb71b99b051e7a4b41 F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660 F test/nan.test dc212a22b36109fd1ae37154292444ef249c5ec2 @@ -611,17 +615,17 @@ F test/notnull.test cc7c78340328e6112a13c3e311a9ab3127114347 F test/null.test a8b09b8ed87852742343b33441a9240022108993 F test/openv2.test af02ed0a9cbc0d2a61b8f35171d4d117e588e4ec F test/oserror.test 498d8337e9d15543eb7b004fef8594bf204ff43c -F test/pager1.test 8baf4470b29511503abcaf1f17d16b16462e4d54 +F test/pager1.test 228a831060dab96bc91b03ba2a85cedefd1ab38a F test/pager2.test 745b911dde3d1f24ae0870bd433dfa83d7c658c1 F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f -F test/pagerfault.test 9de4d3e0c59970b4c6cb8dac511fa242f335d8a7 +F test/pagerfault.test 4194b8ea2a5da7958cd155556605ff554e1b065a F test/pagerfault2.test 1f79ea40d1133b2683a2f811b00f2399f7ec2401 F test/pagerfault3.test f16e2efcb5fc9996d1356f7cbc44c998318ae1d7 F test/pageropt.test 8146bf448cf09e87bb1867c2217b921fb5857806 F test/pagesize.test 76aa9f23ecb0741a4ed9d2e16c5fa82671f28efb F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test 0d85f2ab6963aee28c671d4c71bec038c00a1d16 -F test/permutations.test 1e35edce72e6d9e2e392420caed18652a97b1a95 +F test/permutations.test 8331b2897502df0671dd1390a3c87db44fb8757c F test/pragma.test fdfc09067ea104a0c247a1a79d8093b56656f850 F test/pragma2.test 5364893491b9231dd170e3459bfc2e2342658b47 F test/printf.test 05970cde31b1a9f54bd75af60597be75a5c54fea @@ -702,7 +706,7 @@ F test/tclsqlite.test 1ce9b6340d6d412420634e129a2e3722c651056a F test/tempdb.test 19d0f66e2e3eeffd68661a11c83ba5e6ace9128c F test/temptable.test f42121a0d29a62f00f93274464164177ab1cc24a F test/temptrigger.test b0273db072ce5f37cf19140ceb1f0d524bbe9f05 -F test/tester.tcl 1949b4af9701daaca189fd5d53a6e48173c162af +F test/tester.tcl 174f2bc00ddacc6c7666a15d3dbac9669dfc2373 F test/thread001.test a3e6a7254d1cb057836cb3145b60c10bf5b7e60f F test/thread002.test afd20095e6e845b405df4f2c920cb93301ca69db F test/thread003.test b824d4f52b870ae39fc5bae4d8070eca73085dca @@ -829,7 +833,7 @@ F test/tkt3997.test a335fa41ca3985660a139df7b734a26ef53284bd F test/tkt4018.test 7c2c9ba4df489c676a0a7a0e809a1fb9b2185bd1 F test/tokenize.test ce430a7aed48fc98301611429595883fdfcab5d7 F test/trace.test 4b36a41a3e9c7842151af6da5998f5080cdad9e5 -F test/trace2.test 0ce11265c83333d8f5beeca19e71ed93a88d386c +F test/trace2.test 962175290996d5f06dc4402ca218bbfc7df4cb20 F test/trans.test 6e1b4c6a42dba31bd65f8fa5e61a2708e08ddde6 F test/trans2.test d5337e61de45e66b1fcbf9db833fa8c82e624b22 F test/trans3.test d728abaa318ca364dc370e06576aa7e5fbed7e97 @@ -886,7 +890,7 @@ F test/wal3.test 5c396cc22497244d627306f4c1d360167353f8dd F test/wal4.test 3404b048fa5e10605facaf70384e6d2943412e30 F test/wal5.test 1bbfaa316dc2a1d0d1fac3f4500c38a90055a41b F test/wal6.test 07aa31ca8892d0527f2c5c5a9a2a87aa421dfaa8 -F test/wal7.test 09bc8de3d11949571d6f7a4188b308059cec27e5 +F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd F test/wal_common.tcl a98f17fba96206122eff624db0ab13ec377be4fe F test/walbak.test 4df1c7369da0301caeb9a48fa45997fd592380e4 F test/walbig.test e882bc1d014afffbfa2b6ba36e0f07d30a633ad0 @@ -897,6 +901,7 @@ F test/walfault.test 58fce626359c9376fe35101b5c0f2df8040aa839 F test/walhook.test ed00a40ba7255da22d6b66433ab61fab16a63483 F test/walmode.test 22ddccd073c817ac9ead62b88ac446e8dedc7d2c F test/walnoshm.test a074428046408f4eb5c6a00e09df8cc97ff93317 +F test/walro.test 2d5d69e2e99da19ce6faab340330234fc4ca0720 F test/walshared.test 6dda2293880c300baf5d791c307f653094585761 F test/walslow.test d21625e2e99e11c032ce949e8a94661576548933 F test/walthread.test a25a393c068a2b42b44333fa3fdaae9072f1617c @@ -914,24 +919,25 @@ F test/whereA.test 24c234263c8fe358f079d5e57d884fb569d2da0a F test/whereB.test 0def95db3bdec220a731c7e4bec5930327c1d8c5 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 F test/zeroblob.test caaecfb4f908f7bc086ed238668049f96774d688 +F tool/build-shell.sh 12aa4391073a777fcb6dcc490b219a018ae98bac F tool/diffdb.c 7524b1b5df217c20cd0431f6789851a4e0cb191b F tool/fragck.tcl 5265a95126abcf6ab357f7efa544787e5963f439 F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce -F tool/lemon.c dfd81a51b6e27e469ba21d01a75ddf092d429027 +F tool/lemon.c 2f182cf58a44a29107ad0027e4e696c79cbb9ad6 F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc F tool/mkkeywordhash.c d2e6b4a5965e23afb80fbe74bb54648cd371f309 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 -F tool/mksqlite3c.tcl 623e26cc8c83322e4151d3ad85ac69d41221bae8 -F tool/mksqlite3h.tcl d76c226a5e8e1f3b5f6593bcabe5e98b3b1ec9ff +F tool/mksqlite3c.tcl b23027b185d3e7c7a1803c6f977f68bebd7bc3ec +F tool/mksqlite3h.tcl 78013ad79a5e492e5f764f3c7a8ef834255061f8 F tool/mksqlite3internalh.tcl 7b43894e21bcb1bb39e11547ce7e38a063357e87 F tool/omittest.tcl b1dd290c1596e0f31fd335160a74ec5dfea3df4a F tool/opcodeDoc.awk b3a2a3d5d3075b8bd90b7afe24283efdd586659c F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5 -F tool/shell1.test 5542ecdc952f91121a835ed817e6feaf8988b333 +F tool/shell1.test 20dfe7099cf2afe37aecd69afb7678d14f7a0abf F tool/shell2.test 5dc76b8005b465f420fed8241621da7513060ff3 F tool/shell3.test 4fad469e8003938426355afdf34155f08c587836 F tool/shell4.test 35f9c3d452b4e76d5013c63e1fd07478a62f14ce @@ -948,8 +954,10 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c +F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 5b1b536cf828850d0e8ac2ab08e8696082715877 edb865c35415f9553f8279028120f7b8de2bf7e2 -R 15c7d86baba462087df3e66721c44f54 +F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d +P 832886b1e6edb916d9824924c7d88202f4eb1969 228c43c726e637daadc0c9b5a8b24243f239b1cf +R 425dc88cb473dc3555e5dfae20be1cc8 U drh -Z 3723c6e367168438b5fa877d64e1595b +Z d13f89a0688fa26bf060c192253dab5c diff --git a/manifest.uuid b/manifest.uuid index a8da7a29c8..4ffebda9e6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -832886b1e6edb916d9824924c7d88202f4eb1969 \ No newline at end of file +4c5e276c902e0b93cfc05bf2e1db966ecdac0ed0 \ No newline at end of file diff --git a/src/alter.c b/src/alter.c index aa3fa929f8..fb6d89de6f 100644 --- a/src/alter.c +++ b/src/alter.c @@ -358,14 +358,14 @@ static void reloadTableSchema(Parse *pParse, Table *pTab, const char *zName){ /* Reload the table, index and permanent trigger schemas. */ zWhere = sqlite3MPrintf(pParse->db, "tbl_name=%Q", zName); if( !zWhere ) return; - sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, zWhere, P4_DYNAMIC); + sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere); #ifndef SQLITE_OMIT_TRIGGER /* Now, if the table is not stored in the temp database, reload any temp ** triggers. Don't use IN(...) in case SQLITE_OMIT_SUBQUERY is defined. */ if( (zWhere=whereTempTriggers(pParse, pTab))!=0 ){ - sqlite3VdbeAddOp4(v, OP_ParseSchema, 1, 0, 0, zWhere, P4_DYNAMIC); + sqlite3VdbeAddParseSchemaOp(v, 1, zWhere); } #endif } diff --git a/src/btree.c b/src/btree.c index df75053743..3d7162dbb3 100644 --- a/src/btree.c +++ b/src/btree.c @@ -857,6 +857,8 @@ static int ptrmapGet(BtShared *pBt, Pgno key, u8 *pEType, Pgno *pPgno){ */ #define findCell(P,I) \ ((P)->aData + ((P)->maskPage & get2byte(&(P)->aData[(P)->cellOffset+2*(I)]))) +#define findCellv2(D,M,O,I) (D+(M&get2byte(D+(O+2*(I))))) + /* ** This a more complex version of findCell() that works for @@ -4451,7 +4453,7 @@ int sqlite3BtreeMovetoUnpacked( } assert( pCur->apPage[0]->intKey || pIdxKey ); for(;;){ - int lwr, upr; + int lwr, upr, idx; Pgno chldPg; MemPage *pPage = pCur->apPage[pCur->iPage]; int c; @@ -4467,14 +4469,14 @@ int sqlite3BtreeMovetoUnpacked( lwr = 0; upr = pPage->nCell-1; if( biasRight ){ - pCur->aiIdx[pCur->iPage] = (u16)upr; + pCur->aiIdx[pCur->iPage] = (u16)(idx = upr); }else{ - pCur->aiIdx[pCur->iPage] = (u16)((upr+lwr)/2); + pCur->aiIdx[pCur->iPage] = (u16)(idx = (upr+lwr)/2); } for(;;){ - int idx = pCur->aiIdx[pCur->iPage]; /* Index of current cell in pPage */ u8 *pCell; /* Pointer to current cell in pPage */ + assert( idx==pCur->aiIdx[pCur->iPage] ); pCur->info.nSize = 0; pCell = findCell(pPage, idx) + pPage->childPtrSize; if( pPage->intKey ){ @@ -4557,7 +4559,7 @@ int sqlite3BtreeMovetoUnpacked( if( lwr>upr ){ break; } - pCur->aiIdx[pCur->iPage] = (u16)((lwr+upr)/2); + pCur->aiIdx[pCur->iPage] = (u16)(idx = (lwr+upr)/2); } assert( lwr==upr+1 ); assert( pPage->isInit ); @@ -5390,10 +5392,10 @@ static int fillInCell( ** "sz" must be the number of bytes in the cell. */ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ - int i; /* Loop counter */ u32 pc; /* Offset to cell content of cell being deleted */ u8 *data; /* pPage->aData */ u8 *ptr; /* Used to move bytes around within data[] */ + u8 *endPtr; /* End of loop */ int rc; /* The return code */ int hdr; /* Beginning of the header. 0 most pages. 100 page 1 */ @@ -5418,9 +5420,11 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ *pRC = rc; return; } - for(i=idx+1; inCell; i++, ptr+=2){ - ptr[0] = ptr[2]; - ptr[1] = ptr[3]; + endPtr = &data[pPage->cellOffset + 2*pPage->nCell - 2]; + assert( (SQLITE_PTR_TO_INT(ptr)&1)==0 ); /* ptr is always 2-byte aligned */ + while( ptrnCell--; put2byte(&data[hdr+3], pPage->nCell); @@ -5460,6 +5464,7 @@ static void insertCell( int cellOffset; /* Address of first cell pointer in data[] */ u8 *data; /* The content of the whole page */ u8 *ptr; /* Used for moving information around in data[] */ + u8 *endPtr; /* End of the loop */ int nSkip = (iChild ? 4 : 0); @@ -5510,9 +5515,12 @@ static void insertCell( if( iChild ){ put4byte(&data[idx], iChild); } - for(j=end, ptr=&data[j]; j>ins; j-=2, ptr-=2){ - ptr[0] = ptr[-2]; - ptr[1] = ptr[-1]; + ptr = &data[end]; + endPtr = &data[ins]; + assert( (SQLITE_PTR_TO_INT(ptr)&1)==0 ); /* ptr is always 2-byte aligned */ + while( ptr>endPtr ){ + *(u16*)ptr = *(u16*)&ptr[-2]; + ptr -= 2; } put2byte(&data[ins], idx); put2byte(&data[pPage->hdrOffset+3], pPage->nCell); @@ -5557,10 +5565,11 @@ static void assemblePage( pCellptr = &data[pPage->cellOffset + nCell*2]; cellbody = nUsable; for(i=nCell-1; i>=0; i--){ + u16 sz = aSize[i]; pCellptr -= 2; - cellbody -= aSize[i]; + cellbody -= sz; put2byte(pCellptr, cellbody); - memcpy(&data[cellbody], apCell[i], aSize[i]); + memcpy(&data[cellbody], apCell[i], sz); } put2byte(&data[hdr+3], nCell); put2byte(&data[hdr+5], cellbody); @@ -6014,12 +6023,24 @@ static int balance_nonroot( memcpy(pOld->aData, apOld[i]->aData, pBt->pageSize); limit = pOld->nCell+pOld->nOverflow; - for(j=0; jnOverflow>0 ){ + for(j=0; jaData; + u16 maskPage = pOld->maskPage; + u16 cellOffset = pOld->cellOffset; + for(j=0; jpAinc!=0 && pParse->nTab==0 ) pParse->nTab = 1; - sqlite3VdbeMakeReady(v, pParse->nVar, pParse->nMem, - pParse->nTab, pParse->nMaxArg, pParse->explain, - pParse->isMultiWrite && pParse->mayAbort); + sqlite3VdbeMakeReady(v, pParse); pParse->rc = SQLITE_DONE; pParse->colNamesSet = 0; }else{ @@ -1621,8 +1619,8 @@ void sqlite3EndTable( #endif /* Reparse everything to update our internal data structures */ - sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, - sqlite3MPrintf(db, "tbl_name='%q'",p->zName), P4_DYNAMIC); + sqlite3VdbeAddParseSchemaOp(v, iDb, + sqlite3MPrintf(db, "tbl_name='%q'", p->zName)); } @@ -2819,9 +2817,8 @@ Index *sqlite3CreateIndex( if( pTblName ){ sqlite3RefillIndex(pParse, pIndex, iMem); sqlite3ChangeCookie(pParse, iDb); - sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, - sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName), - P4_DYNAMIC); + sqlite3VdbeAddParseSchemaOp(v, iDb, + sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName)); sqlite3VdbeAddOp1(v, OP_Expire, 0); } } diff --git a/src/delete.c b/src/delete.c index 8789fd2e5e..e6038ddbbb 100644 --- a/src/delete.c +++ b/src/delete.c @@ -408,6 +408,7 @@ void sqlite3DeleteFrom( const char *pVTab = (const char *)sqlite3GetVTable(db, pTab); sqlite3VtabMakeWritable(pParse, pTab); sqlite3VdbeAddOp4(v, OP_VUpdate, 0, 1, iRowid, (char*)pVTab, P4_VTAB); + sqlite3VdbeChangeP5(v, OE_Abort); sqlite3MayAbort(pParse); }else #endif diff --git a/src/expr.c b/src/expr.c index c0e9ba6fd5..be2f4d7c66 100644 --- a/src/expr.c +++ b/src/expr.c @@ -555,53 +555,53 @@ void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr){ /* Wildcard of the form "?". Assign the next variable number */ assert( z[0]=='?' ); pExpr->iColumn = (ynVar)(++pParse->nVar); - }else if( z[0]=='?' ){ - /* Wildcard of the form "?nnn". Convert "nnn" to an integer and - ** use it as the variable number */ - i64 i; - int bOk = 0==sqlite3Atoi64(&z[1], &i, sqlite3Strlen30(&z[1]), SQLITE_UTF8); - pExpr->iColumn = (ynVar)i; - testcase( i==0 ); - testcase( i==1 ); - testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]-1 ); - testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ); - if( bOk==0 || i<1 || i>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ - sqlite3ErrorMsg(pParse, "variable number must be between ?1 and ?%d", - db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]); - } - if( i>pParse->nVar ){ - pParse->nVar = (int)i; - } }else{ - /* Wildcards like ":aaa", "$aaa" or "@aaa". Reuse the same variable - ** number as the prior appearance of the same name, or if the name - ** has never appeared before, reuse the same variable number - */ - int i; - u32 n; - n = sqlite3Strlen30(z); - for(i=0; inVarExpr; i++){ - Expr *pE = pParse->apVarExpr[i]; - assert( pE!=0 ); - if( memcmp(pE->u.zToken, z, n)==0 && pE->u.zToken[n]==0 ){ - pExpr->iColumn = pE->iColumn; - break; + ynVar x = 0; + u32 n = sqlite3Strlen30(z); + if( z[0]=='?' ){ + /* Wildcard of the form "?nnn". Convert "nnn" to an integer and + ** use it as the variable number */ + i64 i; + int bOk = 0==sqlite3Atoi64(&z[1], &i, n-1, SQLITE_UTF8); + pExpr->iColumn = x = (ynVar)i; + testcase( i==0 ); + testcase( i==1 ); + testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]-1 ); + testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ); + if( bOk==0 || i<1 || i>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ + sqlite3ErrorMsg(pParse, "variable number must be between ?1 and ?%d", + db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]); + x = 0; } + if( i>pParse->nVar ){ + pParse->nVar = (int)i; + } + }else{ + /* Wildcards like ":aaa", "$aaa" or "@aaa". Reuse the same variable + ** number as the prior appearance of the same name, or if the name + ** has never appeared before, reuse the same variable number + */ + ynVar i; + for(i=0; inzVar; i++){ + if( pParse->azVar[i] && memcmp(pParse->azVar[i],z,n+1)==0 ){ + pExpr->iColumn = x = (ynVar)i+1; + break; + } + } + if( x==0 ) x = pExpr->iColumn = (ynVar)(++pParse->nVar); } - if( i>=pParse->nVarExpr ){ - pExpr->iColumn = (ynVar)(++pParse->nVar); - if( pParse->nVarExpr>=pParse->nVarExprAlloc-1 ){ - pParse->nVarExprAlloc += pParse->nVarExprAlloc + 10; - pParse->apVarExpr = - sqlite3DbReallocOrFree( - db, - pParse->apVarExpr, - pParse->nVarExprAlloc*sizeof(pParse->apVarExpr[0]) - ); + if( x>0 ){ + if( x>pParse->nzVar ){ + char **a; + a = sqlite3DbRealloc(db, pParse->azVar, x*sizeof(a[0])); + if( a==0 ) return; /* Error reported through db->mallocFailed */ + pParse->azVar = a; + memset(&a[pParse->nzVar], 0, (x-pParse->nzVar)*sizeof(a[0])); + pParse->nzVar = x; } - if( !db->mallocFailed ){ - assert( pParse->apVarExpr!=0 ); - pParse->apVarExpr[pParse->nVarExpr++] = pExpr; + if( z[0]!='?' || pParse->azVar[x-1]==0 ){ + sqlite3DbFree(db, pParse->azVar[x-1]); + pParse->azVar[x-1] = sqlite3DbStrNDup(db, z, n); } } } @@ -2345,7 +2345,9 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ assert( pExpr->u.zToken[0]!=0 ); sqlite3VdbeAddOp2(v, OP_Variable, pExpr->iColumn, target); if( pExpr->u.zToken[1]!=0 ){ - sqlite3VdbeChangeP4(v, -1, pExpr->u.zToken, P4_TRANSIENT); + assert( pExpr->u.zToken[0]=='?' + || strcmp(pExpr->u.zToken, pParse->azVar[pExpr->iColumn-1])==0 ); + sqlite3VdbeChangeP4(v, -1, pParse->azVar[pExpr->iColumn-1], P4_STATIC); } break; } diff --git a/src/fkey.c b/src/fkey.c index 34fdfda5d9..dda8a50d44 100644 --- a/src/fkey.c +++ b/src/fkey.c @@ -386,13 +386,25 @@ static void fkLookupParent( /* If the parent table is the same as the child table, and we are about ** to increment the constraint-counter (i.e. this is an INSERT operation), ** then check if the row being inserted matches itself. If so, do not - ** increment the constraint-counter. */ + ** increment the constraint-counter. + ** + ** If any of the parent-key values are NULL, then the row cannot match + ** itself. So set JUMPIFNULL to make sure we do the OP_Found if any + ** of the parent-key values are NULL (at this point it is known that + ** none of the child key values are). + */ if( pTab==pFKey->pFrom && nIncr==1 ){ int iJump = sqlite3VdbeCurrentAddr(v) + nCol + 1; for(i=0; iaiColumn[i]+1+regData; + assert( aiCol[i]!=pTab->iPKey ); + if( pIdx->aiColumn[i]==pTab->iPKey ){ + /* The parent key is a composite key that includes the IPK column */ + iParent = regData; + } sqlite3VdbeAddOp3(v, OP_Ne, iChild, iJump, iParent); + sqlite3VdbeChangeP5(v, SQLITE_JUMPIFNULL); } sqlite3VdbeAddOp2(v, OP_Goto, 0, iOk); } diff --git a/src/func.c b/src/func.c index 0b9b600d79..16de6bbbdf 100644 --- a/src/func.c +++ b/src/func.c @@ -506,10 +506,10 @@ struct compareInfo { ** whereas only characters less than 0x80 do in ASCII. */ #if defined(SQLITE_EBCDIC) -# define sqlite3Utf8Read(A,C) (*(A++)) -# define GlogUpperToLower(A) A = sqlite3UpperToLower[A] +# define sqlite3Utf8Read(A,C) (*(A++)) +# define GlogUpperToLower(A) A = sqlite3UpperToLower[A] #else -# define GlogUpperToLower(A) if( A<0x80 ){ A = sqlite3UpperToLower[A]; } +# define GlogUpperToLower(A) if( !((A)&~0x7f) ){ A = sqlite3UpperToLower[A]; } #endif static const struct compareInfo globInfo = { '*', '?', '[', 0 }; @@ -552,9 +552,9 @@ static int patternCompare( const u8 *zPattern, /* The glob pattern */ const u8 *zString, /* The string to compare against the glob */ const struct compareInfo *pInfo, /* Information about how to do the compare */ - const int esc /* The escape character */ + u32 esc /* The escape character */ ){ - int c, c2; + u32 c, c2; int invert; int seen; u8 matchOne = pInfo->matchOne; @@ -608,7 +608,7 @@ static int patternCompare( return 0; } }else if( c==matchSet ){ - int prior_c = 0; + u32 prior_c = 0; assert( esc==0 ); /* This only occurs for GLOB, not LIKE */ seen = 0; invert = 0; @@ -684,7 +684,7 @@ static void likeFunc( sqlite3_value **argv ){ const unsigned char *zA, *zB; - int escape = 0; + u32 escape = 0; int nPat; sqlite3 *db = sqlite3_context_db_handle(context); diff --git a/src/main.c b/src/main.c index 90c9d2e9e4..120327554a 100644 --- a/src/main.c +++ b/src/main.c @@ -1959,9 +1959,9 @@ int sqlite3ParseUri( const char *z; int mode; } *aMode = 0; - char *zModeType; - int mask; - int limit; + char *zModeType = 0; + int mask = 0; + int limit = 0; if( nOpt==5 && memcmp("cache", zOpt, 5)==0 ){ static struct OpenMode aCacheMode[] = { diff --git a/src/os_unix.c b/src/os_unix.c index c768dcfa76..5860ae71a9 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3537,7 +3537,8 @@ struct unixShmNode { char *zFilename; /* Name of the mmapped file */ int h; /* Open file descriptor */ int szRegion; /* Size of shared-memory regions */ - int nRegion; /* Size of array apRegion */ + u16 nRegion; /* Size of array apRegion */ + u8 isReadonly; /* True if read-only */ char **apRegion; /* Array of mapped shared-memory regions */ int nRef; /* Number of unixShm objects pointing to this */ unixShm *pFirst; /* All unixShm objects pointing to this */ @@ -3784,8 +3785,17 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ pShmNode->h = robust_open(zShmFilename, O_RDWR|O_CREAT, (sStat.st_mode & 0777)); if( pShmNode->h<0 ){ - rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zShmFilename); - goto shm_open_err; + const char *zRO; + zRO = sqlite3_uri_parameter(pDbFd->zPath, "readonly_shm"); + if( zRO && sqlite3GetBoolean(zRO) ){ + pShmNode->h = robust_open(zShmFilename, O_RDONLY, + (sStat.st_mode & 0777)); + pShmNode->isReadonly = 1; + } + if( pShmNode->h<0 ){ + rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zShmFilename); + goto shm_open_err; + } } /* Check to see if another process is holding the dead-man switch. @@ -3924,7 +3934,8 @@ static int unixShmMap( while(pShmNode->nRegion<=iRegion){ void *pMem; if( pShmNode->h>=0 ){ - pMem = mmap(0, szRegion, PROT_READ|PROT_WRITE, + pMem = mmap(0, szRegion, + pShmNode->isReadonly ? PROT_READ : PROT_READ|PROT_WRITE, MAP_SHARED, pShmNode->h, pShmNode->nRegion*szRegion ); if( pMem==MAP_FAILED ){ @@ -3950,6 +3961,7 @@ shmpage_out: }else{ *pp = 0; } + if( pShmNode->isReadonly && rc==SQLITE_OK ) rc = SQLITE_READONLY; sqlite3_mutex_leave(pShmNode->mutex); return rc; } diff --git a/src/pcache1.c b/src/pcache1.c index ad443954bf..e47265a225 100644 --- a/src/pcache1.c +++ b/src/pcache1.c @@ -574,7 +574,7 @@ static sqlite3_pcache *pcache1Create(int szPage, int bPurgeable){ pGroup = (PGroup*)&pCache[1]; pGroup->mxPinned = 10; }else{ - pGroup = &pcache1_g.grp; + pGroup = &pcache1.grp; } pCache->pGroup = pGroup; pCache->szPage = szPage; diff --git a/src/pragma.c b/src/pragma.c index 799805c40b..cd2aab223c 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -13,10 +13,6 @@ */ #include "sqliteInt.h" -/* Ignore this whole file if pragmas are disabled -*/ -#if !defined(SQLITE_OMIT_PRAGMA) - /* ** Interpret the given string as a safety level. Return 0 for OFF, ** 1 for ON or NORMAL and 2 for FULL. Return 1 for an empty or @@ -53,6 +49,12 @@ u8 sqlite3GetBoolean(const char *z){ return getSafetyLevel(z)&1; } +/* The sqlite3GetBoolean() function is used by other modules but the +** remainder of this file is specific to PRAGMA processing. So omit +** the rest of the file if PRAGMAs are omitted from the build. +*/ +#if !defined(SQLITE_OMIT_PRAGMA) + /* ** Interpret the given string as a locking mode value. */ diff --git a/src/shell.c b/src/shell.c index 8e9870a5a8..a54c922e87 100644 --- a/src/shell.c +++ b/src/shell.c @@ -2302,6 +2302,11 @@ static int do_meta_command(char *zLine, struct callback_data *p){ enableTimer = booleanValue(azArg[1]); }else + if( c=='v' && strncmp(azArg[0], "version", n)==0 ){ + printf("SQLite %s %s\n", + sqlite3_libversion(), sqlite3_sourceid()); + }else + if( c=='w' && strncmp(azArg[0], "width", n)==0 && nArg>1 ){ int j; assert( nArg<=ArraySize(azArg) ); @@ -2836,7 +2841,7 @@ int main(int argc, char **argv){ }else if( strcmp(z,"-bail")==0 ){ bail_on_error = 1; }else if( strcmp(z,"-version")==0 ){ - printf("%s\n", sqlite3_libversion()); + printf("%s %s\n", sqlite3_libversion(), sqlite3_sourceid()); return 0; }else if( strcmp(z,"-interactive")==0 ){ stdin_is_interactive = 1; @@ -2881,10 +2886,10 @@ int main(int argc, char **argv){ char *zHistory = 0; int nHistory; printf( - "SQLite version %s\n" + "SQLite version %s %.19s\n" "Enter \".help\" for instructions\n" "Enter SQL statements terminated with a \";\"\n", - sqlite3_libversion() + sqlite3_libversion(), sqlite3_sourceid() ); zHome = find_home_dir(); if( zHome ){ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index b2da5709fe..31f247bf6f 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -454,6 +454,8 @@ int sqlite3_exec( #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) #define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8)) #define SQLITE_CORRUPT_VTAB (SQLITE_CORRUPT | (1<<8)) +#define SQLITE_READONLY_RECOVERY (SQLITE_READONLY | (1<<8)) +#define SQLITE_READONLY_CANTLOCK (SQLITE_READONLY | (2<<8)) /* ** CAPI3REF: Flags For File Open Operations diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 73a466d062..59814d0c33 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -683,7 +683,7 @@ struct Db { ** A thread must be holding a mutex on the corresponding Btree in order ** to access Schema content. This implies that the thread must also be ** holding a mutex on the sqlite3 connection pointer that owns the Btree. -** For a TEMP Schema, on the connection mutex is required. +** For a TEMP Schema, only the connection mutex is required. */ struct Schema { int schema_cookie; /* Database schema version number for this file */ @@ -2238,9 +2238,8 @@ struct Parse { ** each recursion */ int nVar; /* Number of '?' variables seen in the SQL so far */ - int nVarExpr; /* Number of used slots in apVarExpr[] */ - int nVarExprAlloc; /* Number of allocated slots in apVarExpr[] */ - Expr **apVarExpr; /* Pointers to :aaa and $aaaa wildcard expressions */ + int nzVar; /* Number of available slots in azVar[] */ + char **azVar; /* Pointers to names of parameters */ Vdbe *pReprepare; /* VM being reprepared (sqlite3Reprepare()) */ int nAlias; /* Number of aliased result set columns */ int nAliasAlloc; /* Number of allocated slots for aAlias[] */ @@ -2889,7 +2888,7 @@ int sqlite3GetInt32(const char *, int*); int sqlite3Atoi(const char*); int sqlite3Utf16ByteLen(const void *pData, int nChar); int sqlite3Utf8CharLen(const char *pData, int nByte); -int sqlite3Utf8Read(const u8*, const u8**); +u32 sqlite3Utf8Read(const u8*, const u8**); /* ** Routines to read and write variable-length integers. These used to diff --git a/src/tclsqlite.c b/src/tclsqlite.c index e1f451541b..10d73ad8d2 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -3733,6 +3733,10 @@ static void init_all(Tcl_Interp *interp){ #if defined(SQLITE_ENABLE_SESSION) && defined(SQLITE_ENABLE_PREUPDATE_HOOK) extern int TestSession_Init(Tcl_Interp*); #endif +#ifdef SQLITE_ENABLE_FTS3 + extern int Sqlitetestfts3_Init(Tcl_Interp *interp); +#endif + #ifdef SQLITE_ENABLE_ZIPVFS extern int Zipvfs_Init(Tcl_Interp*); Zipvfs_Init(interp); @@ -3775,6 +3779,9 @@ static void init_all(Tcl_Interp *interp){ #if defined(SQLITE_ENABLE_SESSION) && defined(SQLITE_ENABLE_PREUPDATE_HOOK) TestSession_Init(interp); #endif +#ifdef SQLITE_ENABLE_FTS3 + Sqlitetestfts3_Init(interp); +#endif Tcl_CreateObjCommand(interp,"load_testfixture_extensions",init_all_cmd,0,0); diff --git a/src/test1.c b/src/test1.c index 50575afbbc..4d5f49fd1d 100644 --- a/src/test1.c +++ b/src/test1.c @@ -164,7 +164,8 @@ const char *sqlite3TestErrorName(int rc){ zName = "SQLITE_IOERR_CHECKRESERVEDLOCK"; break; case SQLITE_IOERR_LOCK: zName = "SQLITE_IOERR_LOCK"; break; case SQLITE_CORRUPT_VTAB: zName = "SQLITE_CORRUPT_VTAB"; break; - zName = "SQLITE_IOERR_CHECKRESERVEDLOCK"; break; + case SQLITE_READONLY_RECOVERY: zName = "SQLITE_READONLY_RECOVERY"; break; + case SQLITE_READONLY_CANTLOCK: zName = "SQLITE_READONLY_CANTLOCK"; break; default: zName = "SQLITE_Unknown"; break; } return zName; diff --git a/src/test_quota.c b/src/test_quota.c index 3c6db4d7cd..9b0e4a9ddc 100644 --- a/src/test_quota.c +++ b/src/test_quota.c @@ -323,7 +323,7 @@ static int quotaOpen( pFile=pFile->pNext){} if( pFile==0 ){ int nName = strlen(zName); - pFile = sqlite3_malloc( sizeof(*pFile) + nName + 1 ); + pFile = (quotaFile *)sqlite3_malloc( sizeof(*pFile) + nName + 1 ); if( pFile==0 ){ quotaLeave(); pSubOpen->pMethods->xClose(pSubOpen); @@ -683,7 +683,7 @@ int sqlite3_quota_set( quotaLeave(); return SQLITE_OK; } - pGroup = sqlite3_malloc( sizeof(*pGroup) + nPattern + 1 ); + pGroup = (quotaGroup *)sqlite3_malloc( sizeof(*pGroup) + nPattern + 1 ); if( pGroup==0 ){ quotaLeave(); return SQLITE_NOMEM; diff --git a/src/tokenize.c b/src/tokenize.c index c624efdcc7..b32989277e 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -353,13 +353,12 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){ testcase( z[0]=='x' ); testcase( z[0]=='X' ); if( z[1]=='\'' ){ *tokenType = TK_BLOB; - for(i=2; (c=z[i])!=0 && c!='\''; i++){ - if( !sqlite3Isxdigit(c) ){ - *tokenType = TK_ILLEGAL; - } + for(i=2; sqlite3Isxdigit(z[i]); i++){} + if( z[i]!='\'' || i%2 ){ + *tokenType = TK_ILLEGAL; + while( z[i] && z[i]!='\'' ){ i++; } } - if( i%2 || !c ) *tokenType = TK_ILLEGAL; - if( c ) i++; + if( z[i] ) i++; return i; } /* Otherwise fall through to the next case */ @@ -412,9 +411,8 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ assert( pParse->pNewTable==0 ); assert( pParse->pNewTrigger==0 ); assert( pParse->nVar==0 ); - assert( pParse->nVarExpr==0 ); - assert( pParse->nVarExprAlloc==0 ); - assert( pParse->apVarExpr==0 ); + assert( pParse->nzVar==0 ); + assert( pParse->azVar==0 ); enableLookaside = db->lookaside.bEnabled; if( db->lookaside.pStart ) db->lookaside.bEnabled = 1; while( !db->mallocFailed && zSql[i]!=0 ){ @@ -508,7 +506,8 @@ abort_parse: } sqlite3DeleteTrigger(db, pParse->pNewTrigger); - sqlite3DbFree(db, pParse->apVarExpr); + for(i=pParse->nzVar-1; i>=0; i--) sqlite3DbFree(db, pParse->azVar[i]); + sqlite3DbFree(db, pParse->azVar); sqlite3DbFree(db, pParse->aAlias); while( pParse->pAinc ){ AutoincInfo *p = pParse->pAinc; diff --git a/src/trigger.c b/src/trigger.c index 0f3f5bad39..6242de5e6c 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -301,9 +301,8 @@ void sqlite3FinishTrigger( pTrig->table, z); sqlite3DbFree(db, z); sqlite3ChangeCookie(pParse, iDb); - sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, sqlite3MPrintf( - db, "type='trigger' AND name='%q'", zName), P4_DYNAMIC - ); + sqlite3VdbeAddParseSchemaOp(v, iDb, + sqlite3MPrintf(db, "type='trigger' AND name='%q'", zName)); } if( db->init.busy ){ diff --git a/src/update.c b/src/update.c index 100f13c4bb..a2373dea59 100644 --- a/src/update.c +++ b/src/update.c @@ -244,7 +244,7 @@ void sqlite3Update( } for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ int reg; - if( chngRowid ){ + if( hasFK || chngRowid ){ reg = ++pParse->nMem; }else{ reg = 0; diff --git a/src/utf.c b/src/utf.c index 95182694d3..17f3a09a4f 100644 --- a/src/utf.c +++ b/src/utf.c @@ -163,7 +163,7 @@ static const unsigned char sqlite3Utf8Trans1[] = { || (c&0xFFFFF800)==0xD800 \ || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \ } -int sqlite3Utf8Read( +u32 sqlite3Utf8Read( const unsigned char *zIn, /* First byte of UTF-8 character */ const unsigned char **pzNext /* Write first byte past UTF-8 char here */ ){ diff --git a/src/vdbe.c b/src/vdbe.c index 9a0b9a1d74..d093354fb5 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -998,6 +998,7 @@ case OP_Variable: { /* out2-prerelease */ Mem *pVar; /* Value being transferred */ assert( pOp->p1>0 && pOp->p1<=p->nVar ); + assert( pOp->p4.z==0 || pOp->p4.z==p->azVar[pOp->p1-1] ); pVar = &p->aVar[pOp->p1 - 1]; if( sqlite3VdbeMemTooBig(pVar) ){ goto too_big; @@ -1408,15 +1409,6 @@ case OP_Function: { db->lastRowid = lastRowid; (*ctx.pFunc->xFunc)(&ctx, n, apVal); /* IMP: R-24505-23230 */ lastRowid = db->lastRowid; - if( db->mallocFailed ){ - /* Even though a malloc() has failed, the implementation of the - ** user function may have called an sqlite3_result_XXX() function - ** to return a value. The following call releases any resources - ** associated with such a value. - */ - sqlite3VdbeMemRelease(&ctx.s); - goto no_mem; - } /* If any auxiliary data functions have been called by this user function, ** immediately call the destructor for any non-static values. @@ -1427,6 +1419,16 @@ case OP_Function: { pOp->p4type = P4_VDBEFUNC; } + if( db->mallocFailed ){ + /* Even though a malloc() has failed, the implementation of the + ** user function may have called an sqlite3_result_XXX() function + ** to return a value. The following call releases any resources + ** associated with such a value. + */ + sqlite3VdbeMemRelease(&ctx.s); + goto no_mem; + } + /* If the function returned an error, throw an exception */ if( ctx.isError ){ sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&ctx.s)); @@ -1776,7 +1778,7 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ pIn3 = &aMem[pOp->p3]; flags1 = pIn1->flags; flags3 = pIn3->flags; - if( (pIn1->flags | pIn3->flags)&MEM_Null ){ + if( (flags1 | flags3)&MEM_Null ){ /* One or both operands are NULL */ if( pOp->p5 & SQLITE_NULLEQ ){ /* If SQLITE_NULLEQ is set (which will only happen if the operator is @@ -1784,7 +1786,7 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ ** or not both operands are null. */ assert( pOp->opcode==OP_Eq || pOp->opcode==OP_Ne ); - res = (pIn1->flags & pIn3->flags & MEM_Null)==0; + res = (flags1 & flags3 & MEM_Null)==0; }else{ /* SQLITE_NULLEQ is clear and at least one operand is NULL, ** then the result is always NULL. @@ -2594,7 +2596,7 @@ case OP_Savepoint: { }else{ nName = sqlite3Strlen30(zName); -#ifndef SQLITE_OMIT_VIRTUAL_TABLE +#ifndef SQLITE_OMIT_VIRTUALTABLE /* This call is Ok even if this savepoint is actually a transaction ** savepoint (and therefore should not prompt xSavepoint()) callbacks. ** If this is a transaction savepoint being opened, it is guaranteed @@ -5934,20 +5936,20 @@ case OP_MaxPgcnt: { /* out2-prerelease */ */ case OP_Trace: { char *zTrace; + char *z; - zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql); - if( zTrace ){ - if( db->xTrace ){ - char *z = sqlite3VdbeExpandSql(p, zTrace); - db->xTrace(db->pTraceArg, z); - sqlite3DbFree(db, z); - } -#ifdef SQLITE_DEBUG - if( (db->flags & SQLITE_SqlTrace)!=0 ){ - sqlite3DebugPrintf("SQL-trace: %s\n", zTrace); - } -#endif /* SQLITE_DEBUG */ + if( db->xTrace && (zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql))!=0 ){ + z = sqlite3VdbeExpandSql(p, zTrace); + db->xTrace(db->pTraceArg, z); + sqlite3DbFree(db, z); } +#ifdef SQLITE_DEBUG + if( (db->flags & SQLITE_SqlTrace)!=0 + && (zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql))!=0 + ){ + sqlite3DebugPrintf("SQL-trace: %s\n", zTrace); + } +#endif /* SQLITE_DEBUG */ break; } #endif diff --git a/src/vdbe.h b/src/vdbe.h index ef6b99fb38..402352e5a0 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -174,6 +174,7 @@ int sqlite3VdbeAddOp3(Vdbe*,int,int,int,int); int sqlite3VdbeAddOp4(Vdbe*,int,int,int,int,const char *zP4,int); int sqlite3VdbeAddOp4Int(Vdbe*,int,int,int,int,int); int sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp); +void sqlite3VdbeAddParseSchemaOp(Vdbe*,int,char*); void sqlite3VdbeChangeP1(Vdbe*, int addr, int P1); void sqlite3VdbeChangeP2(Vdbe*, int addr, int P2); void sqlite3VdbeChangeP3(Vdbe*, int addr, int P3); @@ -187,7 +188,7 @@ int sqlite3VdbeMakeLabel(Vdbe*); void sqlite3VdbeRunOnlyOnce(Vdbe*); void sqlite3VdbeDelete(Vdbe*); void sqlite3VdbeDeleteObject(sqlite3*,Vdbe*); -void sqlite3VdbeMakeReady(Vdbe*,int,int,int,int,int,int); +void sqlite3VdbeMakeReady(Vdbe*,Parse*); int sqlite3VdbeFinalize(Vdbe*); void sqlite3VdbeResolveLabel(Vdbe*, int); int sqlite3VdbeCurrentAddr(Vdbe*); @@ -196,6 +197,7 @@ int sqlite3VdbeCurrentAddr(Vdbe*); void sqlite3VdbeTrace(Vdbe*,FILE*); #endif void sqlite3VdbeResetStepResult(Vdbe*); +void sqlite3VdbeRewind(Vdbe*); int sqlite3VdbeReset(Vdbe*); void sqlite3VdbeSetNumCols(Vdbe*,int); int sqlite3VdbeSetColName(Vdbe*, int, int, const char *, void(*)(void*)); diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 893022ed67..62f2c105ba 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -287,11 +287,11 @@ struct Vdbe { Mem *aVar; /* Values for the OP_Variable opcode. */ char **azVar; /* Name of variables */ ynVar nVar; /* Number of entries in aVar[] */ + ynVar nzVar; /* Number of entries in azVar[] */ u32 cacheCtr; /* VdbeCursor row cache generation counter */ int pc; /* The program counter */ int rc; /* Value to return */ u8 errorAction; /* Recovery action to do in case of an error */ - u8 okVar; /* True if azVar[] has been initialized */ u8 explain; /* True if EXPLAIN present on SQL command */ u8 changeCntOn; /* True to update the change-counter */ u8 expired; /* True if the VM needs to be recompiled */ diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 8d6e862bcb..cdc82f1930 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -102,7 +102,7 @@ int sqlite3_reset(sqlite3_stmt *pStmt){ Vdbe *v = (Vdbe*)pStmt; sqlite3_mutex_enter(v->db->mutex); rc = sqlite3VdbeReset(v); - sqlite3VdbeMakeReady(v, -1, 0, 0, 0, 0, 0); + sqlite3VdbeRewind(v); assert( (rc & (v->db->errMask))==rc ); rc = sqlite3ApiExit(v->db, rc); sqlite3_mutex_leave(v->db->mutex); @@ -1187,32 +1187,6 @@ int sqlite3_bind_parameter_count(sqlite3_stmt *pStmt){ return p ? p->nVar : 0; } -/* -** Create a mapping from variable numbers to variable names -** in the Vdbe.azVar[] array, if such a mapping does not already -** exist. -*/ -static void createVarMap(Vdbe *p){ - if( !p->okVar ){ - int j; - Op *pOp; - sqlite3_mutex_enter(p->db->mutex); - /* The race condition here is harmless. If two threads call this - ** routine on the same Vdbe at the same time, they both might end - ** up initializing the Vdbe.azVar[] array. That is a little extra - ** work but it results in the same answer. - */ - for(j=0, pOp=p->aOp; jnOp; j++, pOp++){ - if( pOp->opcode==OP_Variable ){ - assert( pOp->p1>0 && pOp->p1<=p->nVar ); - p->azVar[pOp->p1-1] = pOp->p4.z; - } - } - p->okVar = 1; - sqlite3_mutex_leave(p->db->mutex); - } -} - /* ** Return the name of a wildcard parameter. Return NULL if the index ** is out of range or if the wildcard is unnamed. @@ -1221,10 +1195,9 @@ static void createVarMap(Vdbe *p){ */ const char *sqlite3_bind_parameter_name(sqlite3_stmt *pStmt, int i){ Vdbe *p = (Vdbe*)pStmt; - if( p==0 || i<1 || i>p->nVar ){ + if( p==0 || i<1 || i>p->nzVar ){ return 0; } - createVarMap(p); return p->azVar[i-1]; } @@ -1238,9 +1211,8 @@ int sqlite3VdbeParameterIndex(Vdbe *p, const char *zName, int nName){ if( p==0 ){ return 0; } - createVarMap(p); if( zName ){ - for(i=0; inVar; i++){ + for(i=0; inzVar; i++){ const char *z = p->azVar[i]; if( z && memcmp(z,zName,nName)==0 && z[nName]==0 ){ return i+1; diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 26e8bf9c77..a45f7a2892 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -157,13 +157,6 @@ int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){ pOp->p3 = p3; pOp->p4.p = 0; pOp->p4type = P4_NOTUSED; - p->expired = 0; - if( op==OP_ParseSchema ){ - /* Any program that uses the OP_ParseSchema opcode needs to lock - ** all btrees. */ - int j; - for(j=0; jdb->nDb; j++) sqlite3VdbeUsesBtree(p, j); - } #ifdef SQLITE_DEBUG pOp->zComment = 0; if( sqlite3VdbeAddopTrace ) sqlite3VdbePrintOp(0, i, &p->aOp[i]); @@ -202,6 +195,20 @@ int sqlite3VdbeAddOp4( return addr; } +/* +** Add an OP_ParseSchema opcode. This routine is broken out from +** sqlite3VdbeAddOp4() since it needs to also local all btrees. +** +** The zWhere string must have been obtained from sqlite3_malloc(). +** This routine will take ownership of the allocated memory. +*/ +void sqlite3VdbeAddParseSchemaOp(Vdbe *p, int iDb, char *zWhere){ + int j; + int addr = sqlite3VdbeAddOp3(p, OP_ParseSchema, iDb, 0, 0); + sqlite3VdbeChangeP4(p, addr, zWhere, P4_DYNAMIC); + for(j=0; jdb->nDb; j++) sqlite3VdbeUsesBtree(p, j); +} + /* ** Add an opcode that includes the p4 value as an integer. */ @@ -1392,34 +1399,13 @@ static void *allocSpace( } /* -** Prepare a virtual machine for execution. This involves things such -** as allocating stack space and initializing the program counter. -** After the VDBE has be prepped, it can be executed by one or more -** calls to sqlite3VdbeExec(). -** -** This is the only way to move a VDBE from VDBE_MAGIC_INIT to -** VDBE_MAGIC_RUN. -** -** This function may be called more than once on a single virtual machine. -** The first call is made while compiling the SQL statement. Subsequent -** calls are made as part of the process of resetting a statement to be -** re-executed (from a call to sqlite3_reset()). The nVar, nMem, nCursor -** and isExplain parameters are only passed correct values the first time -** the function is called. On subsequent calls, from sqlite3_reset(), nVar -** is passed -1 and nMem, nCursor and isExplain are all passed zero. +** Rewind the VDBE back to the beginning in preparation for +** running it. */ -void sqlite3VdbeMakeReady( - Vdbe *p, /* The VDBE */ - int nVar, /* Number of '?' see in the SQL statement */ - int nMem, /* Number of memory cells to allocate */ - int nCursor, /* Number of cursors to allocate */ - int nArg, /* Maximum number of args in SubPrograms */ - int isExplain, /* True if the EXPLAIN keywords is present */ - int usesStmtJournal /* True to set Vdbe.usesStmtJournal */ -){ - int n; - sqlite3 *db = p->db; - +void sqlite3VdbeRewind(Vdbe *p){ +#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) + int i; +#endif assert( p!=0 ); assert( p->magic==VDBE_MAGIC_INIT ); @@ -1430,6 +1416,71 @@ void sqlite3VdbeMakeReady( /* Set the magic to VDBE_MAGIC_RUN sooner rather than later. */ p->magic = VDBE_MAGIC_RUN; +#ifdef SQLITE_DEBUG + for(i=1; inMem; i++){ + assert( p->aMem[i].db==p->db ); + } +#endif + p->pc = -1; + p->rc = SQLITE_OK; + p->errorAction = OE_Abort; + p->magic = VDBE_MAGIC_RUN; + p->nChange = 0; + p->cacheCtr = 1; + p->minWriteFileFormat = 255; + p->iStatement = 0; + p->nFkConstraint = 0; +#ifdef VDBE_PROFILE + for(i=0; inOp; i++){ + p->aOp[i].cnt = 0; + p->aOp[i].cycles = 0; + } +#endif +} + +/* +** Prepare a virtual machine for execution for the first time after +** creating the virtual machine. This involves things such +** as allocating stack space and initializing the program counter. +** After the VDBE has be prepped, it can be executed by one or more +** calls to sqlite3VdbeExec(). +** +** This function may be called exact once on a each virtual machine. +** After this routine is called the VM has been "packaged" and is ready +** to run. After this routine is called, futher calls to +** sqlite3VdbeAddOp() functions are prohibited. This routine disconnects +** the Vdbe from the Parse object that helped generate it so that the +** the Vdbe becomes an independent entity and the Parse object can be +** destroyed. +** +** Use the sqlite3VdbeRewind() procedure to restore a virtual machine back +** to its initial state after it has been run. +*/ +void sqlite3VdbeMakeReady( + Vdbe *p, /* The VDBE */ + Parse *pParse /* Parsing context */ +){ + sqlite3 *db; /* The database connection */ + int nVar; /* Number of parameters */ + int nMem; /* Number of VM memory registers */ + int nCursor; /* Number of cursors required */ + int nArg; /* Number of arguments in subprograms */ + int n; /* Loop counter */ + u8 *zCsr; /* Memory available for allocation */ + u8 *zEnd; /* First byte past allocated memory */ + int nByte; /* How much extra memory is needed */ + + assert( p!=0 ); + assert( p->nOp>0 ); + assert( pParse!=0 ); + assert( p->magic==VDBE_MAGIC_INIT ); + db = p->db; + assert( db->mallocFailed==0 ); + nVar = pParse->nVar; + nMem = pParse->nMem; + nCursor = pParse->nTab; + nArg = pParse->nMaxArg; + /* For each cursor required, also allocate a memory cell. Memory ** cells (nMem+1-nCursor)..nMem, inclusive, will never be used by ** the vdbe program. Instead they are used to allocate space for @@ -1442,91 +1493,68 @@ void sqlite3VdbeMakeReady( nMem += nCursor; /* Allocate space for memory registers, SQL variables, VDBE cursors and - ** an array to marshal SQL function arguments in. This is only done the - ** first time this function is called for a given VDBE, not when it is - ** being called from sqlite3_reset() to reset the virtual machine. + ** an array to marshal SQL function arguments in. */ - if( nVar>=0 && ALWAYS(db->mallocFailed==0) ){ - u8 *zCsr = (u8 *)&p->aOp[p->nOp]; /* Memory avaliable for alloation */ - u8 *zEnd = (u8 *)&p->aOp[p->nOpAlloc]; /* First byte past available mem */ - int nByte; /* How much extra memory needed */ + zCsr = (u8*)&p->aOp[p->nOp]; /* Memory avaliable for allocation */ + zEnd = (u8*)&p->aOp[p->nOpAlloc]; /* First byte past end of zCsr[] */ - resolveP2Values(p, &nArg); - p->usesStmtJournal = (u8)usesStmtJournal; - if( isExplain && nMem<10 ){ - nMem = 10; + resolveP2Values(p, &nArg); + p->usesStmtJournal = (u8)(pParse->isMultiWrite && pParse->mayAbort); + if( pParse->explain && nMem<10 ){ + nMem = 10; + } + memset(zCsr, 0, zEnd-zCsr); + zCsr += (zCsr - (u8*)0)&7; + assert( EIGHT_BYTE_ALIGNMENT(zCsr) ); + + /* Memory for registers, parameters, cursor, etc, is allocated in two + ** passes. On the first pass, we try to reuse unused space at the + ** end of the opcode array. If we are unable to satisfy all memory + ** requirements by reusing the opcode array tail, then the second + ** pass will fill in the rest using a fresh allocation. + ** + ** This two-pass approach that reuses as much memory as possible from + ** the leftover space at the end of the opcode array can significantly + ** reduce the amount of memory held by a prepared statement. + */ + do { + nByte = 0; + p->aMem = allocSpace(p->aMem, nMem*sizeof(Mem), &zCsr, zEnd, &nByte); + p->aVar = allocSpace(p->aVar, nVar*sizeof(Mem), &zCsr, zEnd, &nByte); + p->apArg = allocSpace(p->apArg, nArg*sizeof(Mem*), &zCsr, zEnd, &nByte); + p->azVar = allocSpace(p->azVar, nVar*sizeof(char*), &zCsr, zEnd, &nByte); + p->apCsr = allocSpace(p->apCsr, nCursor*sizeof(VdbeCursor*), + &zCsr, zEnd, &nByte); + if( nByte ){ + p->pFree = sqlite3DbMallocZero(db, nByte); } - memset(zCsr, 0, zEnd-zCsr); - zCsr += (zCsr - (u8*)0)&7; - assert( EIGHT_BYTE_ALIGNMENT(zCsr) ); + zCsr = p->pFree; + zEnd = &zCsr[nByte]; + }while( nByte && !db->mallocFailed ); - /* Memory for registers, parameters, cursor, etc, is allocated in two - ** passes. On the first pass, we try to reuse unused space at the - ** end of the opcode array. If we are unable to satisfy all memory - ** requirements by reusing the opcode array tail, then the second - ** pass will fill in the rest using a fresh allocation. - ** - ** This two-pass approach that reuses as much memory as possible from - ** the leftover space at the end of the opcode array can significantly - ** reduce the amount of memory held by a prepared statement. - */ - do { - nByte = 0; - p->aMem = allocSpace(p->aMem, nMem*sizeof(Mem), &zCsr, zEnd, &nByte); - p->aVar = allocSpace(p->aVar, nVar*sizeof(Mem), &zCsr, zEnd, &nByte); - p->apArg = allocSpace(p->apArg, nArg*sizeof(Mem*), &zCsr, zEnd, &nByte); - p->azVar = allocSpace(p->azVar, nVar*sizeof(char*), &zCsr, zEnd, &nByte); - p->apCsr = allocSpace(p->apCsr, nCursor*sizeof(VdbeCursor*), - &zCsr, zEnd, &nByte); - if( nByte ){ - p->pFree = sqlite3DbMallocZero(db, nByte); - } - zCsr = p->pFree; - zEnd = &zCsr[nByte]; - }while( nByte && !db->mallocFailed ); - - p->nCursor = (u16)nCursor; - if( p->aVar ){ - p->nVar = (ynVar)nVar; - for(n=0; naVar[n].flags = MEM_Null; - p->aVar[n].db = db; - } - } - if( p->aMem ){ - p->aMem--; /* aMem[] goes from 1..nMem */ - p->nMem = nMem; /* not from 0..nMem-1 */ - for(n=1; n<=nMem; n++){ - p->aMem[n].flags = MEM_Null; - p->aMem[n].db = db; - } + p->nCursor = (u16)nCursor; + if( p->aVar ){ + p->nVar = (ynVar)nVar; + for(n=0; naVar[n].flags = MEM_Null; + p->aVar[n].db = db; } } -#ifdef SQLITE_DEBUG - for(n=1; nnMem; n++){ - assert( p->aMem[n].db==db ); + if( p->azVar ){ + p->nzVar = pParse->nzVar; + memcpy(p->azVar, pParse->azVar, p->nzVar*sizeof(p->azVar[0])); + memset(pParse->azVar, 0, pParse->nzVar*sizeof(pParse->azVar[0])); } -#endif - - p->pc = -1; - p->rc = SQLITE_OK; - p->errorAction = OE_Abort; - p->explain |= isExplain; - p->magic = VDBE_MAGIC_RUN; - p->nChange = 0; - p->cacheCtr = 1; - p->minWriteFileFormat = 255; - p->iStatement = 0; - p->nFkConstraint = 0; -#ifdef VDBE_PROFILE - { - int i; - for(i=0; inOp; i++){ - p->aOp[i].cnt = 0; - p->aOp[i].cycles = 0; + if( p->aMem ){ + p->aMem--; /* aMem[] goes from 1..nMem */ + p->nMem = nMem; /* not from 0..nMem-1 */ + for(n=1; n<=nMem; n++){ + p->aMem[n].flags = MEM_Null; + p->aMem[n].db = db; } } -#endif + p->explain = pParse->explain; + sqlite3VdbeRewind(p); } /* @@ -2400,6 +2428,7 @@ void sqlite3VdbeDeleteAuxData(VdbeFunc *pVdbeFunc, int mask){ */ void sqlite3VdbeDeleteObject(sqlite3 *db, Vdbe *p){ SubProgram *pSub, *pNext; + int i; assert( p->db==0 || p->db==db ); releaseMemArray(p->aVar, p->nVar); releaseMemArray(p->aColName, p->nResColumn*COLNAME_N); @@ -2408,6 +2437,7 @@ void sqlite3VdbeDeleteObject(sqlite3 *db, Vdbe *p){ vdbeFreeOpArray(db, pSub->aOp, pSub->nOp); sqlite3DbFree(db, pSub); } + for(i=p->nzVar-1; i>=0; i--) sqlite3DbFree(db, p->azVar[i]); vdbeFreeOpArray(db, p->aOp, p->nOp); sqlite3DbFree(db, p->aLabel); sqlite3DbFree(db, p->aColName); @@ -2853,7 +2883,7 @@ UnpackedRecord *sqlite3VdbeRecordUnpack( idx += getVarint32(&aKey[idx], serial_type); pMem->enc = pKeyInfo->enc; pMem->db = pKeyInfo->db; - pMem->flags = 0; + /* pMem->flags = 0; // sqlite3VdbeSerialGet() will set this for us */ pMem->zMalloc = 0; pMem->z = 0; d += sqlite3VdbeSerialGet(&aKey[d], serial_type, pMem); @@ -2869,6 +2899,7 @@ UnpackedRecord *sqlite3VdbeRecordUnpack( ** This routine destroys a UnpackedRecord object. */ void sqlite3VdbeDeleteUnpackedRecord(UnpackedRecord *p){ +#ifdef SQLITE_DEBUG int i; Mem *pMem; @@ -2882,6 +2913,7 @@ void sqlite3VdbeDeleteUnpackedRecord(UnpackedRecord *p){ */ if( pMem->zMalloc ) sqlite3VdbeMemRelease(pMem); } +#endif if( p->flags & UNPACKED_NEED_FREE ){ sqlite3DbFree(p->pKeyInfo->db, p); } diff --git a/src/vdbeblob.c b/src/vdbeblob.c index 18fdd465ae..a8728e6d25 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -297,7 +297,10 @@ int sqlite3_blob_open( sqlite3VdbeChangeP4(v, 3+flags, SQLITE_INT_TO_PTR(pTab->nCol+1),P4_INT32); sqlite3VdbeChangeP2(v, 7, pTab->nCol); if( !db->mallocFailed ){ - sqlite3VdbeMakeReady(v, 1, 1, 1, 0, 0, 0); + pParse->nVar = 1; + pParse->nMem = 1; + pParse->nTab = 1; + sqlite3VdbeMakeReady(v, pParse); } } diff --git a/src/vtab.c b/src/vtab.c index dffd6a2664..223ef4e7bd 100644 --- a/src/vtab.c +++ b/src/vtab.c @@ -383,7 +383,7 @@ void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){ sqlite3VdbeAddOp2(v, OP_Expire, 0, 0); zWhere = sqlite3MPrintf(db, "name='%q' AND type='table'", pTab->zName); - sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, zWhere, P4_DYNAMIC); + sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere); sqlite3VdbeAddOp4(v, OP_VCreate, iDb, 0, 0, pTab->zName, sqlite3Strlen30(pTab->zName) + 1); } diff --git a/src/wal.c b/src/wal.c index d90fb05d8b..b9a03dff23 100644 --- a/src/wal.c +++ b/src/wal.c @@ -420,7 +420,7 @@ struct Wal { u8 exclusiveMode; /* Non-zero if connection is in exclusive mode */ u8 writeLock; /* True if in a write transaction */ u8 ckptLock; /* True if holding a checkpoint lock */ - u8 readOnly; /* True if the WAL file is open read-only */ + u8 readOnly; /* WAL_RDWR, WAL_RDONLY, or WAL_SHM_RDONLY */ WalIndexHdr hdr; /* Wal-index header for current transaction */ const char *zWalName; /* Name of WAL file */ u32 nCkpt; /* Checkpoint sequence counter in the wal-header */ @@ -436,6 +436,13 @@ struct Wal { #define WAL_EXCLUSIVE_MODE 1 #define WAL_HEAPMEMORY_MODE 2 +/* +** Possible values for WAL.readOnly +*/ +#define WAL_RDWR 0 /* Normal read/write connection */ +#define WAL_RDONLY 1 /* The WAL file is readonly */ +#define WAL_SHM_RDONLY 2 /* The SHM file is readonly */ + /* ** Each page of the wal-index mapping contains a hash-table made up of ** an array of HASHTABLE_NSLOT elements of the following type. @@ -529,6 +536,10 @@ static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){ rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, pWal->writeLock, (void volatile **)&pWal->apWiData[iPage] ); + if( rc==SQLITE_READONLY ){ + pWal->readOnly |= WAL_SHM_RDONLY; + rc = SQLITE_OK; + } } } @@ -1276,7 +1287,7 @@ int sqlite3WalOpen( flags = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL); rc = sqlite3OsOpen(pVfs, zWalName, pRet->pWalFd, flags, &flags); if( rc==SQLITE_OK && flags&SQLITE_OPEN_READONLY ){ - pRet->readOnly = 1; + pRet->readOnly = WAL_RDONLY; } if( rc!=SQLITE_OK ){ @@ -1917,21 +1928,28 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ ** with a writer. So get a WRITE lock and try again. */ assert( badHdr==0 || pWal->writeLock==0 ); - if( badHdr && SQLITE_OK==(rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1)) ){ - pWal->writeLock = 1; - if( SQLITE_OK==(rc = walIndexPage(pWal, 0, &page0)) ){ - badHdr = walIndexTryHdr(pWal, pChanged); - if( badHdr ){ - /* If the wal-index header is still malformed even while holding - ** a WRITE lock, it can only mean that the header is corrupted and - ** needs to be reconstructed. So run recovery to do exactly that. - */ - rc = walIndexRecover(pWal); - *pChanged = 1; + if( badHdr ){ + if( pWal->readOnly & WAL_SHM_RDONLY ){ + if( SQLITE_OK==(rc = walLockShared(pWal, WAL_WRITE_LOCK)) ){ + walUnlockShared(pWal, WAL_WRITE_LOCK); + rc = SQLITE_READONLY_RECOVERY; } + }else if( SQLITE_OK==(rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1)) ){ + pWal->writeLock = 1; + if( SQLITE_OK==(rc = walIndexPage(pWal, 0, &page0)) ){ + badHdr = walIndexTryHdr(pWal, pChanged); + if( badHdr ){ + /* If the wal-index header is still malformed even while holding + ** a WRITE lock, it can only mean that the header is corrupted and + ** needs to be reconstructed. So run recovery to do exactly that. + */ + rc = walIndexRecover(pWal); + *pChanged = 1; + } + } + pWal->writeLock = 0; + walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); } - pWal->writeLock = 0; - walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); } /* If the header is read successfully, check the version number to make @@ -2118,7 +2136,9 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ } /* There was once an "if" here. The extra "{" is to preserve indentation. */ { - if( mxReadMark < pWal->hdr.mxFrame || mxI==0 ){ + if( (pWal->readOnly & WAL_SHM_RDONLY)==0 + && (mxReadMarkhdr.mxFrame || mxI==0) + ){ for(i=1; ireadOnly & WAL_SHM_RDONLY)!=0 ); + return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTLOCK; } rc = walLockShared(pWal, WAL_READ_LOCK(mxI)); @@ -2775,6 +2795,7 @@ int sqlite3WalCheckpoint( assert( pWal->ckptLock==0 ); assert( pWal->writeLock==0 ); + if( pWal->readOnly ) return SQLITE_READONLY; WALTRACE(("WAL%p: checkpoint begins\n", pWal)); rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); if( rc ){ diff --git a/test/e_uri.test b/test/e_uri.test index b0e2f876de..3e47427941 100644 --- a/test/e_uri.test +++ b/test/e_uri.test @@ -48,10 +48,11 @@ proc open_uri_error {uri} { # and the filename argument begins with "file:", then the filename is # interpreted as a URI. # -# EVIDENCE-OF: R-00067-59538 URI filename interpretation is enabled if +# EVIDENCE-OF: R-32637-34037 URI filename interpretation is enabled if # the SQLITE_OPEN_URI flag is is set in the fourth argument to # sqlite3_open_v2(), or if it has been enabled globally using the -# SQLITE_CONFIG_URI option with the sqlite3_config() method. +# SQLITE_CONFIG_URI option with the sqlite3_config() method or by the +# SQLITE_USE_URI compile-time option. # if {$tcl_platform(platform) == "unix"} { set flags [list SQLITE_OPEN_READWRITE SQLITE_OPEN_CREATE] @@ -140,8 +141,8 @@ if {$tcl_platform(platform) == "unix"} { } } -# EVIDENCE-OF: R-43804-65312 The 'fragment' component of a URI, if -# present, is always ignored. +# EVIDENCE-OF: R-45981-25528 The fragment component of a URI, if +# present, is ignored. # # It is difficult to test that something is ignore correctly. So these tests # just show that adding a fragment does not interfere with the pathname or @@ -157,14 +158,15 @@ if {$tcl_platform(platform) == "unix"} { } } -# EVIDENCE-OF: R-00273-20588 SQLite uses the 'path' component of the URI -# as the path to the database file to open. +# EVIDENCE-OF: R-62557-09390 SQLite uses the path component of the URI +# as the name of the disk file which contains the database. # # EVIDENCE-OF: R-28659-11035 If the path begins with a '/' character, # then it is interpreted as an absolute path. # -# EVIDENCE-OF: R-39349-47203 If it does not begin with a '/', it is -# interpreted as a relative path. +# EVIDENCE-OF: R-46234-61323 If the path does not begin with a '/' +# (meaning that the authority section is omitted from the URI) then the +# path is interpreted as a relative path. # if {$tcl_platform(platform) == "unix"} { foreach {tn uri parse} " diff --git a/test/fkey3.test b/test/fkey3.test index 88b5aad370..6a11f8883e 100644 --- a/test/fkey3.test +++ b/test/fkey3.test @@ -21,6 +21,8 @@ ifcapable {!foreignkey||!trigger} { return } +set testprefix fkey3 + # Create a table and some data to work with. # do_test fkey3-1.1 { @@ -77,4 +79,108 @@ do_test fkey3-2.1 { } } {1 100 1 101 2 100 2 101} + +#------------------------------------------------------------------------- +# The following tests - fkey-3.* - test some edge cases to do with +# inserting rows into tables that have foreign keys where the parent +# table is the same as the child table. Especially cases where the +# new row being inserted matches itself. +# +do_execsql_test 3.1.1 { + CREATE TABLE t3(a, b, c, d, + UNIQUE(a, b), + FOREIGN KEY(c, d) REFERENCES t3(a, b) + ); + INSERT INTO t3 VALUES(1, 2, 1, 2); +} {} +do_catchsql_test 3.1.2 { + INSERT INTO t3 VALUES(NULL, 2, 5, 2); +} {1 {foreign key constraint failed}} +do_catchsql_test 3.1.3 { + INSERT INTO t3 VALUES(NULL, 3, 5, 2); +} {1 {foreign key constraint failed}} + +do_execsql_test 3.2.1 { + CREATE TABLE t4(a UNIQUE, b REFERENCES t4(a)); +} +do_catchsql_test 3.2.2 { + INSERT INTO t4 VALUES(NULL, 1); +} {1 {foreign key constraint failed}} + +do_execsql_test 3.3.1 { + CREATE TABLE t5(a INTEGER PRIMARY KEY, b REFERENCES t5(a)); + INSERT INTO t5 VALUES(NULL, 1); +} {} +do_catchsql_test 3.3.2 { + INSERT INTO t5 VALUES(NULL, 3); +} {1 {foreign key constraint failed}} + +do_execsql_test 3.4.1 { + CREATE TABLE t6(a INTEGER PRIMARY KEY, b, c, d, + FOREIGN KEY(c, d) REFERENCES t6(a, b) + ); + CREATE UNIQUE INDEX t6i ON t6(b, a); +} +do_execsql_test 3.4.2 { INSERT INTO t6 VALUES(NULL, 'a', 1, 'a'); } {} +do_execsql_test 3.4.3 { INSERT INTO t6 VALUES(2, 'a', 2, 'a'); } {} +do_execsql_test 3.4.4 { INSERT INTO t6 VALUES(NULL, 'a', 1, 'a'); } {} +do_execsql_test 3.4.5 { INSERT INTO t6 VALUES(5, 'a', 2, 'a'); } {} +do_catchsql_test 3.4.6 { + INSERT INTO t6 VALUES(NULL, 'a', 65, 'a'); +} {1 {foreign key constraint failed}} + +do_execsql_test 3.4.7 { + INSERT INTO t6 VALUES(100, 'one', 100, 'one'); + DELETE FROM t6 WHERE a = 100; +} +do_execsql_test 3.4.8 { + INSERT INTO t6 VALUES(100, 'one', 100, 'one'); + UPDATE t6 SET c = 1, d = 'a' WHERE a = 100; + DELETE FROM t6 WHERE a = 100; +} + +do_execsql_test 3.5.1 { + CREATE TABLE t7(a, b, c, d INTEGER PRIMARY KEY, + FOREIGN KEY(c, d) REFERENCES t7(a, b) + ); + CREATE UNIQUE INDEX t7i ON t7(a, b); +} +do_execsql_test 3.5.2 { INSERT INTO t7 VALUES('x', 1, 'x', NULL) } {} +do_execsql_test 3.5.3 { INSERT INTO t7 VALUES('x', 2, 'x', 2) } {} +do_catchsql_test 3.5.4 { + INSERT INTO t7 VALUES('x', 450, 'x', NULL); +} {1 {foreign key constraint failed}} +do_catchsql_test 3.5.5 { + INSERT INTO t7 VALUES('x', 450, 'x', 451); +} {1 {foreign key constraint failed}} + + +do_execsql_test 3.6.1 { + CREATE TABLE t8(a, b, c, d, e, FOREIGN KEY(c, d) REFERENCES t8(a, b)); + CREATE UNIQUE INDEX t8i1 ON t8(a, b); + CREATE UNIQUE INDEX t8i2 ON t8(c); + INSERT INTO t8 VALUES(1, 1, 1, 1, 1); +} +do_catchsql_test 3.6.2 { + UPDATE t8 SET d = 2; +} {1 {foreign key constraint failed}} +do_execsql_test 3.6.3 { UPDATE t8 SET d = 1; } +do_execsql_test 3.6.4 { UPDATE t8 SET e = 2; } + +do_catchsql_test 3.6.5 { + CREATE TABLE TestTable ( + id INTEGER PRIMARY KEY, + name text, + source_id integer not null, + parent_id integer, + + foreign key(source_id, parent_id) references TestTable(source_id, id) + ); + CREATE UNIQUE INDEX testindex on TestTable(source_id, id); + PRAGMA foreign_keys=1; + INSERT INTO TestTable VALUES (1, 'parent', 1, null); + INSERT INTO TestTable VALUES (2, 'child', 1, 1); + UPDATE TestTable SET parent_id=1000 where id=2; +} {1 {foreign key constraint failed}} + finish_test diff --git a/test/fts3auto.test b/test/fts3auto.test new file mode 100644 index 0000000000..ed96129a70 --- /dev/null +++ b/test/fts3auto.test @@ -0,0 +1,592 @@ +# 2011 June 10 +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# If this build does not include FTS3, skip the tests in this file. +# +ifcapable !fts3 { finish_test ; return } +source $testdir/fts3_common.tcl +source $testdir/malloc_common.tcl + +set testprefix fts3auto +set sfep $sqlite_fts3_enable_parentheses +set sqlite_fts3_enable_parentheses 1 + +#-------------------------------------------------------------------------- +# Start of Tcl infrastructure used by tests. The entry points are: +# +# do_fts3query_test +# fts3_make_deferrable +# fts3_zero_long_segments +# + +# +# do_fts3query_test TESTNAME ?OPTIONS? TABLE MATCHEXPR +# +# This proc runs several test cases on FTS3/4 table $TABLE using match +# expression $MATCHEXPR. All documents in $TABLE must be formatted so that +# they can be "tokenized" using the Tcl list commands (llength, lindex etc.). +# The name and column names used by $TABLE must not require any quoting or +# escaping when used in SQL statements. +# +# $MATCHINFO may be any expression accepted by the FTS4 MATCH operator, +# except that the ":token" syntax is not supported. Tcl list +# commands are used to tokenize the expression. Any parenthesis must appear +# either as separate list elements, or as the first (for opening) or last +# (for closing) character of a list element. i.e. the expression "(a OR b)c" +# will not be parsed correctly, but "( a OR b) c" will. +# +# Available OPTIONS are: +# +# -deferred TOKENLIST +# +# If the "deferred" option is supplied, it is passed a list of tokens that +# are deferred by FTS and result in the relevant matchinfo() stats being an +# approximation. +# +set sqlite_fts3_enable_parentheses 1 +proc do_fts3query_test {tn args} { + + set nArg [llength $args] + if {$nArg < 2 || ($nArg % 2)} { + set cmd do_fts3query_test + error "wrong # args: should be \"$cmd ?-deferred LIST? TABLE MATCHEXPR\"" + } + set tbl [lindex $args [expr $nArg-2]] + set match [lindex $args [expr $nArg-1]] + set deferred [list] + + foreach {k v} [lrange $args 0 [expr $nArg-3]] { + switch -- $k { + -deferred { + set deferred $v + } + default { + error "bad option \"$k\": must be -deferred" + } + } + } + + get_near_results $tbl $match $deferred aMatchinfo + + set matchinfo_asc [list] + foreach docid [lsort -integer -incr [array names aMatchinfo]] { + lappend matchinfo_asc $docid $aMatchinfo($docid) + } + set matchinfo_desc [list] + foreach docid [lsort -integer -decr [array names aMatchinfo]] { + lappend matchinfo_desc $docid $aMatchinfo($docid) + } + + set title "(\"$match\" -> [llength [array names aMatchinfo]] rows)" + + do_execsql_test $tn$title.1 " + SELECT docid FROM $tbl WHERE $tbl MATCH '$match' ORDER BY docid ASC + " [lsort -integer -incr [array names aMatchinfo]] + + do_execsql_test $tn$title.2 " + SELECT docid FROM $tbl WHERE $tbl MATCH '$match' ORDER BY docid DESC + " [lsort -integer -decr [array names aMatchinfo]] + + do_execsql_test $tn$title.3 " + SELECT docid, mit(matchinfo($tbl, 'x')) FROM $tbl + WHERE $tbl MATCH '$match' ORDER BY docid DESC + " $matchinfo_desc + + do_execsql_test $tn$title.4 " + SELECT docid, mit(matchinfo($tbl, 'x')) FROM $tbl + WHERE $tbl MATCH '$match' ORDER BY docid ASC + " $matchinfo_asc +} + +# fts3_make_deferrable TABLE TOKEN +# +proc fts3_make_deferrable {tbl token} { + + set stmt [sqlite3_prepare db "SELECT * FROM $tbl" -1 dummy] + set name [sqlite3_column_name $stmt 0] + sqlite3_finalize $stmt + + set nRow [db one "SELECT count(*) FROM $tbl"] + set pgsz [db one "PRAGMA page_size"] + execsql BEGIN + for {set i 0} {$i < ($nRow * $pgsz * 1.2)/100} {incr i} { + set doc [string repeat "$token " 100] + execsql "INSERT INTO $tbl ($name) VALUES(\$doc)" + } + execsql "INSERT INTO $tbl ($name) VALUES('aaaaaaa ${token}aaaaa')" + execsql COMMIT + + return [expr $nRow*$pgsz] +} + +# fts3_zero_long_segments TABLE ?LIMIT? +# +proc fts3_zero_long_segments {tbl limit} { + execsql " + UPDATE ${tbl}_segments + SET block = zeroblob(length(block)) + WHERE length(block)>$limit + " + return [db changes] +} + + +proc mit {blob} { + set scan(littleEndian) i* + set scan(bigEndian) I* + binary scan $blob $scan($::tcl_platform(byteOrder)) r + return $r +} +db func mit mit + +proc fix_near_expr {expr} { + set out [list] + lappend out [lindex $expr 0] + foreach {a b} [lrange $expr 1 end] { + if {[string match -nocase near $a]} { set a 10 } + if {[string match -nocase near/* $a]} { set a [string range $a 5 end] } + lappend out $a + lappend out $b + } + return $out +} + +proc get_single_near_results {tbl expr deferred arrayvar nullvar} { + upvar $arrayvar aMatchinfo + upvar $nullvar nullentry + catch {array unset aMatchinfo} + + set expr [fix_near_expr $expr] + + # Calculate the expected results using [fts3_near_match]. The following + # loop populates the "hits" and "counts" arrays as follows: + # + # 1. For each document in the table that matches the NEAR expression, + # hits($docid) is set to 1. The set of docids that match the expression + # can therefore be found using [array names hits]. + # + # 2. For each column of each document in the table, counts($docid,$iCol) + # is set to the -phrasecountvar output. + # + set res [list] + catch { array unset hits } + db eval "SELECT docid, * FROM $tbl" d { + set iCol 0 + foreach col [lrange $d(*) 1 end] { + set docid $d(docid) + set hit [fts3_near_match $d($col) $expr -p counts($docid,$iCol)] + if {$hit} { set hits($docid) 1 } + incr iCol + } + } + set nPhrase [expr ([llength $expr]+1)/2] + set nCol $iCol + + # This block populates the nHit and nDoc arrays. For each phrase/column + # in the query/table, array elements are set as follows: + # + # nHit($iPhrase,$iCol) - Total number of hits for phrase $iPhrase in + # column $iCol. + # + # nDoc($iPhrase,$iCol) - Number of documents with at least one hit for + # phrase $iPhrase in column $iCol. + # + for {set iPhrase 0} {$iPhrase < $nPhrase} {incr iPhrase} { + for {set iCol 0} {$iCol < $nCol} {incr iCol} { + set nHit($iPhrase,$iCol) 0 + set nDoc($iPhrase,$iCol) 0 + } + } + foreach key [array names counts] { + set iCol [lindex [split $key ,] 1] + set iPhrase 0 + foreach c $counts($key) { + if {$c>0} { incr nDoc($iPhrase,$iCol) 1 } + incr nHit($iPhrase,$iCol) $c + incr iPhrase + } + } + + if {[llength $deferred] && [llength $expr]==1} { + set phrase [lindex $expr 0] + set rewritten [list] + set partial 0 + foreach tok $phrase { + if {[lsearch $deferred $tok]>=0} { + lappend rewritten * + } else { + lappend rewritten $tok + set partial 1 + } + } + if {$partial==0} { + set tblsize [db one "SELECT count(*) FROM $tbl"] + for {set iCol 0} {$iCol < $nCol} {incr iCol} { + set nHit(0,$iCol) $tblsize + set nDoc(0,$iCol) $tblsize + } + } elseif {$rewritten != $phrase} { + while {[lindex $rewritten end] == "*"} { + set rewritten [lrange $rewritten 0 end-1] + } + while {[lindex $rewritten 0] == "*"} { + set rewritten [lrange $rewritten 1 end] + } + get_single_near_results $tbl [list $rewritten] {} aRewrite nullentry + foreach docid [array names hits] { + set aMatchinfo($docid) $aRewrite($docid) + } + return + } + } + + # Set up the aMatchinfo array. For each document, set aMatchinfo($docid) to + # contain the output of matchinfo('x') for the document. + # + foreach docid [array names hits] { + set mi [list] + for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} { + for {set iCol 0} {$iCol<$nCol} {incr iCol} { + lappend mi [lindex $counts($docid,$iCol) $iPhrase] + lappend mi $nHit($iPhrase,$iCol) + lappend mi $nDoc($iPhrase,$iCol) + } + } + set aMatchinfo($docid) $mi + } + + # Set up the nullentry output. + # + set nullentry [list] + for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} { + for {set iCol 0} {$iCol<$nCol} {incr iCol} { + lappend nullentry 0 $nHit($iPhrase,$iCol) $nDoc($iPhrase,$iCol) + } + } +} + + +proc matching_brackets {expr} { + if {[string range $expr 0 0]!="(" || [string range $expr end end] !=")"} { + return 0 + } + + set iBracket 1 + set nExpr [string length $expr] + for {set i 1} {$iBracket && $i < $nExpr} {incr i} { + set c [string range $expr $i $i] + if {$c == "("} {incr iBracket} + if {$c == ")"} {incr iBracket -1} + } + + return [expr ($iBracket==0 && $i==$nExpr)] +} + +proc get_near_results {tbl expr deferred arrayvar {nullvar ""}} { + upvar $arrayvar aMatchinfo + if {$nullvar != ""} { upvar $nullvar nullentry } + + set expr [string trim $expr] + while { [matching_brackets $expr] } { + set expr [string trim [string range $expr 1 end-1]] + } + + set prec(NOT) 1 + set prec(AND) 2 + set prec(OR) 3 + + set currentprec 0 + set iBracket 0 + set expr_length [llength $expr] + for {set i 0} {$i < $expr_length} {incr i} { + set op [lindex $expr $i] + if {$iBracket==0 && [info exists prec($op)] && $prec($op)>=$currentprec } { + set opidx $i + set currentprec $prec($op) + } else { + for {set j 0} {$j < [string length $op]} {incr j} { + set c [string range $op $j $j] + if {$c == "("} { incr iBracket +1 } + if {$c == ")"} { incr iBracket -1 } + } + } + } + if {$iBracket!=0} { error "mismatched brackets in: $expr" } + + if {[info exists opidx]==0} { + get_single_near_results $tbl $expr $deferred aMatchinfo nullentry + } else { + set eLeft [lrange $expr 0 [expr $opidx-1]] + set eRight [lrange $expr [expr $opidx+1] end] + + get_near_results $tbl $eLeft $deferred aLeft nullleft + get_near_results $tbl $eRight $deferred aRight nullright + + switch -- [lindex $expr $opidx] { + "NOT" { + foreach hit [array names aLeft] { + if {0==[info exists aRight($hit)]} { + set aMatchinfo($hit) $aLeft($hit) + } + } + set nullentry $nullleft + } + + "AND" { + foreach hit [array names aLeft] { + if {[info exists aRight($hit)]} { + set aMatchinfo($hit) [concat $aLeft($hit) $aRight($hit)] + } + } + set nullentry [concat $nullleft $nullright] + } + + "OR" { + foreach hit [array names aLeft] { + if {[info exists aRight($hit)]} { + set aMatchinfo($hit) [concat $aLeft($hit) $aRight($hit)] + unset aRight($hit) + } else { + set aMatchinfo($hit) [concat $aLeft($hit) $nullright] + } + } + foreach hit [array names aRight] { + set aMatchinfo($hit) [concat $nullleft $aRight($hit)] + } + + set nullentry [concat $nullleft $nullright] + } + } + } +} + + +# End of test procs. Actual tests are below this line. +#-------------------------------------------------------------------------- + +#-------------------------------------------------------------------------- +# The following test cases - fts3auto-1.* - focus on testing the Tcl +# command [fts3_near_match], which is used by other tests in this file. +# +proc test_fts3_near_match {tn doc expr res} { + fts3_near_match $doc $expr -phrasecountvar p + uplevel do_test [list $tn] [list [list set {} $p]] [list $res] +} + +test_fts3_near_match 1.1.1 {a b c a b} a {2} +test_fts3_near_match 1.1.2 {a b c a b} {a 5 b 6 c} {2 2 1} +test_fts3_near_match 1.1.3 {a b c a b} {"a b"} {2} +test_fts3_near_match 1.1.4 {a b c a b} {"b c"} {1} +test_fts3_near_match 1.1.5 {a b c a b} {"c c"} {0} + +test_fts3_near_match 1.2.1 "a b c d e f g" {b 2 f} {0 0} +test_fts3_near_match 1.2.2 "a b c d e f g" {b 3 f} {1 1} +test_fts3_near_match 1.2.3 "a b c d e f g" {f 2 b} {0 0} +test_fts3_near_match 1.2.4 "a b c d e f g" {f 3 b} {1 1} +test_fts3_near_match 1.2.5 "a b c d e f g" {"a b" 2 "f g"} {0 0} +test_fts3_near_match 1.2.6 "a b c d e f g" {"a b" 3 "f g"} {1 1} + +set A "a b c d e f g h i j k l m n o p q r s t u v w x y z" +test_fts3_near_match 1.3.1 $A {"c d" 5 "i j" 1 "e f"} {0 0 0} +test_fts3_near_match 1.3.2 $A {"c d" 5 "i j" 2 "e f"} {1 1 1} + +#-------------------------------------------------------------------------- +# Test cases fts3auto-2.* run some simple tests using the +# [do_fts3query_test] proc. +# +foreach {tn create} { + 1 "fts4(a, b)" + 2 "fts4(a, b, order=DESC)" + 3 "fts4(a, b, order=ASC)" + 4 "fts4(a, b, prefix=1)" + 5 "fts4(a, b, order=DESC, prefix=1)" + 6 "fts4(a, b, order=ASC, prefix=1)" +} { + do_test 2.$tn.1 { + catchsql { DROP TABLE t1 } + execsql "CREATE VIRTUAL TABLE t1 USING $create" + for {set i 0} {$i<32} {incr i} { + set doc [list] + if {$i&0x01} {lappend doc one} + if {$i&0x02} {lappend doc two} + if {$i&0x04} {lappend doc three} + if {$i&0x08} {lappend doc four} + if {$i&0x10} {lappend doc five} + execsql { INSERT INTO t1 VALUES($doc, null) } + } + } {} + + foreach {tn2 expr} { + 1 {one} + 2 {one NEAR/1 five} + 3 {t*} + 4 {t* NEAR/0 five} + 5 {o* NEAR/1 f*} + 6 {one NEAR five NEAR two NEAR four NEAR three} + 7 {one NEAR xyz} + 8 {one OR two} + 9 {one AND two} + 10 {one NOT two} + 11 {one AND two OR three} + 12 {three OR one AND two} + 13 {(three OR one) AND two} + 14 {(three OR one) AND two NOT (five NOT four)} + 15 {"one two"} + 16 {"one two" NOT "three four"} + } { + do_fts3query_test 2.$tn.2.$tn2 t1 $expr + } +} + +#-------------------------------------------------------------------------- +# Some test cases involving deferred tokens. +# + +foreach {tn create} { + 1 "fts4(x)" + 2 "fts4(x, order=DESC)" +} { + catchsql { DROP TABLE t1 } + execsql "CREATE VIRTUAL TABLE t1 USING $create" + do_execsql_test 3.$tn.1 { + INSERT INTO t1(docid, x) VALUES(-2, 'a b c d e f g h i j k'); + INSERT INTO t1(docid, x) VALUES(-1, 'b c d e f g h i j k a'); + INSERT INTO t1(docid, x) VALUES(0, 'c d e f g h i j k a b'); + INSERT INTO t1(docid, x) VALUES(1, 'd e f g h i j k a b c'); + INSERT INTO t1(docid, x) VALUES(2, 'e f g h i j k a b c d'); + INSERT INTO t1(docid, x) VALUES(3, 'f g h i j k a b c d e'); + INSERT INTO t1(docid, x) VALUES(4, 'a c e g i k'); + INSERT INTO t1(docid, x) VALUES(5, 'a d g j'); + INSERT INTO t1(docid, x) VALUES(6, 'c a b'); + } + + set limit [fts3_make_deferrable t1 c] + + do_fts3query_test 3.$tn.2.1 t1 {a OR c} + + do_test 3.$tn.3 { + fts3_zero_long_segments t1 $limit + } {1} + + foreach {tn2 expr def} { + 1 {a NEAR c} {} + 2 {a AND c} c + 3 {"a c"} c + 4 {"c a"} c + 5 {"a c" NEAR/1 g} {} + 6 {"a c" NEAR/0 g} {} + } { + do_fts3query_test 3.$tn.4.$tn2 -deferred $def t1 $expr + } +} + +#-------------------------------------------------------------------------- +# +foreach {tn create} { + 1 "fts4(x, y)" + 2 "fts4(x, y, order=DESC)" + 3 "fts4(x, y, order=DESC, prefix=2)" +} { + + execsql [subst { + DROP TABLE t1; + CREATE VIRTUAL TABLE t1 USING $create; + INSERT INTO t1 VALUES('one two five four five', ''); + INSERT INTO t1 VALUES('', 'one two five four five'); + INSERT INTO t1 VALUES('one two', 'five four five'); + }] + + do_fts3query_test 4.$tn.1.1 t1 {one AND five} + do_fts3query_test 4.$tn.1.2 t1 {one NEAR five} + do_fts3query_test 4.$tn.1.3 t1 {one NEAR/1 five} + do_fts3query_test 4.$tn.1.4 t1 {one NEAR/2 five} + do_fts3query_test 4.$tn.1.5 t1 {one NEAR/3 five} + + do_test 4.$tn.2 { + set limit [fts3_make_deferrable t1 five] + execsql { INSERT INTO t1(t1) VALUES('optimize') } + expr {[fts3_zero_long_segments t1 $limit]>0} + } {1} + + do_fts3query_test 4.$tn.3.1 -deferred five t1 {one AND five} + do_fts3query_test 4.$tn.3.2 -deferred five t1 {one NEAR five} + do_fts3query_test 4.$tn.3.3 -deferred five t1 {one NEAR/1 five} + do_fts3query_test 4.$tn.3.4 -deferred five t1 {one NEAR/2 five} + + do_fts3query_test 4.$tn.3.5 -deferred five t1 {one NEAR/3 five} + + do_fts3query_test 4.$tn.4.1 -deferred fi* t1 {on* AND fi*} + do_fts3query_test 4.$tn.4.2 -deferred fi* t1 {on* NEAR fi*} + do_fts3query_test 4.$tn.4.3 -deferred fi* t1 {on* NEAR/1 fi*} + do_fts3query_test 4.$tn.4.4 -deferred fi* t1 {on* NEAR/2 fi*} + do_fts3query_test 4.$tn.4.5 -deferred fi* t1 {on* NEAR/3 fi*} +} + +#-------------------------------------------------------------------------- +# The following test cases - fts3auto-5.* - focus on using prefix indexes. +# +set chunkconfig [fts3_configure_incr_load 1 1] +foreach {tn create pending} { + 2 "fts4(a, b, order=ASC, prefix=1)" 1 + + 1 "fts4(a, b)" 1 + 3 "fts4(a, b, order=ASC, prefix=1,3)" 0 + 4 "fts4(a, b, order=DESC, prefix=2,4)" 0 + 5 "fts4(a, b, order=DESC, prefix=1)" 0 + 6 "fts4(a, b, order=ASC, prefix=1,3)" 0 +} { + + execsql [subst { + DROP TABLE IF EXISTS t1; + CREATE VIRTUAL TABLE t1 USING $create; + }] + + if {$pending} {execsql BEGIN} + + foreach {a b} { + "the song of songs which is solomons" + "let him kiss me with the kisses of his mouth for thy love is better than wine" + "because of the savour of thy good ointments thy name is as ointment poured forth therefore do the virgins love thee" + "draw me we will run after thee the king hath brought me into his chambers we will be glad and rejoice in thee we will remember thy love more than wine the upright love thee" + "i am black but comely o ye daughters of jerusalem as the tents of kedar as the curtains of solomon" + "look not upon me because i am black because the sun hath looked upon me my mothers children were angry with me they made me the keeper of the vineyards but mine own vineyard have i not kept" + "tell me o thou whom my soul loveth where thou feedest where thou makest thy flock to rest at noon for why should i be as one that turneth aside by the flocks of thy companions?" + "if thou know not o thou fairest among women go thy way forth by the footsteps of the flock and feed thy kids beside the shepherds tents" + "i have compared thee o my love to a company of horses in pharaohs chariots" + "thy cheeks are comely with rows of jewels thy neck with chains of gold" + "we will make thee borders of gold with studs of silver" + "while the king sitteth at his table my spikenard sendeth forth the smell thereof" + "a bundle of myrrh is my wellbeloved unto me he shall lie all night betwixt my breasts" + "my beloved is unto me as a cluster of camphire in the vineyards of en gedi" + "behold thou art fair my love behold thou art fair thou hast doves eyes" + "behold thou art fair my beloved yea pleasant also our bed is green" + "the beams of our house are cedar and our rafters of fir" + } { + execsql {INSERT INTO t1(a, b) VALUES($a, $b)} + } + + + do_fts3query_test 5.$tn.1.1 t1 {s*} + do_fts3query_test 5.$tn.1.2 t1 {so*} + do_fts3query_test 5.$tn.1.3 t1 {"s* o*"} + do_fts3query_test 5.$tn.1.4 t1 {b* NEAR/3 a*} + do_fts3query_test 5.$tn.1.5 t1 {a*} + do_fts3query_test 5.$tn.1.6 t1 {th* NEAR/5 a* NEAR/5 w*} + do_fts3query_test 5.$tn.1.7 t1 {"b* th* art* fair*"} + + if {$pending} {execsql COMMIT} +} +eval fts3_configure_incr_load $chunkconfig + +set sqlite_fts3_enable_parentheses $sfep +finish_test + diff --git a/test/fts3defer.test b/test/fts3defer.test index 1c9056fd9d..4bc0b0a7c3 100644 --- a/test/fts3defer.test +++ b/test/fts3defer.test @@ -20,6 +20,8 @@ ifcapable !fts3 { set sqlite_fts3_enable_parentheses 1 +set fts3_simple_deferred_tokens_only 1 + set ::testprefix fts3defer #-------------------------------------------------------------------------- @@ -257,7 +259,6 @@ foreach {tn setup} { do_select_test 1.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'jk eh' } {100} -if {$tn==3} breakpoint do_select_test 1.3 { SELECT rowid FROM t1 WHERE t1 MATCH 'jk ubwrfqnbjf' } {7 70 98} @@ -282,13 +283,16 @@ if {$tn==3} breakpoint do_select_test 1.10 { SELECT rowid FROM t1 WHERE t1 MATCH 'z* vgsld' } {10 13 17 31 35 51 58 88 89 90 93 100} - do_select_test 1.11 { - SELECT rowid FROM t1 - WHERE t1 MATCH '( - zdu OR zexh OR zf OR zhbrzadb OR zidhxhbtv OR - zk OR zkhdvkw OR zm OR zsmhnf - ) vgsld' - } {10 13 17 31 35 51 58 88 89 90 93 100} + + if { $fts3_simple_deferred_tokens_only==0 } { + do_select_test 1.11 { + SELECT rowid FROM t1 + WHERE t1 MATCH '( + zdu OR zexh OR zf OR zhbrzadb OR zidhxhbtv OR + zk OR zkhdvkw OR zm OR zsmhnf + ) vgsld' + } {10 13 17 31 35 51 58 88 89 90 93 100} + } do_select_test 2.1 { SELECT rowid FROM t1 WHERE t1 MATCH '"zm agmckuiu"' @@ -364,6 +368,7 @@ if {$tn==3} breakpoint foreach DO_MALLOC_TEST $dmt_modes { # Phrase search. + # do_select_test 5.$DO_MALLOC_TEST.1 { SELECT rowid FROM t1 WHERE t1 MATCH '"jk mjpavjuhw"' } {8 15 36 64 67 72} @@ -416,9 +421,11 @@ if {$tn==3} breakpoint do_select_test 6.2.2 { SELECT rowid FROM t1 WHERE t1 MATCH '"zm azavwm"' } {15 26 92 96} - do_select_test 6.2.3 { - SELECT rowid FROM t1 WHERE t1 MATCH '"jk xduvfhk" OR "zm azavwm"' - } {8 15 26 92 96} + if {$fts3_simple_deferred_tokens_only==0} { + do_select_test 6.2.3 { + SELECT rowid FROM t1 WHERE t1 MATCH '"jk xduvfhk" OR "zm azavwm"' + } {8 15 26 92 96} + } } set testprefix fts3defer diff --git a/test/fts3defer2.test b/test/fts3defer2.test index 8441803300..92a4491efd 100644 --- a/test/fts3defer2.test +++ b/test/fts3defer2.test @@ -48,6 +48,10 @@ do_execsql_test 1.1.4 { UPDATE t1_segments SET block = zeroblob(length(block)) WHERE length(block)>10000; } {2} +do_execsql_test 1.2.0 { + SELECT content FROM t1 WHERE t1 MATCH 'f (e a)'; +} {{a b c d e f a x y}} + do_execsql_test 1.2.1 { SELECT content FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)'; } {{a b c d e f a x y}} @@ -58,7 +62,7 @@ do_execsql_test 1.2.2 { } [list \ {a b c d [e] [f] [a] x y} \ {0 1 8 1 0 0 10 1 0 2 12 1} \ - [list 3 1 1 1 1 1 8 8 1 8 8 8 5001 9] + [list 3 1 1 1 1 1 1 1 1 1 1 8 5001 9] ] do_execsql_test 1.2.3 { @@ -67,7 +71,7 @@ do_execsql_test 1.2.3 { } [list \ {[a] b c d [e] [f] [a] x y} \ {0 2 0 1 0 1 8 1 0 0 10 1 0 2 12 1} \ - [list 3 1 1 1 1 1 8 8 2 8 8 8 5001 9] + [list 3 1 1 1 1 1 1 1 2 2 1 8 5001 9] ] do_execsql_test 1.3.1 { DROP TABLE t1 } @@ -99,8 +103,14 @@ foreach {tn sql} { [list 2 1 1 54 54 1 3 3 54 372 7] \ ] - set sqlite_fts3_enable_parentheses 1 do_execsql_test 2.2.$tn.2 { + SELECT mit(matchinfo(t2, 'x')) FROM t2 WHERE t2 MATCH 'g z'; + } [list \ + [list 1 2 2 1 54 54] \ + ] + + set sqlite_fts3_enable_parentheses 1 + do_execsql_test 2.2.$tn.3 { SELECT mit(matchinfo(t2, 'x')) FROM t2 WHERE t2 MATCH 'g OR (g z)'; } [list \ [list 1 2 2 1 2 2 1 54 54] \ diff --git a/test/fts3matchinfo.test b/test/fts3matchinfo.test index 8f194e72cb..40366b6aef 100644 --- a/test/fts3matchinfo.test +++ b/test/fts3matchinfo.test @@ -68,7 +68,11 @@ do_execsql_test 3.1 { do_execsql_test 3.2 { CREATE VIRTUAL TABLE xx USING FTS4; +} +do_execsql_test 3.3 { SELECT * FROM xx WHERE xx MATCH 'abc'; +} +do_execsql_test 3.4 { SELECT * FROM xx WHERE xx MATCH 'a b c'; } @@ -240,9 +244,13 @@ do_matchinfo_test 4.2.6 t5 {t5 MATCH 'a OR b'} { s {1 2 1} } do_execsql_test 4.3.0 "INSERT INTO t5 VALUES('x y [string repeat {b } 50000]')"; -do_matchinfo_test 4.3.1 t5 {t5 MATCH 'a a'} { - x {{5 8 2 5 5 5} {3 8 2 3 5 5}} - s {2 1} +# It used to be that the second 'a' token would be deferred. That doesn't +# work any longer. +if 0 { + do_matchinfo_test 4.3.1 t5 {t5 MATCH 'a a'} { + x {{5 8 2 5 5 5} {3 8 2 3 5 5}} + s {2 1} + } } do_matchinfo_test 4.3.2 t5 {t5 MATCH 'a b'} { s {2} } diff --git a/test/fts3prefix.test b/test/fts3prefix.test new file mode 100644 index 0000000000..f5e31f3208 --- /dev/null +++ b/test/fts3prefix.test @@ -0,0 +1,203 @@ +# 2011 May 04 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS3 module. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix fts3prefix + +ifcapable !fts3 { + finish_test + return +} + +# This proc tests that the prefixes index appears to represent the same content +# as the terms index. +# +proc fts3_terms_and_prefixes {db tbl prefixlengths} { + + set iIndex 0 + foreach len $prefixlengths { + incr iIndex + $db eval { + DROP TABLE IF EXISTS fts3check1; + DROP TABLE IF EXISTS fts3check2; + } + $db eval "CREATE VIRTUAL TABLE fts3check1 USING fts4term($tbl, 0);" + $db eval "CREATE VIRTUAL TABLE fts3check2 USING fts4term($tbl, $iIndex);" + + $db eval { + DROP TABLE IF EXISTS temp.terms; + DROP TABLE IF EXISTS temp.prefixes; + CREATE TEMP TABLE terms AS SELECT * FROM fts3check1; + CREATE TEMP TABLE prefixes AS SELECT * FROM fts3check2; + CREATE INDEX temp.idx ON prefixes(term); + DROP TABLE fts3check1; + DROP TABLE fts3check2; + } + + set nExpect 0 + $db eval { SELECT term, docid, col, pos FROM temp.terms } a { + if {[string length $a(term)]<$len} continue + incr nExpect + set prefix [string range $a(term) 0 [expr $len-1]] + set r [$db one { + SELECT count(*) FROM temp.prefixes WHERE + term = $prefix AND docid = $a(docid) AND col = $a(col) AND pos = $a(pos) + }] + if {$r != 1} { + error "$t, $a(docid), $a(col), $a(pos)" + } + } + + set nCount [$db one {SELECT count(*) FROM temp.prefixes}] + if {$nCount != $nExpect} { + error "prefixes.count(*) is $nCount expected $nExpect" + } + + execsql { DROP TABLE temp.prefixes } + execsql { DROP TABLE temp.terms } + + set list [list] + $db eval " + SELECT sum( 1 << (16*(level%1024)) ) AS total, (level/1024) AS tree + FROM ${tbl}_segdir GROUP BY tree + " { + lappend list [list $total $tree] + } + + if { [lsort -integer -index 0 $list] != [lsort -integer -index 1 $list] } { + error "inconsistent tree structures: $list" + } + } + + return "" +} +proc fts3_tap_test {tn db tbl lens} { + uplevel [list do_test $tn [list fts3_terms_and_prefixes $db $tbl $lens] ""] +} + +#------------------------------------------------------------------------- +# Test cases 1.* are a sanity check. They test that the prefixes index is +# being constructed correctly for the simplest possible case. +# +do_execsql_test 1.1 { + CREATE VIRTUAL TABLE t1 USING fts4(prefix='1,3,6'); + + CREATE VIRTUAL TABLE p1 USING fts4term(t1, 1); + CREATE VIRTUAL TABLE p2 USING fts4term(t1, 2); + CREATE VIRTUAL TABLE p3 USING fts4term(t1, 3); + CREATE VIRTUAL TABLE terms USING fts4term(t1); +} +do_execsql_test 1.2 { + INSERT INTO t1 VALUES('sqlite mysql firebird'); +} +do_execsql_test 1.3.1 { SELECT term FROM p1 } {f m s} +do_execsql_test 1.3.2 { SELECT term FROM p2 } {fir mys sql} +do_execsql_test 1.3.3 { SELECT term FROM p3 } {firebi sqlite} +do_execsql_test 1.4 { + SELECT term FROM terms; +} {firebird mysql sqlite} + +fts3_tap_test 1.5 db t1 {1 3 6} + +#------------------------------------------------------------------------- +# A slightly more complicated dataset. This test also verifies that DELETE +# operations do not corrupt the prefixes index. +# +do_execsql_test 2.1 { + INSERT INTO t1 VALUES('FTS3 and FTS4 are an SQLite virtual table modules'); + INSERT INTO t1 VALUES('that allows users to perform full-text searches on'); + INSERT INTO t1 VALUES('a set of documents. The most common (and'); + INSERT INTO t1 VALUES('effective) way to describe full-text searches is'); + INSERT INTO t1 VALUES('"what Google, Yahoo and Altavista do with'); + INSERT INTO t1 VALUES('documents placed on the World Wide Web". Users'); + INSERT INTO t1 VALUES('input a term, or series of terms, perhaps'); + INSERT INTO t1 VALUES('connected by a binary operator or grouped together'); + INSERT INTO t1 VALUES('into a phrase, and the full-text query system'); + INSERT INTO t1 VALUES('finds the set of documents that best matches those'); + INSERT INTO t1 VALUES('terms considering the operators and groupings the'); + INSERT INTO t1 VALUES('user has specified. This article describes the'); + INSERT INTO t1 VALUES('deployment and usage of FTS3 and FTS4.'); + INSERT INTO t1 VALUES('FTS1 and FTS2 are obsolete full-text search'); + INSERT INTO t1 VALUES('modules for SQLite. There are known issues with'); + INSERT INTO t1 VALUES('these older modules and their use should be'); + INSERT INTO t1 VALUES('avoided. Portions of the original FTS3 code were'); + INSERT INTO t1 VALUES('contributed to the SQLite project by Scott Hess of'); + INSERT INTO t1 VALUES('Google. It is now developed and maintained as part'); + INSERT INTO t1 VALUES('of SQLite. '); +} +fts3_tap_test 2.2 db t1 {1 3 6} +do_execsql_test 2.3 { DELETE FROM t1 WHERE docid%2; } +fts3_tap_test 2.4 db t1 {1 3 6} + +do_execsql_test 2.5 { INSERT INTO t1(t1) VALUES('optimize') } +fts3_tap_test 2.6 db t1 {1 3 6} + +do_execsql_test 3.1 { + CREATE VIRTUAL TABLE t2 USING fts4(prefix='1,2,3'); + INSERT INTO t2 VALUES('On 12 September the wind direction turned and'); + INSERT INTO t2 VALUES('William''s fleet sailed. A storm blew up and the'); + INSERT INTO t2 VALUES('fleet was forced to take shelter at'); + INSERT INTO t2 VALUES('Saint-Valery-sur-Somme and again wait for the wind'); + INSERT INTO t2 VALUES('to change. On 27 September the Norman fleet'); + INSERT INTO t2 VALUES('finally set sail, landing in England at Pevensey'); + INSERT INTO t2 VALUES('Bay (Sussex) on 28 September. William then moved'); + INSERT INTO t2 VALUES('to Hastings, a few miles to the east, where he'); + INSERT INTO t2 VALUES('built a prefabricated wooden castle for a base of'); + INSERT INTO t2 VALUES('operations. From there, he ravaged the hinterland'); + INSERT INTO t2 VALUES('and waited for Harold''s return from the north.'); + INSERT INTO t2 VALUES('On 12 September the wind direction turned and'); + INSERT INTO t2 VALUES('William''s fleet sailed. A storm blew up and the'); + INSERT INTO t2 VALUES('fleet was forced to take shelter at'); + INSERT INTO t2 VALUES('Saint-Valery-sur-Somme and again wait for the wind'); + INSERT INTO t2 VALUES('to change. On 27 September the Norman fleet'); + INSERT INTO t2 VALUES('finally set sail, landing in England at Pevensey'); + INSERT INTO t2 VALUES('Bay (Sussex) on 28 September. William then moved'); + INSERT INTO t2 VALUES('to Hastings, a few miles to the east, where he'); + INSERT INTO t2 VALUES('built a prefabricated wooden castle for a base of'); + INSERT INTO t2 VALUES('operations. From there, he ravaged the hinterland'); + INSERT INTO t2 VALUES('and waited for Harold''s return from the north.'); +} + +fts3_tap_test 3.2 db t2 {1 2 3} +do_execsql_test 3.3 { SELECT optimize(t2) FROM t2 LIMIT 1 } {{Index optimized}} +fts3_tap_test 3.4 db t2 {1 2 3} + + +#------------------------------------------------------------------------- +# Simple tests for reading the prefix-index. +# +do_execsql_test 4.1 { + CREATE VIRTUAL TABLE t3 USING fts4(prefix="1,4"); + INSERT INTO t3 VALUES('one two three'); + INSERT INTO t3 VALUES('four five six'); + INSERT INTO t3 VALUES('seven eight nine'); +} +do_execsql_test 4.2 { + SELECT * FROM t3 WHERE t3 MATCH 'f*' +} {{four five six}} +do_execsql_test 4.3 { + SELECT * FROM t3 WHERE t3 MATCH 'four*' +} {{four five six}} +do_execsql_test 4.4 { + SELECT * FROM t3 WHERE t3 MATCH 's*' +} {{four five six} {seven eight nine}} +do_execsql_test 4.5 { + SELECT * FROM t3 WHERE t3 MATCH 'sev*' +} {{seven eight nine}} +do_execsql_test 4.6 { + SELECT * FROM t3 WHERE t3 MATCH 'one*' +} {{one two three}} + +finish_test diff --git a/test/fts3rnd.test b/test/fts3rnd.test index 0909cee614..97af54925f 100644 --- a/test/fts3rnd.test +++ b/test/fts3rnd.test @@ -162,7 +162,7 @@ proc simple_phrase {zPrefix} { # This [proc] is used to test the FTS3 matchinfo() function. # -proc simple_token_matchinfo {zToken} { +proc simple_token_matchinfo {zToken bDesc} { set nDoc(0) 0 set nDoc(1) 0 @@ -171,6 +171,8 @@ proc simple_token_matchinfo {zToken} { set nHit(1) 0 set nHit(2) 0 + set dir -inc + if {$bDesc} { set dir -dec } foreach key [array names ::t1] { set value $::t1($key) @@ -184,7 +186,7 @@ proc simple_token_matchinfo {zToken} { } set ret [list] - foreach docid [lsort -integer [array names a]] { + foreach docid [lsort -integer $dir [array names a]] { if { [lindex [lsort -integer $a($docid)] end] } { set matchinfo [list 1 3] foreach i {0 1 2} hit $a($docid) { @@ -262,22 +264,37 @@ proc mit {blob} { return $r } db func mit mit - set sqlite_fts3_enable_parentheses 1 -foreach nodesize {50 500 1000 2000} { +proc do_orderbydocid_test {tn sql res} { + uplevel [list do_select_test $tn.asc "$sql ORDER BY docid ASC" $res] + uplevel [list do_select_test $tn.desc "$sql ORDER BY docid DESC" \ + [lsort -int -dec $res] + ] +} + +set NUM_TRIALS 100 + +foreach {nodesize order} { + 50 DESC + 50 ASC + 500 ASC + 1000 DESC + 2000 ASC +} { catch { array unset ::t1 } + set testname "$nodesize/$order" # Create the FTS3 table. Populate it (and the Tcl array) with 100 rows. # db transaction { catchsql { DROP TABLE t1 } - execsql "CREATE VIRTUAL TABLE t1 USING fts3(a, b, c)" + execsql "CREATE VIRTUAL TABLE t1 USING fts4(a, b, c, order=$order)" execsql "INSERT INTO t1(t1) VALUES('nodesize=$nodesize')" for {set i 0} {$i < 100} {incr i} { insert_row $i } } - for {set iTest 1} {$iTest <= 100} {incr iTest} { + for {set iTest 1} {$iTest <= $NUM_TRIALS} {incr iTest} { catchsql COMMIT set DO_MALLOC_TEST 0 @@ -286,6 +303,8 @@ foreach nodesize {50 500 1000 2000} { set DO_MALLOC_TEST 1 set nRep 2 } + + set ::testprefix fts3rnd-1.$testname.$iTest # Delete one row, update one row and insert one row. # @@ -307,7 +326,7 @@ foreach nodesize {50 500 1000 2000} { if {0==($iTest%2)} { execsql COMMIT } if {0==($iTest%2)} { - do_test fts3rnd-1.$nodesize.$iTest.0 { fts3_integrity_check t1 } ok + #do_test 0 { fts3_integrity_check t1 } ok } # Pick 10 terms from the vocabulary. Check that the results of querying @@ -317,9 +336,14 @@ foreach nodesize {50 500 1000 2000} { # for {set i 0} {$i < 10} {incr i} { set term [random_term] - do_select_test fts3rnd-1.$nodesize.$iTest.1.$i { + do_select_test 1.$i.asc { SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH $term - } [simple_token_matchinfo $term] + ORDER BY docid ASC + } [simple_token_matchinfo $term 0] + do_select_test 1.$i.desc { + SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH $term + ORDER BY docid DESC + } [simple_token_matchinfo $term 1] } # This time, use the first two characters of each term as a term prefix @@ -329,7 +353,7 @@ foreach nodesize {50 500 1000 2000} { for {set i 0} {$i < $nRep} {incr i} { set prefix [string range [random_term] 0 end-1] set match "${prefix}*" - do_select_test fts3rnd-1.$nodesize.$iTest.2.$i { + do_orderbydocid_test 2.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [simple_phrase $match] } @@ -339,7 +363,7 @@ foreach nodesize {50 500 1000 2000} { for {set i 0} {$i < $nRep} {incr i} { set term [list [random_term] [random_term]] set match "\"$term\"" - do_select_test fts3rnd-1.$nodesize.$iTest.3.$i { + do_orderbydocid_test 3.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [simple_phrase $term] } @@ -349,7 +373,7 @@ foreach nodesize {50 500 1000 2000} { for {set i 0} {$i < $nRep} {incr i} { set term [list [random_term] [random_term] [random_term]] set match "\"$term\"" - do_select_test fts3rnd-1.$nodesize.$iTest.4.$i { + do_orderbydocid_test 4.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [simple_phrase $term] } @@ -362,17 +386,19 @@ foreach nodesize {50 500 1000 2000} { append query "[string range [random_term] 0 end-1]*" set match "\"$query\"" - do_select_test fts3rnd-1.$nodesize.$iTest.5.$i { + do_orderbydocid_test 5.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [simple_phrase $query] } - # A NEAR query with terms as the arguments. + # A NEAR query with terms as the arguments: + # + # ... MATCH '$term1 NEAR $term2' ... # for {set i 0} {$i < $nRep} {incr i} { set terms [list [random_term] [random_term]] set match [join $terms " NEAR "] - do_select_test fts3rnd-1.$nodesize.$iTest.6.$i { + do_orderbydocid_test 6.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [simple_near $terms 10] } @@ -383,7 +409,7 @@ foreach nodesize {50 500 1000 2000} { set terms [list [random_term] [random_term] [random_term]] set nNear 11 set match [join $terms " NEAR/$nNear "] - do_select_test fts3rnd-1.$nodesize.$iTest.7.$i { + do_orderbydocid_test 7.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [simple_near $terms $nNear] } @@ -399,7 +425,7 @@ foreach nodesize {50 500 1000 2000} { set term1 [random_term] set term2 [random_term] set match "$term1 $op $term2" - do_select_test fts3rnd-1.$nodesize.$iTest.$tn.$i { + do_orderbydocid_test $tn.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [$proc [simple_phrase $term1] [simple_phrase $term2]] } @@ -408,9 +434,9 @@ foreach nodesize {50 500 1000 2000} { # Set operations on NEAR queries. # foreach {tn op proc} { - 8 OR setop_or - 9 NOT setop_not - 10 AND setop_and + 11 OR setop_or + 12 NOT setop_not + 13 AND setop_and } { for {set i 0} {$i < $nRep} {incr i} { set term1 [random_term] @@ -418,7 +444,7 @@ foreach nodesize {50 500 1000 2000} { set term3 [random_term] set term4 [random_term] set match "$term1 NEAR $term2 $op $term3 NEAR $term4" - do_select_test fts3rnd-1.$nodesize.$iTest.$tn.$i { + do_orderbydocid_test $tn.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [$proc \ [simple_near [list $term1 $term2] 10] \ diff --git a/test/fts3sort.test b/test/fts3sort.test index 001bef1135..ce5b5df76b 100644 --- a/test/fts3sort.test +++ b/test/fts3sort.test @@ -21,9 +21,8 @@ ifcapable !fts3 { return } -set testprefix fts3sort -proc build_database {nRow} { +proc build_database {nRow param} { db close forcedelete test.db sqlite3 db test.db @@ -31,7 +30,7 @@ proc build_database {nRow} { set vocab [list aa ab ac ba bb bc ca cb cc da] expr srand(0) - execsql { CREATE VIRTUAL TABLE t1 USING fts4 } + execsql "CREATE VIRTUAL TABLE t1 USING fts4($param)" for {set i 0} {$i < $nRow} {incr i} { set v [expr int(rand()*1000000)] set doc [list] @@ -42,13 +41,24 @@ proc build_database {nRow} { } } -set nRow 1000 -do_test 1.0 { - build_database $nRow - execsql { SELECT count(*) FROM t1 } -} $nRow +set testprefix fts3sort -foreach {tn query} { +unset -nocomplain CONTROL +foreach {t param} { + 1 "" + 2 "order=asc" + 3 "order=desc" +} { + + set testprefix fts3sort-1.$t + + set nRow 1000 + do_test 1.0 { + build_database $nRow $param + execsql { SELECT count(*) FROM t1 } + } $nRow + + foreach {tn query} { 1 "SELECT docid, * FROM t1" 2 "SELECT docid, * FROM t1 WHERE t1 MATCH 'aa'" 3 "SELECT docid, * FROM t1 WHERE t1 MATCH 'a*'" @@ -59,50 +69,94 @@ foreach {tn query} { 8 "SELECT docid, * FROM t1 WHERE t1 MATCH 'nosuchtoken'" 9 "SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH 'aa OR da'" 10 "SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH 'aa OR nosuchtoken'" + 11 "SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH 'aa NEAR bb'" + 12 "SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH '\"aa bb\"'" + 13 "SELECT docid, content FROM t1 WHERE t1 MATCH 'aa NEAR/2 bb NEAR/3 cc'" + 14 "SELECT docid, content FROM t1 WHERE t1 MATCH '\"aa bb cc\"'" + } { + + unset -nocomplain A B C D + set A_list [list] + set B_list [list] + set C_list [list] + set D_list [list] + + unset -nocomplain X + db eval "$query ORDER BY rowid ASC" X { + set A($X(docid)) [array get X] + lappend A_list $X(docid) + } + unset -nocomplain X + db eval "$query ORDER BY rowid DESC" X { + set B($X(docid)) [array get X] + lappend B_list $X(docid) + } + unset -nocomplain X + db eval "$query ORDER BY docid ASC" X { + set C($X(docid)) [array get X] + lappend C_list $X(docid) + } + unset -nocomplain X + db eval "$query ORDER BY docid DESC" X { + set D($X(docid)) [array get X] + lappend D_list $X(docid) + } + + do_test $tn.1 { set A_list } [lsort -integer -increasing $A_list] + do_test $tn.2 { set B_list } [lsort -integer -decreasing $B_list] + do_test $tn.3 { set C_list } [lsort -integer -increasing $C_list] + do_test $tn.4 { set D_list } [lsort -integer -decreasing $D_list] + + unset -nocomplain DATA + unset -nocomplain X + db eval "$query" X { + set DATA($X(docid)) [array get X] + } + + do_test $tn.5 { lsort [array get A] } [lsort [array get DATA]] + do_test $tn.6 { lsort [array get B] } [lsort [array get DATA]] + do_test $tn.7 { lsort [array get C] } [lsort [array get DATA]] + do_test $tn.8 { lsort [array get D] } [lsort [array get DATA]] + + if {[info exists CONTROL($tn)]} { + do_test $tn.9 { set CONTROL($tn) } [lsort [array get DATA]] + } else { + set CONTROL($tn) [lsort [array get DATA]] + } + } +} +unset -nocomplain CONTROL + +set testprefix fts3sort + +#------------------------------------------------------------------------- +# Tests for parsing the "order=asc" and "order=desc" directives. +# +foreach {tn param res} { + 1 "order=asc" {0 {}} + 2 "order=desc" {0 {}} + 3 "order=dec" {1 {unrecognized order: dec}} + 4 "order=xxx, order=asc" {1 {unrecognized order: xxx}} + 5 "order=desc, order=asc" {0 {}} } { - - unset -nocomplain A B C D - set A_list [list] - set B_list [list] - set C_list [list] - set D_list [list] - - unset -nocomplain X - db eval "$query ORDER BY rowid ASC" X { - set A($X(docid)) [array get X] - lappend A_list $X(docid) - } - unset -nocomplain X - db eval "$query ORDER BY rowid DESC" X { - set B($X(docid)) [array get X] - lappend B_list $X(docid) - } - unset -nocomplain X - db eval "$query ORDER BY docid ASC" X { - set C($X(docid)) [array get X] - lappend C_list $X(docid) - } - unset -nocomplain X - db eval "$query ORDER BY docid DESC" X { - set D($X(docid)) [array get X] - lappend D_list $X(docid) - } - - do_test 1.$tn.1 { set A_list } [lsort -integer -increasing $A_list] - do_test 1.$tn.2 { set B_list } [lsort -integer -decreasing $B_list] - do_test 1.$tn.3 { set C_list } [lsort -integer -increasing $C_list] - do_test 1.$tn.4 { set D_list } [lsort -integer -decreasing $D_list] - - unset -nocomplain DATA - unset -nocomplain X - db eval "$query" X { - set DATA($X(docid)) [array get X] - } - - do_test 1.$tn.5 { lsort [array get A] } [lsort [array get DATA]] - do_test 1.$tn.6 { lsort [array get B] } [lsort [array get DATA]] - do_test 1.$tn.7 { lsort [array get C] } [lsort [array get DATA]] - do_test 1.$tn.8 { lsort [array get D] } [lsort [array get DATA]] + execsql { DROP TABLE IF EXISTS t1 } + do_catchsql_test 2.1.$tn " + CREATE VIRTUAL TABLE t1 USING fts4(a, b, $param) + " $res } +do_execsql_test 2.2 { + BEGIN; + CREATE VIRTUAL TABLE t2 USING fts4(order=desc); + INSERT INTO t2 VALUES('aa bb'); + INSERT INTO t2 VALUES('bb cc'); + INSERT INTO t2 VALUES('cc aa'); + SELECT docid FROM t2 WHERE t2 MATCH 'aa'; + END; +} {3 1} +do_execsql_test 2.3 { + SELECT docid FROM t2 WHERE t2 MATCH 'aa'; +} {3 1} + finish_test + diff --git a/test/hook.test b/test/hook.test index 58a6fed8bf..90aa8a945c 100644 --- a/test/hook.test +++ b/test/hook.test @@ -275,6 +275,34 @@ ifcapable compound&&attach { set ::update_hook } [list] } + +do_test hook-4.4 { + execsql { + CREATE TABLE t4(a UNIQUE, b); + INSERT INTO t4 VALUES(1, 'a'); + INSERT INTO t4 VALUES(2, 'b'); + } + set ::update_hook [list] + execsql { + REPLACE INTO t4 VALUES(1, 'c'); + } + set ::update_hook +} [list INSERT main t4 3 ] +do_execsql_test hook-4.4.1 { + SELECT * FROM t4 ORDER BY a; +} {1 c 2 b} +do_test hook-4.4.2 { + set ::update_hook [list] + execsql { + PRAGMA recursive_triggers = on; + REPLACE INTO t4 VALUES(1, 'd'); + } + set ::update_hook +} [list INSERT main t4 4 ] +do_execsql_test hook-4.4.3 { + SELECT * FROM t4 ORDER BY a; +} {1 d 2 b} + db update_hook {} # #---------------------------------------------------------------------------- diff --git a/test/lock_common.tcl b/test/lock_common.tcl index 21f584264a..bc1eb86bdc 100644 --- a/test/lock_common.tcl +++ b/test/lock_common.tcl @@ -55,8 +55,8 @@ proc do_multiclient_test {varname script} { uplevel set $varname $tn uplevel $script - code2 { db2 close } - code3 { db3 close } + catch { code2 { db2 close } } + catch { code3 { db3 close } } catch { close $::code2_chan } catch { close $::code3_chan } catch { db close } diff --git a/test/malloc.test b/test/malloc.test index 678b2be075..9bd5314bf1 100644 --- a/test/malloc.test +++ b/test/malloc.test @@ -894,6 +894,18 @@ ifcapable stat2&&utf16 { } } +# Test that if an OOM error occurs, aux-data is still correctly destroyed. +# This test case was causing either a memory-leak or an assert() failure +# at one point, depending on the configuration. +# +do_malloc_test 39 -tclprep { + sqlite3 db test.db +} -sqlbody { + SELECT test_auxdata('abc', 'def'); +} -cleanup { + db close +} + # Ensure that no file descriptors were leaked. do_test malloc-99.X { catch {db close} diff --git a/test/multiplex.test b/test/multiplex.test index ae60d639e5..9278e84274 100644 --- a/test/multiplex.test +++ b/test/multiplex.test @@ -572,6 +572,7 @@ if {0==[info exists ::G(perm:presql)] || $::G(perm:presql) == ""} { ifcapable vacuum { +sqlite3_multiplex_shutdown do_test multiplex-6.0.0 { multiplex_delete test.db sqlite3_multiplex_initialize "" 1 diff --git a/test/pager1.test b/test/pager1.test index fbdd4b20b9..136ca05f46 100644 --- a/test/pager1.test +++ b/test/pager1.test @@ -441,6 +441,8 @@ do_test pager1.4.2.1 { db close tstvfs delete } {} + +if {$::tcl_platform(platform)!="windows"} { do_test pager1.4.2.2 { faultsim_restore_and_reopen execsql { @@ -473,6 +475,7 @@ do_test pager1.4.2.5 { PRAGMA integrity_check; } } {4 ok} +} do_test pager1.4.3.1 { testvfs tstvfs -default 1 @@ -1548,6 +1551,7 @@ do_execsql_test pager1-13.1.1 { UPDATE t1 SET b = a_string(400); } {persist} +if {$::tcl_platform(platform)!="windows"} { # Run transactions of increasing sizes. Eventually, one (or more than one) # of these will write just enough content that one of the old headers created # by the transaction in the block above lies immediately after the content @@ -1570,7 +1574,9 @@ for {set nUp 1} {$nUp<64} {incr nUp} { } {ok} db2 close } +} +if {$::tcl_platform(platform)!="windows"} { # Same test as above. But this time with an index on the table. # do_execsql_test pager1-13.2.1 { @@ -1591,6 +1597,7 @@ for {set nUp 1} {$nUp<64} {incr nUp} { } {ok} db2 close } +} db close tv delete diff --git a/test/pagerfault.test b/test/pagerfault.test index c5c7d3b33a..ced6da3b3c 100644 --- a/test/pagerfault.test +++ b/test/pagerfault.test @@ -20,6 +20,11 @@ if {[permutation] == "inmemory_journal"} { return } +if {$::tcl_platform(platform)=="windows"} { + finish_test + return +} + set a_string_counter 1 proc a_string {n} { global a_string_counter diff --git a/test/permutations.test b/test/permutations.test index 7e336b16aa..354b4084a8 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -182,10 +182,11 @@ test_suite "fts3" -prefix "" -description { fts3atoken.test fts3b.test fts3c.test fts3cov.test fts3d.test fts3defer.test fts3defer2.test fts3e.test fts3expr.test fts3expr2.test fts3near.test fts3query.test fts3shared.test fts3snippet.test + fts3sort.test fts3fault.test fts3malloc.test fts3matchinfo.test - fts3aux1.test fts3comp1.test + fts3aux1.test fts3comp1.test fts3auto.test } @@ -531,7 +532,7 @@ test_suite "inmemory_journal" -description { stmt.test # WAL mode is different. - wal* + wal* tkt-2d1a5c67d.test }] ifcapable mem3 { diff --git a/test/tester.tcl b/test/tester.tcl index be21334f1d..6e97e69ca8 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -380,11 +380,11 @@ proc normalize_list {L} { proc do_execsql_test {testname sql {result {}}} { fix_testname testname - uplevel do_test $testname [list "execsql {$sql}"] [list [list {*}$result]] + uplevel do_test [list $testname] [list "execsql {$sql}"] [list [list {*}$result]] } proc do_catchsql_test {testname sql result} { fix_testname testname - uplevel do_test $testname [list "catchsql {$sql}"] [list $result] + uplevel do_test [list $testname] [list "catchsql {$sql}"] [list $result] } proc do_eqp_test {name sql res} { uplevel do_execsql_test $name [list "EXPLAIN QUERY PLAN $sql"] [list $res] diff --git a/test/trace2.test b/test/trace2.test index a70dd89d1d..42738db3aa 100644 --- a/test/trace2.test +++ b/test/trace2.test @@ -141,10 +141,10 @@ ifcapable fts3 { INSERT INTO x1(x1) VALUES('optimize'); } { "INSERT INTO x1(x1) VALUES('optimize');" - "-- SELECT idx, start_block, leaves_end_block, end_block, root FROM 'main'.'x1_segdir' ORDER BY level DESC, idx ASC" - "-- SELECT count(*), max(level) FROM 'main'.'x1_segdir'" + "-- SELECT idx, start_block, leaves_end_block, end_block, root FROM 'main'.'x1_segdir' WHERE level BETWEEN ? AND ?ORDER BY level DESC, idx ASC" + "-- SELECT max(level) FROM 'main'.'x1_segdir' WHERE level BETWEEN ? AND ?" "-- SELECT coalesce((SELECT max(blockid) FROM 'main'.'x1_segments') + 1, 1)" - "-- DELETE FROM 'main'.'x1_segdir'" + "-- DELETE FROM 'main'.'x1_segdir' WHERE level BETWEEN ? AND ?" "-- INSERT INTO 'main'.'x1_segdir' VALUES(?,?,?,?,?,?)" } } diff --git a/test/wal7.test b/test/wal7.test index cfe2d7b8b0..cacfaedacb 100644 --- a/test/wal7.test +++ b/test/wal7.test @@ -61,7 +61,7 @@ do_test wal7-2.0 { INSERT INTO t1 VALUES(zeroblob(200000),4); CREATE TABLE t2(z); DELETE FROM t1; - INSERT INTO t2 SELECT x FROM t1; + INSERT INTO t2 VALUES(1); } file size test.db-wal } 25000 @@ -83,10 +83,10 @@ do_test wal7-3.0 { INSERT INTO t1 VALUES(zeroblob(200000),4); CREATE TABLE t2(z); DELETE FROM t1; - INSERT INTO t2 SELECT x FROM t1; + INSERT INTO t2 VALUES(1); } set sz [file size test.db-wal] - expr {$sz>0 && $sz<10000} + expr {$sz>0 && $sz<13700} } 1 @@ -106,7 +106,7 @@ do_test wal7-4.0 { INSERT INTO t1 VALUES(zeroblob(200000),4); CREATE TABLE t2(z); DELETE FROM t1; - INSERT INTO t2 SELECT x FROM t1; + INSERT INTO t2 VALUES(1); } set sz [file size test.db-wal] } 25000 diff --git a/test/walro.test b/test/walro.test new file mode 100644 index 0000000000..60bdce9e04 --- /dev/null +++ b/test/walro.test @@ -0,0 +1,161 @@ +# 2011 May 09 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for using WAL databases in read-only mode. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +set ::testprefix walro + +# These tests are only going to work on unix. +# +if {$::tcl_platform(platform) != "unix"} { + finish_test + return +} + +do_multiclient_test tn { + # Do not run tests with the connections in the same process. + # + if {$tn==2} continue + + # Close all connections and delete the database. + # + code1 { db close } + code2 { db2 close } + code3 { db3 close } + forcedelete test.db + forcedelete walro + + foreach c {code1 code2 code3} { + $c { + sqlite3_shutdown + sqlite3_config_uri 1 + } + } + + file mkdir walro + + do_test 1.1.1 { + code2 { sqlite3 db2 test.db } + sql2 { + PRAGMA journal_mode = WAL; + CREATE TABLE t1(x, y); + INSERT INTO t1 VALUES('a', 'b'); + } + file exists test.db-shm + } {1} + + do_test 1.1.2 { + file attributes test.db-shm -permissions r--r--r-- + code1 { sqlite3 db file:test.db?readonly_shm=1 } + } {} + + do_test 1.1.3 { sql1 "SELECT * FROM t1" } {a b} + do_test 1.1.4 { sql2 "INSERT INTO t1 VALUES('c', 'd')" } {} + do_test 1.1.5 { sql1 "SELECT * FROM t1" } {a b c d} + + # Check that the read-only connection cannot write or checkpoint the db. + # + do_test 1.1.6 { + csql1 "INSERT INTO t1 VALUES('e', 'f')" + } {1 {attempt to write a readonly database}} + do_test 1.1.7 { + csql1 "PRAGMA wal_checkpoint" + } {1 {attempt to write a readonly database}} + + do_test 1.1.9 { sql2 "INSERT INTO t1 VALUES('e', 'f')" } {} + do_test 1.1.10 { sql1 "SELECT * FROM t1" } {a b c d e f} + + do_test 1.1.11 { + sql2 { + INSERT INTO t1 VALUES('g', 'h'); + PRAGMA wal_checkpoint; + } + set {} {} + } {} + do_test 1.1.12 { sql1 "SELECT * FROM t1" } {a b c d e f g h} + do_test 1.1.13 { sql2 "INSERT INTO t1 VALUES('i', 'j')" } {} + + do_test 1.2.1 { + code2 { db2 close } + code1 { db close } + list [file exists test.db-wal] [file exists test.db-shm] + } {1 1} + do_test 1.2.2 { + code1 { sqlite3 db file:test.db?readonly_shm=1 } + sql1 { SELECT * FROM t1 } + } {a b c d e f g h i j} + + do_test 1.2.3 { + code1 { db close } + file attributes test.db-shm -permissions rw-r--r-- + hexio_write test.db-shm 0 01020304 + file attributes test.db-shm -permissions r--r--r-- + code1 { sqlite3 db file:test.db?readonly_shm=1 } + csql1 { SELECT * FROM t1 } + } {1 {attempt to write a readonly database}} + do_test 1.2.4 { + code1 { sqlite3_extended_errcode db } + } {SQLITE_READONLY_RECOVERY} + + do_test 1.2.5 { + file attributes test.db-shm -permissions rw-r--r-- + code2 { sqlite3 db2 test.db } + sql2 "SELECT * FROM t1" + } {a b c d e f g h i j} + file attributes test.db-shm -permissions r--r--r-- + do_test 1.2.6 { sql1 "SELECT * FROM t1" } {a b c d e f g h i j} + + do_test 1.2.7 { + sql2 { + PRAGMA wal_checkpoint; + INSERT INTO t1 VALUES('k', 'l'); + } + set {} {} + } {} + do_test 1.2.8 { sql1 "SELECT * FROM t1" } {a b c d e f g h i j k l} + + # Now check that if the readonly_shm option is not supplied, or if it + # is set to zero, it is not possible to connect to the database without + # read-write access to the shm. + do_test 1.3.1 { + code1 { db close } + code1 { sqlite3 db test.db } + csql1 { SELECT * FROM t1 } + } {1 {unable to open database file}} + + # Also test that if the -shm file can be opened for read/write access, + # it is, even if readonly_shm=1 is present in the URI. + do_test 1.3.2.1 { + code1 { db close } + code2 { db2 close } + file exists test.db-shm + } {0} + do_test 1.3.2.2 { + code1 { sqlite3 db file:test.db?readonly_shm=1 } + sql1 { SELECT * FROM t1 } + } {a b c d e f g h i j k l} + do_test 1.3.2.3 { + code1 { db close } + close [open test.db-shm w] + file attributes test.db-shm -permissions r--r--r-- + code1 { sqlite3 db file:test.db?readonly_shm=1 } + csql1 { SELECT * FROM t1 } + } {1 {attempt to write a readonly database}} + do_test 1.3.2.4 { + code1 { sqlite3_extended_errcode db } + } {SQLITE_READONLY_RECOVERY} +} + +finish_test diff --git a/tool/build-shell.sh b/tool/build-shell.sh new file mode 100644 index 0000000000..54e8308095 --- /dev/null +++ b/tool/build-shell.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# +# This script demonstrates how to do a full-featured build of the sqlite3 +# command-line shell on Linux. +# +# SQLite source code should be in a sibling directory named "sqlite". For +# example, put SQLite sources in ~/sqlite/sqlite and run this script from +# ~/sqlite/bld. There should be an appropriate Makefile in the current +# directory as well. +# +make sqlite3.c +gcc -o sqlite3 -g -Os -I. \ + -DSQLITE_THREADSAFE=0 \ + -DSQLITE_ENABLE_VFSTRACE \ + -DSQLITE_ENABLE_STAT2 \ + -DSQLITE_ENABLE_FTS3 \ + -DSQLITE_ENABLE_RTREE \ + -DHAVE_READLINE \ + -DHAVE_USLEEP=1 \ + ../sqlite/src/shell.c ../sqlite/src/test_vfstrace.c \ + sqlite3.c -ldl -lreadline -lncurses diff --git a/tool/lemon.c b/tool/lemon.c index 898022e28c..0a50524854 100644 --- a/tool/lemon.c +++ b/tool/lemon.c @@ -3434,6 +3434,10 @@ void print_stack_union( /* Allocate and initialize types[] and allocate stddt[] */ arraysize = lemp->nsymbol * 2; types = (char**)calloc( arraysize, sizeof(char*) ); + if( types==0 ){ + fprintf(stderr,"Out of memory.\n"); + exit(1); + } for(i=0; ivartype ){ @@ -3447,7 +3451,7 @@ void print_stack_union( if( len>maxdtlength ) maxdtlength = len; } stddt = (char*)malloc( maxdtlength*2 + 1 ); - if( types==0 || stddt==0 ){ + if( stddt==0 ){ fprintf(stderr,"Out of memory.\n"); exit(1); } diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl index fa99f2df7b..b56eaea0c2 100644 --- a/tool/mksqlite3c.tcl +++ b/tool/mksqlite3c.tcl @@ -46,6 +46,8 @@ close $in # of the file. # set out [open sqlite3.c w] +# Force the output to use unix line endings, even on Windows. +# fconfigure $out -translation lf set today [clock format [clock seconds] -format "%Y-%m-%d %H:%M:%S UTC" -gmt 1] puts $out [subst \ {/****************************************************************************** diff --git a/tool/mksqlite3h.tcl b/tool/mksqlite3h.tcl index 554069c323..f68f61a368 100644 --- a/tool/mksqlite3h.tcl +++ b/tool/mksqlite3h.tcl @@ -65,6 +65,9 @@ close $in set varpattern {^[a-zA-Z][a-zA-Z_0-9 *]+sqlite3_[_a-zA-Z0-9]+(\[|;| =)} set declpattern {^ *[a-zA-Z][a-zA-Z_0-9 ]+ \**sqlite3_[_a-zA-Z0-9]+\(} +# Force the output to use unix line endings, even on Windows. +fconfigure stdout -translation lf + # Process the src/sqlite.h.in ext/rtree/sqlite3rtree.h files. # foreach file [list $TOP/src/sqlite.h.in $TOP/ext/rtree/sqlite3rtree.h] { diff --git a/tool/shell1.test b/tool/shell1.test index 068f8ea25c..9dd9df5555 100644 --- a/tool/shell1.test +++ b/tool/shell1.test @@ -199,8 +199,9 @@ do_test shell1-1.15.3 { # -version show SQLite version do_test shell1-1.16.1 { - catchcmd "-version test.db" "" -} {0 3.7.7} + set x [catchcmd "-version test.db" ""] + regexp {0 \{3.\d.\d+ 20\d\d-[01]\d-\d\d \d\d:\d\d:\d\d [0-9a-f]+\}} $x +} 1 #---------------------------------------------------------------------------- # Test cases shell1-2.*: Basic "dot" command token parsing. diff --git a/tool/symbols.sh b/tool/symbols.sh new file mode 100644 index 0000000000..8aec00569a --- /dev/null +++ b/tool/symbols.sh @@ -0,0 +1,34 @@ +#!/bin/sh +# +# Run this script in a directory that contains a valid SQLite makefile in +# order to verify that unintentionally exported symbols. +# +make sqlite3.c + +echo '****** Exported symbols from a build including RTREE, FTS4 & ICU ******' +gcc -c -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE \ + -DSQLITE_ENABLE_MEMORY_MANAGEMENT -DSQLITE_ENABLE_STAT2 \ + -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_ENABLE_UNLOCK_NOTIFY \ + -DSQLITE_ENABLE_COLUMN_METADATA -DSQLITE_ENABLE_ATOMIC_WRITE \ + -DSQLITE_ENABLE_ICU \ + sqlite3.c +nm sqlite3.o | grep ' T ' | sort -k 3 + +echo '****** Surplus symbols from a build including RTREE, FTS4 & ICU ******' +nm sqlite3.o | grep ' T ' | grep -v ' sqlite3_' + +echo '****** Dependencies of the core. No extensions. No OS interface *******' +gcc -c -DSQLITE_ENABLE_MEMORY_MANAGEMENT -DSQLITE_ENABLE_STAT2 \ + -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_ENABLE_UNLOCK_NOTIFY \ + -DSQLITE_ENABLE_COLUMN_METADATA -DSQLITE_ENABLE_ATOMIC_WRITE \ + -DSQLITE_OS_OTHER -DSQLITE_THREADSAFE=0 \ + sqlite3.c +nm sqlite3.o | grep ' U ' | sort -k 3 + +echo '****** Dependencies including RTREE & FTS4 *******' +gcc -c -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE \ + -DSQLITE_ENABLE_MEMORY_MANAGEMENT -DSQLITE_ENABLE_STAT2 \ + -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_ENABLE_UNLOCK_NOTIFY \ + -DSQLITE_ENABLE_COLUMN_METADATA -DSQLITE_ENABLE_ATOMIC_WRITE \ + sqlite3.c +nm sqlite3.o | grep ' U ' | sort -k 3 diff --git a/tool/warnings.sh b/tool/warnings.sh new file mode 100644 index 0000000000..e1fa2b2d91 --- /dev/null +++ b/tool/warnings.sh @@ -0,0 +1,14 @@ +#/bin/sh +# +# Run this script in a directory with a working makefile to check for +# compiler warnings in SQLite. +# +make sqlite3.c +echo '********** No optimizations. Includes FTS4 and RTREE *********' +gcc -c -Wshadow -Wall -Wextra -pedantic-errors -Wno-long-long -std=c89 \ + -ansi -DHAVE_STDINT_H -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_RTREE \ + sqlite3.c +echo '********** Optimized -O3. Includes FTS4 and RTREE *********' +gcc -O3 -c -Wshadow -Wall -Wextra -pedantic-errors -Wno-long-long -std=c89 \ + -ansi -DHAVE_STDINT_H -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_RTREE \ + sqlite3.c