diff --git a/Makefile.in b/Makefile.in index 72f7cfc007..cd024e950a 100644 --- a/Makefile.in +++ b/Makefile.in @@ -177,7 +177,7 @@ LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \ notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \ pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \ random.lo resolve.lo rowset.lo rtree.lo select.lo status.lo \ - table.lo tokenize.lo trigger.lo \ + table.lo threads.lo tokenize.lo trigger.lo \ update.lo util.lo vacuum.lo \ vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \ vdbetrace.lo wal.lo walker.lo where.lo utf.lo vtab.lo @@ -263,6 +263,7 @@ SRC = \ $(TOP)/src/sqliteInt.h \ $(TOP)/src/sqliteLimit.h \ $(TOP)/src/table.c \ + $(TOP)/src/threads.c \ $(TOP)/src/tclsqlite.c \ $(TOP)/src/tokenize.c \ $(TOP)/src/trigger.c \ @@ -358,6 +359,7 @@ TESTSRC = \ $(TOP)/src/test_autoext.c \ $(TOP)/src/test_async.c \ $(TOP)/src/test_backup.c \ + $(TOP)/src/test_blob.c \ $(TOP)/src/test_btree.c \ $(TOP)/src/test_config.c \ $(TOP)/src/test_demovfs.c \ @@ -393,6 +395,7 @@ TESTSRC = \ TESTSRC += \ $(TOP)/ext/misc/amatch.c \ $(TOP)/ext/misc/closure.c \ + $(TOP)/ext/misc/eval.c \ $(TOP)/ext/misc/fileio.c \ $(TOP)/ext/misc/fuzzer.c \ $(TOP)/ext/misc/ieee754.c \ @@ -527,6 +530,11 @@ mptester$(EXE): sqlite3.c $(TOP)/mptest/mptest.c $(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.c \ $(TLIBS) -rpath "$(libdir)" +mptest: mptester$(EXE) + rm -f mptest1.db + ./mptester$(EXE) mptest1.db $(TOP)/mptest/crash01.test + rm -f mptest2.db + ./mptester$(EXE) mptest2.db $(TOP)/mptest/multiwrite01.test # This target creates a directory named "tsrc" and fills it with # copies of all of the C source code and header files needed to @@ -737,6 +745,9 @@ status.lo: $(TOP)/src/status.c $(HDR) table.lo: $(TOP)/src/table.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/table.c +threads.lo: $(TOP)/src/threads.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/threads.c + tokenize.lo: $(TOP)/src/tokenize.c keywordhash.h $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/tokenize.c @@ -939,8 +950,20 @@ sqlite3_analyzer.c: sqlite3.c $(TOP)/src/test_stat.c $(TOP)/src/tclsqlite.c $(TO sqlite3_analyzer$(TEXE): sqlite3_analyzer.c $(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS) -showdb$(TEXE): $(TOP)/tool/showdb.c sqlite3.c - $(LTLINK) -o $@ $(TOP)/tool/showdb.c sqlite3.c $(TLIBS) +showdb$(TEXE): $(TOP)/tool/showdb.c sqlite3.lo + $(LTLINK) -o $@ $(TOP)/tool/showdb.c sqlite3.lo $(TLIBS) + +showstat4$(TEXE): $(TOP)/tool/showstat4.c sqlite3.lo + $(LTLINK) -o $@ $(TOP)/tool/showstat4.c sqlite3.lo $(TLIBS) + +showjournal$(TEXE): $(TOP)/tool/showjournal.c sqlite3.lo + $(LTLINK) -o $@ $(TOP)/tool/showjournal.c sqlite3.lo $(TLIBS) + +showwal$(TEXE): $(TOP)/tool/showwal.c sqlite3.lo + $(LTLINK) -o $@ $(TOP)/tool/showwal.c sqlite3.lo $(TLIBS) + +rollback-test$(TEXE): $(TOP)/tool/rollback-test.c sqlite3.lo + $(LTLINK) -o $@ $(TOP)/tool/rollback-test.c sqlite3.lo $(TLIBS) LogEst$(TEXE): $(TOP)/tool/logest.c sqlite3.h $(LTLINK) -I. -o $@ $(TOP)/tool/logest.c @@ -951,6 +974,34 @@ wordcount$(TEXE): $(TOP)/test/wordcount.c sqlite3.c speedtest1$(TEXE): $(TOP)/test/wordcount.c sqlite3.lo $(LTLINK) -o $@ $(TOP)/test/speedtest1.c sqlite3.lo $(TLIBS) +# This target will fail if the SQLite amalgamation contains any exported +# symbols that do not begin with "sqlite3_". It is run as part of the +# releasetest.tcl script. +# +checksymbols: sqlite3.lo + nm -g --defined-only sqlite3.o | grep -v " sqlite3_" ; test $$? -ne 0 + echo '0 errors out of 1 tests' + +# The next two rules are used to support the "threadtest" target. Building +# threadtest runs a few thread-safety tests that are implemented in C. This +# target is invoked by the releasetest.tcl script. +# +THREADTEST3_SRC = $(TOP)/test/threadtest3.c \ + $(TOP)/test/tt3_checkpoint.c \ + $(TOP)/test/tt3_index.c \ + $(TOP)/test/tt3_vacuum.c \ + $(TOP)/test/tt3_stress.c \ + $(TOP)/test/tt3_lookaside1.c + +threadtest3$(TEXE): sqlite3.lo $(THREADTEST3_SRC) + $(LTLINK) $(TOP)/test/threadtest3.c sqlite3.lo -o $@ $(TLIBS) + +threadtest: threadtest3$(TEXE) + ./threadtest3$(TEXE) + +releasetest: + $(TCLSH_CMD) $(TOP)/test/releasetest.tcl + # Standard install and cleanup targets # lib_install: libsqlite3.la @@ -985,6 +1036,9 @@ clean: rm -rf tsrc .target_source rm -f tclsqlite3$(TEXE) rm -f testfixture$(TEXE) test.db + rm -f LogEst$(TEXE) fts3view$(TEXE) rollback-test$(TEXE) showdb$(TEXE) + rm -f showjournal$(TEXE) showstat4$(TEXE) showwal$(TEXE) speedtest1$(TEXE) + rm -f wordcount$(TEXE) rm -f sqlite3.dll sqlite3.lib sqlite3.exp sqlite3.def rm -f sqlite3.c rm -f sqlite3rc.h diff --git a/Makefile.msc b/Makefile.msc index 409f7b3d28..3fc675ee4f 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -108,11 +108,12 @@ WIN32HEAP = 0 # levels. Currently, the recognized values for DEBUG are: # # 0 == NDEBUG: Disables assert() and other runtime diagnostics. -# 1 == Disables NDEBUG and all optimizations and then enables PDBs. -# 2 == SQLITE_DEBUG: Enables various diagnostics messages and code. -# 3 == SQLITE_WIN32_MALLOC_VALIDATE: Validate the Win32 native heap per call. -# 4 == SQLITE_DEBUG_OS_TRACE: Enables output from the OSTRACE() macros. -# 5 == SQLITE_ENABLE_IOTRACE: Enables output from the IOTRACE() macros. +# 1 == SQLITE_ENABLE_API_ARMOR: extra attempts to detect misuse of the API. +# 2 == Disables NDEBUG and all optimizations and then enables PDBs. +# 3 == SQLITE_DEBUG: Enables various diagnostics messages and code. +# 4 == SQLITE_WIN32_MALLOC_VALIDATE: Validate the Win32 native heap per call. +# 5 == SQLITE_DEBUG_OS_TRACE: Enables output from the OSTRACE() macros. +# 6 == SQLITE_ENABLE_IOTRACE: Enables output from the IOTRACE() macros. # !IFNDEF DEBUG DEBUG = 0 @@ -280,7 +281,7 @@ RCC = $(RCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP # MSVC runtime library. # !IF $(FOR_WINRT)!=0 || $(USE_CRT_DLL)!=0 -!IF $(DEBUG)>0 +!IF $(DEBUG)>1 TCC = $(TCC) -MDd BCC = $(BCC) -MDd !ELSE @@ -288,7 +289,7 @@ TCC = $(TCC) -MD BCC = $(BCC) -MD !ENDIF !ELSE -!IF $(DEBUG)>0 +!IF $(DEBUG)>1 TCC = $(TCC) -MTd BCC = $(BCC) -MTd !ELSE @@ -313,7 +314,7 @@ RCC = $(RCC) -I$(TOP)\ext\rtree # options are necessary in order to allow debugging symbols to # work correctly with Visual Studio when using the amalgamation. # -!IF $(DEBUG)>0 +!IF $(DEBUG)>1 MKSQLITE3C_ARGS = --linemacros !ELSE MKSQLITE3C_ARGS = @@ -329,17 +330,22 @@ BCC = $(BCC) -DNDEBUG RCC = $(RCC) -DNDEBUG !ENDIF -!IF $(DEBUG)>1 +!IF $(DEBUG)>0 +TCC = $(TCC) -DSQLITE_ENABLE_API_ARMOR +RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR +!ENDIF + +!IF $(DEBUG)>2 TCC = $(TCC) -DSQLITE_DEBUG RCC = $(RCC) -DSQLITE_DEBUG !ENDIF -!IF $(DEBUG)>3 +!IF $(DEBUG)>4 TCC = $(TCC) -DSQLITE_DEBUG_OS_TRACE=1 RCC = $(RCC) -DSQLITE_DEBUG_OS_TRACE=1 !ENDIF -!IF $(DEBUG)>4 +!IF $(DEBUG)>5 TCC = $(TCC) -DSQLITE_ENABLE_IOTRACE RCC = $(RCC) -DSQLITE_ENABLE_IOTRACE !ENDIF @@ -371,7 +377,7 @@ RCC = $(RCC) -DSQLITE_WIN32_MALLOC=1 # Validate the heap on every call into the native Win32 heap subsystem? # -!IF $(DEBUG)>2 +!IF $(DEBUG)>3 TCC = $(TCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1 RCC = $(RCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1 !ENDIF @@ -482,7 +488,7 @@ RCC = $(RCC) $(OPTS) # If compiling for debugging, add some defines. # -!IF $(DEBUG)>0 +!IF $(DEBUG)>1 TCC = $(TCC) -D_DEBUG BCC = $(BCC) -D_DEBUG RCC = $(RCC) -D_DEBUG @@ -491,7 +497,7 @@ RCC = $(RCC) -D_DEBUG # If optimizations are enabled or disabled (either implicitly or # explicitly), add the necessary flags. # -!IF $(DEBUG)>0 || $(OPTIMIZATIONS)==0 +!IF $(DEBUG)>1 || $(OPTIMIZATIONS)==0 TCC = $(TCC) -Od BCC = $(BCC) -Od !ELSEIF $(OPTIMIZATIONS)>=3 @@ -507,7 +513,7 @@ BCC = $(BCC) -O1 # If symbols are enabled (or compiling for debugging), enable PDBs. # -!IF $(DEBUG)>0 || $(SYMBOLS)!=0 +!IF $(DEBUG)>1 || $(SYMBOLS)!=0 TCC = $(TCC) -Zi BCC = $(BCC) -Zi !ENDIF @@ -592,7 +598,7 @@ LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:kernel32.lib /NODEFAULTLIB:ole32.lib # If either debugging or symbols are enabled, enable PDBs. # -!IF $(DEBUG)>0 || $(SYMBOLS)!=0 +!IF $(DEBUG)>1 || $(SYMBOLS)!=0 LDFLAGS = /DEBUG !ENDIF @@ -636,7 +642,7 @@ LIBOBJS0 = vdbe.lo parse.lo alter.lo analyze.lo attach.lo auth.lo \ notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \ pager.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \ random.lo resolve.lo rowset.lo rtree.lo select.lo status.lo \ - table.lo tokenize.lo trigger.lo \ + table.lo threads.lo tokenize.lo trigger.lo \ update.lo util.lo vacuum.lo \ vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \ vdbetrace.lo wal.lo walker.lo where.lo utf.lo vtab.lo @@ -733,6 +739,7 @@ SRC = \ $(TOP)\src\sqliteInt.h \ $(TOP)\src\sqliteLimit.h \ $(TOP)\src\table.c \ + $(TOP)\src\threads.c \ $(TOP)\src\tclsqlite.c \ $(TOP)\src\tokenize.c \ $(TOP)\src\trigger.c \ @@ -827,6 +834,7 @@ TESTSRC = \ $(TOP)\src\test_autoext.c \ $(TOP)\src\test_async.c \ $(TOP)\src\test_backup.c \ + $(TOP)\src\test_blob.c \ $(TOP)\src\test_btree.c \ $(TOP)\src\test_config.c \ $(TOP)\src\test_demovfs.c \ @@ -862,6 +870,7 @@ TESTSRC = \ TESTEXT = \ $(TOP)\ext\misc\amatch.c \ $(TOP)\ext\misc\closure.c \ + $(TOP)\ext\misc\eval.c \ $(TOP)\ext\misc\fileio.c \ $(TOP)\ext\misc\fuzzer.c \ $(TOP)\ext\misc\ieee754.c \ @@ -1216,6 +1225,9 @@ status.lo: $(TOP)\src\status.c $(HDR) table.lo: $(TOP)\src\table.c $(HDR) $(LTCOMPILE) -c $(TOP)\src\table.c +threads.lo: $(TOP)\src\threads.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\threads.c + tokenize.lo: $(TOP)\src\tokenize.c keywordhash.h $(HDR) $(LTCOMPILE) -c $(TOP)\src\tokenize.c @@ -1435,6 +1447,26 @@ showdb.exe: $(TOP)\tool\showdb.c $(SQLITE3C) $(LTLINK) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\tool\showdb.c $(SQLITE3C) +showstat4.exe: $(TOP)\tool\showstat4.c $(SQLITE3C) + $(LTLINK) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ + $(TOP)\tool\showstat4.c $(SQLITE3C) + +showjournal.exe: $(TOP)\tool\showjournal.c $(SQLITE3C) + $(LTLINK) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ + $(TOP)\tool\showjournal.c $(SQLITE3C) + +showwal.exe: $(TOP)\tool\showwal.c $(SQLITE3C) + $(LTLINK) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ + $(TOP)\tool\showwal.c $(SQLITE3C) + +fts3view.exe: $(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) + $(LTLINK) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ + $(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) + +rollback-test.exe: $(TOP)\tool\rollback-test.c $(SQLITE3C) + $(LTLINK) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ + $(TOP)\tool\rollback-test.c $(SQLITE3C) + LogEst.exe: $(TOP)\tool\logest.c sqlite3.h $(LTLINK) -Fe$@ $(TOP)\tool\LogEst.c @@ -1463,6 +1495,9 @@ clean: del /Q tclsqlite3.exe tclsqlite3.exp del /Q testloadext.dll testloadext.exp del /Q testfixture.exe testfixture.exp test.db + del /Q LogEst.exe fts3view.exe rollback-test.exe showdb.exe + del /Q showjournal.exe showstat4.exe showwal.exe speedtest1.exe + del /Q wordcount.exe del /Q sqlite3.dll sqlite3.lib sqlite3.exp sqlite3.def del /Q sqlite3.c sqlite3-*.c del /Q sqlite3rc.h diff --git a/README.md b/README.md index 5e52dea507..ff8dc34aef 100644 --- a/README.md +++ b/README.md @@ -53,10 +53,10 @@ to the "sqlite3.dll" command line above. When debugging into the SQLite code, adding the "DEBUG=1" argument to one of the above command lines is recommended. -SQLite does not require Tcl to run, but a Tcl installation is required -by the makefiles (including those for MSVC). SQLite contains a lot of -generated code and Tcl is used to do much of that code generation. The -makefiles also require AWK. +SQLite does not require [Tcl](http://www.tcl.tk/) to run, but a Tcl installation +is required by the makefiles (including those for MSVC). SQLite contains +a lot of generated code and Tcl is used to do much of that code generation. +The makefiles also require AWK. ## Source Code Tour @@ -95,14 +95,14 @@ manually-edited files and automatically-generated files. The SQLite interface is defined by the **sqlite3.h** header file, which is generated from src/sqlite.h.in, ./manifest.uuid, and ./VERSION. The -Tcl script at tool/mksqlite3h.tcl does the conversion. The manifest.uuid -file contains the SHA1 hash of the particular check-in and is used to generate -the SQLITE_SOURCE_ID macro. The VERSION file contains the current SQLite -version number. The sqlite3.h header is really just a copy of src/sqlite.h.in -with the source-id and version number inserted at just the right spots. -Note that comment text in the sqlite3.h file is used to generate much of -the SQLite API documentation. The Tcl scripts used to generate that -documentation are in a separate source repository. +[Tcl script](http://www.tcl.tk) at tool/mksqlite3h.tcl does the conversion. +The manifest.uuid file contains the SHA1 hash of the particular check-in +and is used to generate the SQLITE\_SOURCE\_ID macro. The VERSION file +contains the current SQLite version number. The sqlite3.h header is really +just a copy of src/sqlite.h.in with the source-id and version number inserted +at just the right spots. Note that comment text in the sqlite3.h file is +used to generate much of the SQLite API documentation. The Tcl scripts +used to generate that documentation are in a separate source repository. The SQL language parser is **parse.c** which is generate from a grammar in the src/parse.y file. The conversion of "parse.y" into "parse.c" is done diff --git a/VERSION b/VERSION index 0cbfaed0d9..d7d8e42315 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.8.5 +3.8.8 diff --git a/autoconf/tea/Makefile.in b/autoconf/tea/Makefile.in index 08b1a44182..a8708974b0 100644 --- a/autoconf/tea/Makefile.in +++ b/autoconf/tea/Makefile.in @@ -73,6 +73,7 @@ exec_prefix = @exec_prefix@ bindir = @bindir@ libdir = @libdir@ +datarootdir = @datarootdir@ datadir = @datadir@ mandir = @mandir@ includedir = @includedir@ diff --git a/autoconf/tea/configure.in b/autoconf/tea/configure.in index ec9c565c6f..8df0af6195 100644 --- a/autoconf/tea/configure.in +++ b/autoconf/tea/configure.in @@ -166,8 +166,10 @@ AC_DEFINE(USE_TCL_STUBS, 1, [Use Tcl stubs]) #-------------------------------------------------------------------- # Redefine fdatasync as fsync on systems that lack fdatasync #-------------------------------------------------------------------- - -AC_CHECK_FUNC(fdatasync, , AC_DEFINE(fdatasync, fsync)) +# +#AC_CHECK_FUNC(fdatasync, , AC_DEFINE(fdatasync, fsync)) +# Check for library functions that SQLite can optionally use. +AC_CHECK_FUNCS([fdatasync usleep fullfsync localtime_r gmtime_r]) AC_FUNC_STRERROR_R diff --git a/autoconf/tea/tclconfig/tcl.m4 b/autoconf/tea/tclconfig/tcl.m4 index 66214e78a2..4b4bd1e888 100644 --- a/autoconf/tea/tclconfig/tcl.m4 +++ b/autoconf/tea/tclconfig/tcl.m4 @@ -1641,6 +1641,7 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [ SHLIB_CFLAGS="-fPIC" SHLIB_LD="${CC} -shared" TCL_SHLIB_LD_EXTRAS="-Wl,-soname=\$[@]" + TK_SHLIB_LD_EXTRAS="-Wl,-soname,\$[@]" SHLIB_SUFFIX=".so" LDFLAGS="" AS_IF([test $doRpath = yes], [ @@ -1651,11 +1652,15 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [ LIBS=`echo $LIBS | sed s/-pthread//` CFLAGS="$CFLAGS $PTHREAD_CFLAGS" LDFLAGS="$LDFLAGS $PTHREAD_LIBS"]) - # Version numbers are dot-stripped by system policy. - TCL_TRIM_DOTS=`echo ${PACKAGE_VERSION} | tr -d .` - UNSHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.a' - SHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}\$\{DBGX\}.so.1' - TCL_LIB_VERSIONS_OK=nodots + case $system in + FreeBSD-3.*) + # Version numbers are dot-stripped by system policy. + TCL_TRIM_DOTS=`echo ${VERSION} | tr -d .` + UNSHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.a' + SHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.so' + TCL_LIB_VERSIONS_OK=nodots + ;; + esac ;; Darwin-*) CFLAGS_OPTIMIZE="-Os" @@ -1826,8 +1831,8 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [ SHLIB_CFLAGS="-fPIC -melf" LDFLAGS="$LDFLAGS -melf -Wl,-Bexport" ], [ - SHLIB_CFLAGS="-Kpic -belf" - LDFLAGS="$LDFLAGS -belf -Wl,-Bexport" + SHLIB_CFLAGS="-Kpic -belf" + LDFLAGS="$LDFLAGS -belf -Wl,-Bexport" ]) SHLIB_LD="ld -G" SHLIB_LD_LIBS="" @@ -4158,8 +4163,6 @@ AC_DEFUN([TEA_PATH_CELIB], [ fi fi ]) - - # Local Variables: # mode: autoconf # End: diff --git a/configure b/configure index 8be06033f0..c0f751db62 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.62 for sqlite 3.8.5. +# Generated by GNU Autoconf 2.62 for sqlite 3.8.8. # # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, # 2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc. @@ -743,8 +743,8 @@ SHELL=${CONFIG_SHELL-/bin/sh} # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.8.5' -PACKAGE_STRING='sqlite 3.8.5' +PACKAGE_VERSION='3.8.8' +PACKAGE_STRING='sqlite 3.8.8' PACKAGE_BUGREPORT='' # Factoring default headers for most tests. @@ -1483,7 +1483,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.8.5 to adapt to many kinds of systems. +\`configure' configures sqlite 3.8.8 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1548,7 +1548,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.8.5:";; + short | recursive ) echo "Configuration of sqlite 3.8.8:";; esac cat <<\_ACEOF @@ -1664,7 +1664,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.8.5 +sqlite configure 3.8.8 generated by GNU Autoconf 2.62 Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, @@ -1678,7 +1678,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.8.5, which was +It was created by sqlite $as_me 3.8.8, which was generated by GNU Autoconf 2.62. Invocation command line was $ $0 $@ @@ -14021,7 +14021,7 @@ exec 6>&1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.8.5, which was +This file was extended by sqlite $as_me 3.8.8, which was generated by GNU Autoconf 2.62. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -14074,7 +14074,7 @@ Report bugs to ." _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_version="\\ -sqlite config.status 3.8.5 +sqlite config.status 3.8.8 configured by $0, generated by GNU Autoconf 2.62, with options \\"`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\" diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 1676fd41e1..2b93c62715 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -3116,6 +3116,7 @@ static int fts3FilterMethod( /* In case the cursor has been used before, clear it now. */ sqlite3_finalize(pCsr->pStmt); sqlite3_free(pCsr->aDoclist); + sqlite3_free(pCsr->aMatchinfo); sqlite3Fts3ExprFree(pCsr->pExpr); memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); @@ -3750,7 +3751,7 @@ static void hashDestroy(void *p){ */ void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule); -#ifdef SQLITE_ENABLE_FTS4_UNICODE61 +#ifndef SQLITE_DISABLE_FTS3_UNICODE void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const**ppModule); #endif #ifdef SQLITE_ENABLE_ICU @@ -3768,7 +3769,7 @@ int sqlite3Fts3Init(sqlite3 *db){ Fts3Hash *pHash = 0; const sqlite3_tokenizer_module *pSimple = 0; const sqlite3_tokenizer_module *pPorter = 0; -#ifdef SQLITE_ENABLE_FTS4_UNICODE61 +#ifndef SQLITE_DISABLE_FTS3_UNICODE const sqlite3_tokenizer_module *pUnicode = 0; #endif @@ -3777,7 +3778,7 @@ int sqlite3Fts3Init(sqlite3 *db){ sqlite3Fts3IcuTokenizerModule(&pIcu); #endif -#ifdef SQLITE_ENABLE_FTS4_UNICODE61 +#ifndef SQLITE_DISABLE_FTS3_UNICODE sqlite3Fts3UnicodeTokenizer(&pUnicode); #endif @@ -3805,7 +3806,7 @@ int sqlite3Fts3Init(sqlite3 *db){ if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple) || sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter) -#ifdef SQLITE_ENABLE_FTS4_UNICODE61 +#ifndef SQLITE_DISABLE_FTS3_UNICODE || sqlite3Fts3HashInsert(pHash, "unicode61", 10, (void *)pUnicode) #endif #ifdef SQLITE_ENABLE_ICU @@ -4426,7 +4427,7 @@ static int fts3EvalIncrPhraseNext( bMaxSet = 1; } } - assert( rc!=SQLITE_OK || a[p->nToken-1].bIgnore==0 ); + assert( rc!=SQLITE_OK || (p->nToken>=1 && a[p->nToken-1].bIgnore==0) ); assert( rc!=SQLITE_OK || bMaxSet ); /* Keep advancing iterators until they all point to the same document */ diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 1383102f4c..b2827b7352 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -585,7 +585,7 @@ int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); int sqlite3Fts3InitTok(sqlite3*, Fts3Hash *); /* fts3_unicode2.c (functions generated by parsing unicode text files) */ -#ifdef SQLITE_ENABLE_FTS4_UNICODE61 +#ifndef SQLITE_DISABLE_FTS3_UNICODE int sqlite3FtsUnicodeFold(int, int); int sqlite3FtsUnicodeIsalnum(int); int sqlite3FtsUnicodeIsdiacritic(int); diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index f5d28cbfcc..2ba786ce80 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -190,7 +190,7 @@ static int getNextToken( /* Set variable i to the maximum number of bytes of input to tokenize. */ for(i=0; ibRemoveDiacritic = 1; @@ -318,7 +318,7 @@ static int unicodeNext( ){ unicode_cursor *pCsr = (unicode_cursor *)pC; unicode_tokenizer *p = ((unicode_tokenizer *)pCsr->base.pTokenizer); - int iCode; + int iCode = 0; char *zOut; const unsigned char *z = &pCsr->aInput[pCsr->iOff]; const unsigned char *zStart = z; @@ -363,11 +363,11 @@ static int unicodeNext( ); /* Set the output variables and return. */ - pCsr->iOff = (z - pCsr->aInput); + pCsr->iOff = (int)(z - pCsr->aInput); *paToken = pCsr->zToken; - *pnToken = zOut - pCsr->zToken; - *piStart = (zStart - pCsr->aInput); - *piEnd = (zEnd - pCsr->aInput); + *pnToken = (int)(zOut - pCsr->zToken); + *piStart = (int)(zStart - pCsr->aInput); + *piEnd = (int)(zEnd - pCsr->aInput); *piPos = pCsr->iToken++; return SQLITE_OK; } @@ -390,4 +390,4 @@ void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const **ppModule){ } #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ -#endif /* ifndef SQLITE_ENABLE_FTS4_UNICODE61 */ +#endif /* ifndef SQLITE_DISABLE_FTS3_UNICODE */ diff --git a/ext/fts3/fts3_unicode2.c b/ext/fts3/fts3_unicode2.c index fba688ff9c..20b7a25dbf 100644 --- a/ext/fts3/fts3_unicode2.c +++ b/ext/fts3/fts3_unicode2.c @@ -15,7 +15,7 @@ ** DO NOT EDIT THIS MACHINE GENERATED FILE. */ -#if defined(SQLITE_ENABLE_FTS4_UNICODE61) +#ifndef SQLITE_DISABLE_FTS3_UNICODE #if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) #include @@ -39,7 +39,7 @@ int sqlite3FtsUnicodeIsalnum(int c){ ** C. It is not possible to represent a range larger than 1023 codepoints ** using this format. */ - const static unsigned int aEntry[] = { + static const unsigned int aEntry[] = { 0x00000030, 0x0000E807, 0x00016C06, 0x0001EC2F, 0x0002AC07, 0x0002D001, 0x0002D803, 0x0002EC01, 0x0002FC01, 0x00035C01, 0x0003DC01, 0x000B0804, 0x000B480E, 0x000B9407, 0x000BB401, @@ -131,7 +131,7 @@ int sqlite3FtsUnicodeIsalnum(int c){ return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 ); }else if( c<(1<<22) ){ unsigned int key = (((unsigned int)c)<<10) | 0x000003FF; - int iRes; + int iRes = 0; int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; int iLo = 0; while( iHi>=iLo ){ @@ -202,7 +202,7 @@ static int remove_diacritic(int c){ } assert( key>=aDia[iRes] ); return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]); -}; +} /* @@ -362,4 +362,4 @@ int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){ return ret; } #endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */ -#endif /* !defined(SQLITE_ENABLE_FTS4_UNICODE61) */ +#endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */ diff --git a/ext/fts3/tool/fts3view.c b/ext/fts3/tool/fts3view.c index 479ae9868d..3dc1ba80fe 100644 --- a/ext/fts3/tool/fts3view.c +++ b/ext/fts3/tool/fts3view.c @@ -376,7 +376,7 @@ static void showSegmentStats(sqlite3 *db, const char *zTab){ sqlite3_finalize(pStmt); nLeaf = nSeg - nIdx; printf("Leaf segments larger than %5d bytes.... %9d %5.2f%%\n", - pgsz-45, n, n*100.0/nLeaf); + pgsz-45, n, nLeaf>0 ? n*100.0/nLeaf : 0.0); pStmt = prepare(db, "SELECT max(level%%1024) FROM '%q_segdir'", zTab); mxLevel = 0; @@ -554,7 +554,7 @@ static void decodeSegment( sqlite3_int64 n; sqlite3_int64 iDocsz; int iHeight; - int i = 0; + sqlite3_int64 i = 0; int cnt = 0; char zTerm[1000]; @@ -576,12 +576,12 @@ static void decodeSegment( fprintf(stderr, "term to long\n"); exit(1); } - memcpy(zTerm+iPrefix, aData+i, nTerm); + memcpy(zTerm+iPrefix, aData+i, (size_t)nTerm); zTerm[iPrefix+nTerm] = 0; i += nTerm; if( iHeight==0 ){ i += getVarint(aData+i, &iDocsz); - printf("term: %-25s doclist %7lld bytes offset %d\n", zTerm, iDocsz, i); + printf("term: %-25s doclist %7lld bytes offset %lld\n", zTerm, iDocsz, i); i += iDocsz; }else{ printf("term: %-25s child %lld\n", zTerm, ++iChild); @@ -749,18 +749,19 @@ static void decodeDoclist( */ static void showDoclist(sqlite3 *db, const char *zTab){ const unsigned char *aData; - sqlite3_int64 offset, nData; + sqlite3_int64 offset; + int nData; sqlite3_stmt *pStmt; offset = atoi64(azExtra[1]); - nData = atoi64(azExtra[2]); + nData = atoi(azExtra[2]); pStmt = prepareToGetSegment(db, zTab, azExtra[0]); if( sqlite3_step(pStmt)!=SQLITE_ROW ){ sqlite3_finalize(pStmt); return; } aData = sqlite3_column_blob(pStmt, 0); - printf("Doclist at %s offset %lld of size %lld bytes:\n", + printf("Doclist at %s offset %lld of size %d bytes:\n", azExtra[0], offset, nData); if( findOption("raw", 0, 0)!=0 ){ printBlob(aData+offset, nData); diff --git a/ext/fts3/unicode/mkunicode.tcl b/ext/fts3/unicode/mkunicode.tcl index f1adb5ffde..b33b05c781 100644 --- a/ext/fts3/unicode/mkunicode.tcl +++ b/ext/fts3/unicode/mkunicode.tcl @@ -160,7 +160,7 @@ proc print_rd {map} { } assert( key>=aDia[iRes] ); return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);} - puts "\};" + puts "\}" } proc print_isdiacritic {zFunc map} { @@ -298,7 +298,7 @@ proc an_print_range_array {lRange} { ** using this format. */ }] - puts -nonewline " const static unsigned int aEntry\[\] = \{" + puts -nonewline " static const unsigned int aEntry\[\] = \{" set i 0 foreach range $lRange { foreach {iFirst nRange} $range {} @@ -349,7 +349,7 @@ proc print_isalnum {zFunc lRange} { return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 ); }else if( c<(1<<22) ){ unsigned int key = (((unsigned int)c)<<10) | 0x000003FF; - int iRes; + int iRes = 0; int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; int iLo = 0; while( iHi>=iLo ){ @@ -735,7 +735,7 @@ proc print_fileheader {} { if {$::generate_fts5_code} { puts "#if defined(SQLITE_ENABLE_FTS5)" } else { - puts "#if defined(SQLITE_ENABLE_FTS4_UNICODE61)" + puts "#ifndef SQLITE_DISABLE_FTS3_UNICODE" puts "#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)" } puts "" @@ -831,5 +831,5 @@ if {$generate_fts5_code} { puts "#endif /* defined(SQLITE_ENABLE_FTS5) */" } else { puts "#endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */" - puts "#endif /* !defined(SQLITE_ENABLE_FTS4_UNICODE61) */" + puts "#endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */" } diff --git a/ext/misc/eval.c b/ext/misc/eval.c new file mode 100644 index 0000000000..71b6b69f20 --- /dev/null +++ b/ext/misc/eval.c @@ -0,0 +1,122 @@ +/* +** 2014-11-10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements SQL function eval() which runs +** SQL statements recursively. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include + +/* +** Structure used to accumulate the output +*/ +struct EvalResult { + char *z; /* Accumulated output */ + const char *zSep; /* Separator */ + int szSep; /* Size of the separator string */ + sqlite3_int64 nAlloc; /* Number of bytes allocated for z[] */ + sqlite3_int64 nUsed; /* Number of bytes of z[] actually used */ +}; + +/* +** Callback from sqlite_exec() for the eval() function. +*/ +static int callback(void *pCtx, int argc, char **argv, char **colnames){ + struct EvalResult *p = (struct EvalResult*)pCtx; + int i; + for(i=0; inUsed+p->szSep+1 > p->nAlloc ){ + char *zNew; + p->nAlloc = p->nAlloc*2 + sz + p->szSep + 1; + /* Using sqlite3_realloc64() would be better, but it is a recent + ** addition and will cause a segfault if loaded by an older version + ** of SQLite. */ + zNew = p->nAlloc<=0x7fffffff ? sqlite3_realloc(p->z, (int)p->nAlloc) : 0; + if( zNew==0 ){ + sqlite3_free(p->z); + memset(p, 0, sizeof(*p)); + return 1; + } + p->z = zNew; + } + if( p->nUsed>0 ){ + memcpy(&p->z[p->nUsed], p->zSep, p->szSep); + p->nUsed += p->szSep; + } + memcpy(&p->z[p->nUsed], z, sz); + p->nUsed += sz; + } + return 0; +} + +/* +** Implementation of the eval(X) and eval(X,Y) SQL functions. +** +** Evaluate the SQL text in X. Return the results, using string +** Y as the separator. If Y is omitted, use a single space character. +*/ +static void sqlEvalFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zSql; + sqlite3 *db; + char *zErr = 0; + int rc; + struct EvalResult x; + + memset(&x, 0, sizeof(x)); + x.zSep = " "; + zSql = (const char*)sqlite3_value_text(argv[0]); + if( zSql==0 ) return; + if( argc>1 ){ + x.zSep = (const char*)sqlite3_value_text(argv[1]); + if( x.zSep==0 ) return; + } + x.szSep = (int)strlen(x.zSep); + db = sqlite3_context_db_handle(context); + rc = sqlite3_exec(db, zSql, callback, &x, &zErr); + if( rc!=SQLITE_OK ){ + sqlite3_result_error(context, zErr, -1); + sqlite3_free(zErr); + }else if( x.zSep==0 ){ + sqlite3_result_error_nomem(context); + sqlite3_free(x.z); + }else{ + sqlite3_result_text(context, x.z, (int)x.nUsed, sqlite3_free); + } +} + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_eval_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + rc = sqlite3_create_function(db, "eval", 1, SQLITE_UTF8, 0, + sqlEvalFunc, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "eval", 2, SQLITE_UTF8, 0, + sqlEvalFunc, 0, 0); + } + return rc; +} diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c index f80949bff2..fbe2d030c0 100644 --- a/ext/misc/fileio.c +++ b/ext/misc/fileio.c @@ -61,7 +61,6 @@ static void writefileFunc( ){ FILE *out; const char *z; - int n; sqlite3_int64 rc; const char *zFile; @@ -71,11 +70,9 @@ static void writefileFunc( if( out==0 ) return; z = (const char*)sqlite3_value_blob(argv[1]); if( z==0 ){ - n = 0; rc = 0; }else{ - n = sqlite3_value_bytes(argv[1]); - rc = fwrite(z, 1, n, out); + rc = fwrite(z, 1, sqlite3_value_bytes(argv[1]), out); } fclose(out); sqlite3_result_int64(context, rc); diff --git a/ext/misc/showauth.c b/ext/misc/showauth.c new file mode 100644 index 0000000000..87a9a6843c --- /dev/null +++ b/ext/misc/showauth.c @@ -0,0 +1,103 @@ +/* +** 2014-09-21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension adds a debug "authorizer" callback to the database +** connection. The callback merely writes the authorization request to +** standard output and returns SQLITE_OK. +** +** This extension can be used (for example) in the command-line shell to +** trace the operation of the authorizer. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include + +/* +** Display the authorization request +*/ +static int authCallback( + void *pClientData, + int op, + const char *z1, + const char *z2, + const char *z3, + const char *z4 +){ + const char *zOp; + char zOpSpace[50]; + switch( op ){ + case SQLITE_CREATE_INDEX: zOp = "CREATE_INDEX"; break; + case SQLITE_CREATE_TABLE: zOp = "CREATE_TABLE"; break; + case SQLITE_CREATE_TEMP_INDEX: zOp = "CREATE_TEMP_INDEX"; break; + case SQLITE_CREATE_TEMP_TABLE: zOp = "CREATE_TEMP_TABLE"; break; + case SQLITE_CREATE_TEMP_TRIGGER: zOp = "CREATE_TEMP_TRIGGER"; break; + case SQLITE_CREATE_TEMP_VIEW: zOp = "CREATE_TEMP_VIEW"; break; + case SQLITE_CREATE_TRIGGER: zOp = "CREATE_TRIGGER"; break; + case SQLITE_CREATE_VIEW: zOp = "CREATE_VIEW"; break; + case SQLITE_DELETE: zOp = "DELETE"; break; + case SQLITE_DROP_INDEX: zOp = "DROP_INDEX"; break; + case SQLITE_DROP_TABLE: zOp = "DROP_TABLE"; break; + case SQLITE_DROP_TEMP_INDEX: zOp = "DROP_TEMP_INDEX"; break; + case SQLITE_DROP_TEMP_TABLE: zOp = "DROP_TEMP_TABLE"; break; + case SQLITE_DROP_TEMP_TRIGGER: zOp = "DROP_TEMP_TRIGGER"; break; + case SQLITE_DROP_TEMP_VIEW: zOp = "DROP_TEMP_VIEW"; break; + case SQLITE_DROP_TRIGGER: zOp = "DROP_TRIGGER"; break; + case SQLITE_DROP_VIEW: zOp = "DROP_VIEW"; break; + case SQLITE_INSERT: zOp = "INSERT"; break; + case SQLITE_PRAGMA: zOp = "PRAGMA"; break; + case SQLITE_READ: zOp = "READ"; break; + case SQLITE_SELECT: zOp = "SELECT"; break; + case SQLITE_TRANSACTION: zOp = "TRANSACTION"; break; + case SQLITE_UPDATE: zOp = "UPDATE"; break; + case SQLITE_ATTACH: zOp = "ATTACH"; break; + case SQLITE_DETACH: zOp = "DETACH"; break; + case SQLITE_ALTER_TABLE: zOp = "ALTER_TABLE"; break; + case SQLITE_REINDEX: zOp = "REINDEX"; break; + case SQLITE_ANALYZE: zOp = "ANALYZE"; break; + case SQLITE_CREATE_VTABLE: zOp = "CREATE_VTABLE"; break; + case SQLITE_DROP_VTABLE: zOp = "DROP_VTABLE"; break; + case SQLITE_FUNCTION: zOp = "FUNCTION"; break; + case SQLITE_SAVEPOINT: zOp = "SAVEPOINT"; break; + case SQLITE_COPY: zOp = "COPY"; break; + case SQLITE_RECURSIVE: zOp = "RECURSIVE"; break; + + + default: { + sqlite3_snprintf(sizeof(zOpSpace), zOpSpace, "%d", op); + zOp = zOpSpace; + break; + } + } + if( z1==0 ) z1 = "NULL"; + if( z2==0 ) z2 = "NULL"; + if( z3==0 ) z3 = "NULL"; + if( z4==0 ) z4 = "NULL"; + printf("AUTH: %s,%s,%s,%s,%s\n", zOp, z1, z2, z3, z4); + return SQLITE_OK; +} + + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_showauth_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + rc = sqlite3_set_authorizer(db, authCallback, 0); + return rc; +} diff --git a/ext/misc/spellfix.c b/ext/misc/spellfix.c index a62d24c592..2a26e08391 100644 --- a/ext/misc/spellfix.c +++ b/ext/misc/spellfix.c @@ -1893,7 +1893,7 @@ static int spellfix1Init( char **pzErr ){ spellfix1_vtab *pNew = 0; - const char *zModule = argv[0]; + /* const char *zModule = argv[0]; // not used */ const char *zDbName = argv[1]; const char *zTableName = argv[2]; int nDbName; @@ -1947,7 +1947,7 @@ static int spellfix1Init( spellfix1DbExec(&rc, db, "CREATE INDEX IF NOT EXISTS \"%w\".\"%w_vocab_index_langid_k2\" " "ON \"%w_vocab\"(langid,k2);", - zDbName, zModule, zTableName + zDbName, zTableName, zTableName ); } for(i=3; rc==SQLITE_OK && izDbName, p->zTableName, - iRank, iLang, zWord, zK1, zK2 - ); + if( sqlite3_value_type(argv[1])==SQLITE_NULL ){ + spellfix1DbExec(&rc, db, + "INSERT INTO \"%w\".\"%w_vocab\"(rank,langid,word,k1,k2) " + "VALUES(%d,%d,%Q,%Q,%Q)", + p->zDbName, p->zTableName, + iRank, iLang, zWord, zK1, zK2 + ); + }else{ + newRowid = sqlite3_value_int64(argv[1]); + spellfix1DbExec(&rc, db, + "INSERT INTO \"%w\".\"%w_vocab\"(id,rank,langid,word,k1,k2) " + "VALUES(%lld,%d,%d,%Q,%Q,%Q)", + p->zDbName, p->zTableName, + newRowid, iRank, iLang, zWord, zK1, zK2 + ); + } *pRowid = sqlite3_last_insert_rowid(db); }else{ rowid = sqlite3_value_int64(argv[0]); diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index 7b540b4be1..8150538d45 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -1533,9 +1533,13 @@ static int rtreeFilter( rtreeReference(pRtree); + /* Reset the cursor to the same state as rtreeOpen() leaves it in. */ freeCursorConstraints(pCsr); - pCsr->iStrategy = idxNum; + sqlite3_free(pCsr->aPoint); + memset(pCsr, 0, sizeof(RtreeCursor)); + pCsr->base.pVtab = (sqlite3_vtab*)pRtree; + pCsr->iStrategy = idxNum; if( idxNum==1 ){ /* Special case - lookup by rowid. */ RtreeNode *pLeaf; /* Leaf on which the required cell resides */ diff --git a/ext/rtree/rtree1.test b/ext/rtree/rtree1.test index 9de5362781..0beb16cc94 100644 --- a/ext/rtree/rtree1.test +++ b/ext/rtree/rtree1.test @@ -33,6 +33,7 @@ set testprefix rtree1 # rtree-8.*: Test constrained scans of r-tree data. # # rtree-12.*: Test that on-conflict clauses are supported. +# rtree-13.*: Test that bug [d2889096e7bdeac6d] has been fixed. # ifcapable !rtree { @@ -513,4 +514,25 @@ foreach {tn sql_template testdata} { db close } } + +#------------------------------------------------------------------------- +# Test that bug [d2889096e7bdeac6d] has been fixed. +# +reset_db +do_execsql_test 13.1 { + CREATE VIRTUAL TABLE t9 USING rtree(id, xmin, xmax); + INSERT INTO t9 VALUES(1,0,0); + INSERT INTO t9 VALUES(2,0,0); + SELECT * FROM t9 WHERE id IN (1, 2); +} {1 0.0 0.0 2 0.0 0.0} + +do_execsql_test 13.2 { + WITH r(x) AS ( + SELECT 1 UNION ALL + SELECT 2 UNION ALL + SELECT 3 + ) + SELECT * FROM r CROSS JOIN t9 WHERE id=x; +} {1 1 0.0 0.0 2 2 0.0 0.0} + finish_test diff --git a/ext/rtree/rtree6.test b/ext/rtree/rtree6.test index cec3a8da41..c9c87e8ad9 100644 --- a/ext/rtree/rtree6.test +++ b/ext/rtree/rtree6.test @@ -102,7 +102,7 @@ do_eqp_test rtree6.2.4.2 { SELECT * FROM t1,t2 WHERE v=10 and x1<10 and x2>10 } { 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:C0E1} - 0 1 1 {SEARCH TABLE t2 USING AUTOMATIC COVERING INDEX (v=?)} + 0 1 1 {SEARCH TABLE t2 USING AUTOMATIC PARTIAL COVERING INDEX (v=?)} } do_eqp_test rtree6.2.5 { diff --git a/ext/rtree/rtreeF.test b/ext/rtree/rtreeF.test new file mode 100644 index 0000000000..c9620d34f7 --- /dev/null +++ b/ext/rtree/rtreeF.test @@ -0,0 +1,81 @@ +# 2014-08-21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file contains tests for the r-tree module. +# +# This file contains test cases for the ticket +# [369d57fb8e5ccdff06f197a37147a88f9de95cda] (2014-08-21) +# +# The following SQL causes an assertion fault while running +# sqlite3_prepare() on the DELETE statement: +# +# CREATE TABLE t1(x); +# CREATE TABLE t2(y); +# CREATE VIRTUAL TABLE t3 USING rtree(a,b,c); +# CREATE TRIGGER t2del AFTER DELETE ON t2 WHEN (SELECT 1 from t1) BEGIN +# DELETE FROM t3 WHERE a=old.y; +# END; +# DELETE FROM t2 WHERE y=1; +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl +ifcapable !rtree { finish_test ; return } + +do_execsql_test rtreeF-1.1 { + CREATE TABLE t1(x); + CREATE TABLE t2(y); + CREATE VIRTUAL TABLE t3 USING rtree(a,b,c); + CREATE TRIGGER t2dwl AFTER DELETE ON t2 WHEN (SELECT 1 from t1) BEGIN + DELETE FROM t3 WHERE a=old.y; + END; + + INSERT INTO t1(x) VALUES(999); + INSERT INTO t2(y) VALUES(1),(2),(3),(4),(5); + INSERT INTO t3(a,b,c) VALUES(1,2,3),(2,3,4),(3,4,5),(4,5,6),(5,6,7); + + SELECT a FROM t3 ORDER BY a; + SELECT '|'; + SELECT y FROM t2 ORDER BY y; +} {1 2 3 4 5 | 1 2 3 4 5} +do_execsql_test rtreeF-1.2 { + DELETE FROM t2 WHERE y=3; + + SELECT a FROM t3 ORDER BY a; + SELECT '|'; + SELECT y FROM t2 ORDER BY y; +} {1 2 4 5 | 1 2 4 5} +do_execsql_test rtreeF-1.3 { + DELETE FROM t1; + DELETE FROM t2 WHERE y=5; + + SELECT a FROM t3 ORDER BY a; + SELECT '|'; + SELECT y FROM t2 ORDER BY y; +} {1 2 4 5 | 1 2 4} +do_execsql_test rtreeF-1.4 { + INSERT INTO t1 DEFAULT VALUES; + DELETE FROM t2 WHERE y=5; + + SELECT a FROM t3 ORDER BY a; + SELECT '|'; + SELECT y FROM t2 ORDER BY y; +} {1 2 4 5 | 1 2 4} +do_execsql_test rtreeF-1.5 { + DELETE FROM t2 WHERE y=2; + + SELECT a FROM t3 ORDER BY a; + SELECT '|'; + SELECT y FROM t2 ORDER BY y; +} {1 4 5 | 1 4} + +finish_test diff --git a/ext/userauth/sqlite3userauth.h b/ext/userauth/sqlite3userauth.h new file mode 100644 index 0000000000..619477cac9 --- /dev/null +++ b/ext/userauth/sqlite3userauth.h @@ -0,0 +1,88 @@ +/* +** 2014-09-08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains the application interface definitions for the +** user-authentication extension feature. +** +** To compile with the user-authentication feature, append this file to +** end of an SQLite amalgamation header file ("sqlite3.h"), then add +** the SQLITE_USER_AUTHENTICATION compile-time option. See the +** user-auth.txt file in the same source directory as this file for +** additional information. +*/ +#ifdef SQLITE_USER_AUTHENTICATION + +/* +** If a database contains the SQLITE_USER table, then the +** sqlite3_user_authenticate() interface must be invoked with an +** appropriate username and password prior to enable read and write +** access to the database. +** +** Return SQLITE_OK on success or SQLITE_ERROR if the username/password +** combination is incorrect or unknown. +** +** If the SQLITE_USER table is not present in the database file, then +** this interface is a harmless no-op returnning SQLITE_OK. +*/ +int sqlite3_user_authenticate( + sqlite3 *db, /* The database connection */ + const char *zUsername, /* Username */ + const char *aPW, /* Password or credentials */ + int nPW /* Number of bytes in aPW[] */ +); + +/* +** The sqlite3_user_add() interface can be used (by an admin user only) +** to create a new user. When called on a no-authentication-required +** database, this routine converts the database into an authentication- +** required database, automatically makes the added user an +** administrator, and logs in the current connection as that user. +** The sqlite3_user_add() interface only works for the "main" database, not +** for any ATTACH-ed databases. Any call to sqlite3_user_add() by a +** non-admin user results in an error. +*/ +int sqlite3_user_add( + sqlite3 *db, /* Database connection */ + const char *zUsername, /* Username to be added */ + const char *aPW, /* Password or credentials */ + int nPW, /* Number of bytes in aPW[] */ + int isAdmin /* True to give new user admin privilege */ +); + +/* +** The sqlite3_user_change() interface can be used to change a users +** login credentials or admin privilege. Any user can change their own +** login credentials. Only an admin user can change another users login +** credentials or admin privilege setting. No user may change their own +** admin privilege setting. +*/ +int sqlite3_user_change( + sqlite3 *db, /* Database connection */ + const char *zUsername, /* Username to change */ + const char *aPW, /* New password or credentials */ + int nPW, /* Number of bytes in aPW[] */ + int isAdmin /* Modified admin privilege for the user */ +); + +/* +** The sqlite3_user_delete() interface can be used (by an admin user only) +** to delete a user. The currently logged-in user cannot be deleted, +** which guarantees that there is always an admin user and hence that +** the database cannot be converted into a no-authentication-required +** database. +*/ +int sqlite3_user_delete( + sqlite3 *db, /* Database connection */ + const char *zUsername /* Username to remove */ +); + +#endif /* SQLITE_USER_AUTHENTICATION */ diff --git a/ext/userauth/user-auth.txt b/ext/userauth/user-auth.txt new file mode 100644 index 0000000000..ba4eabc137 --- /dev/null +++ b/ext/userauth/user-auth.txt @@ -0,0 +1,164 @@ +Activate the user authentication logic by including the +ext/userauth/userauth.c source code file in the build and +adding the -DSQLITE_USER_AUTHENTICATION compile-time option. +The ext/userauth/sqlite3userauth.h header file is available to +applications to define the interface. + +When using the SQLite amalgamation, it is sufficient to append +the ext/userauth/userauth.c source file onto the end of the +amalgamation. + +The following new APIs are available when user authentication is +activated: + + int sqlite3_user_authenticate( + sqlite3 *db, /* The database connection */ + const char *zUsername, /* Username */ + const char *aPW, /* Password or credentials */ + int nPW /* Number of bytes in aPW[] */ + ); + + int sqlite3_user_add( + sqlite3 *db, /* Database connection */ + const char *zUsername, /* Username to be added */ + const char *aPW, /* Password or credentials */ + int nPW, /* Number of bytes in aPW[] */ + int isAdmin /* True to give new user admin privilege */ + ); + + int sqlite3_user_change( + sqlite3 *db, /* Database connection */ + const char *zUsername, /* Username to change */ + const void *aPW, /* Modified password or credentials */ + int nPW, /* Number of bytes in aPW[] */ + int isAdmin /* Modified admin privilege for the user */ + ); + + int sqlite3_user_delete( + sqlite3 *db, /* Database connection */ + const char *zUsername /* Username to remove */ + ); + +With this extension, a database can be marked as requiring authentication. +By default a database does not require authentication. + +The sqlite3_open(), sqlite3_open16(), and sqlite3_open_v2() interfaces +work as before: they open a new database connection. However, if the +database being opened requires authentication, then attempts to read +or write from the database will fail with an SQLITE_AUTH error until +after sqlite3_user_authenticate() has been called successfully. The +sqlite3_user_authenticate() call will return SQLITE_OK if the +authentication credentials are accepted and SQLITE_ERROR if not. + +Calling sqlite3_user_authenticate() on a no-authentication-required +database connection is a harmless no-op. + +If the database is encrypted, then sqlite3_key_v2() must be called first, +with the correct decryption key, prior to invoking sqlite3_user_authenticate(). + +To recapitulate: When opening an existing unencrypted authentication- +required database, the call sequence is: + + sqlite3_open_v2() + sqlite3_user_authenticate(); + /* Database is now usable */ + +To open an existing, encrypted, authentication-required database, the +call sequence is: + + sqlite3_open_v2(); + sqlite3_key_v2(); + sqlite3_user_authenticate(); + /* Database is now usable */ + +When opening a no-authentication-required database, the database +connection is treated as if it was authenticated as an admin user. + +When ATTACH-ing new database files to a connection, each newly attached +database that is an authentication-required database is checked using +the same username and password as supplied to the main database. If that +check fails, then the ATTACH command fails with an SQLITE_AUTH error. + +The sqlite3_user_add() interface can be used (by an admin user only) +to create a new user. When called on a no-authentication-required +database and when A is true, the sqlite3_user_add(D,U,P,N,A) routine +converts the database into an authentication-required database and +logs in the database connection D as user U with password P,N. +To convert a no-authentication-required database into an authentication- +required database, the isAdmin parameter must be true. If +sqlite3_user_add(D,U,P,N,A) is called on a no-authentication-required +database and A is false, then the call fails with an SQLITE_AUTH error. + +Any call to sqlite3_user_add() by a non-admin user results in an error. + +Hence, to create a new, unencrypted, authentication-required database, +the call sequence is: + + sqlite3_open_v2(); + sqlite3_user_add(); + +And to create a new, encrypted, authentication-required database, the call +sequence is: + + sqlite3_open_v2(); + sqlite3_key_v2(); + sqlite3_user_add(); + +The sqlite3_user_delete() interface can be used (by an admin user only) +to delete a user. The currently logged-in user cannot be deleted, +which guarantees that there is always an admin user and hence that +the database cannot be converted into a no-authentication-required +database. + +The sqlite3_user_change() interface can be used to change a users +login credentials or admin privilege. Any user can change their own +password. Only an admin user can change another users login +credentials or admin privilege setting. No user may change their own +admin privilege setting. + +The sqlite3_set_authorizer() callback is modified to take a 7th parameter +which is the username of the currently logged in user, or NULL for a +no-authentication-required database. + +----------------------------------------------------------------------------- +Implementation notes: + +An authentication-required database is identified by the presence of a +new table: + + CREATE TABLE sqlite_user( + uname TEXT PRIMARY KEY, + isAdmin BOOLEAN, + pw BLOB + ) WITHOUT ROWID; + +The sqlite_user table is inaccessible (unreadable and unwriteable) to +non-admin users and is read-only for admin users. However, if the same +database file is opened by a version of SQLite that omits +the -DSQLITE_USER_AUTHENTICATION compile-time option, then the sqlite_user +table will be readable by anybody and writeable by anybody if +the "PRAGMA writable_schema=ON" statement is run first. + +The sqlite_user.pw field is encoded by a built-in SQL function +"sqlite_crypt(X,Y)". The two arguments are both BLOBs. The first argument +is the plaintext password supplied to the sqlite3_user_authenticate() +interface. The second argument is the sqlite_user.pw value and is supplied +so that the function can extract the "salt" used by the password encoder. +The result of sqlite_crypt(X,Y) is another blob which is the value that +ends up being stored in sqlite_user.pw. To verify credentials X supplied +by the sqlite3_user_authenticate() routine, SQLite runs: + + sqlite_user.pw == sqlite_crypt(X, sqlite_user.pw) + +To compute an appropriate sqlite_user.pw value from a new or modified +password X, sqlite_crypt(X,NULL) is run. A new random salt is selected +when the second argument is NULL. + +The built-in version of of sqlite_crypt() uses a simple Ceasar-cypher +which prevents passwords from being revealed by searching the raw database +for ASCII text, but is otherwise trivally broken. For better password +security, the database should be encrypted using the SQLite Encryption +Extension or similar technology. Or, the application can use the +sqlite3_create_function() interface to provide an alternative +implementation of sqlite_crypt() that computes a stronger password hash, +perhaps using a cryptographic hash function like SHA1. diff --git a/ext/userauth/userauth.c b/ext/userauth/userauth.c new file mode 100644 index 0000000000..6ce99053d3 --- /dev/null +++ b/ext/userauth/userauth.c @@ -0,0 +1,355 @@ +/* +** 2014-09-08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains the bulk of the implementation of the +** user-authentication extension feature. Some parts of the user- +** authentication code are contained within the SQLite core (in the +** src/ subdirectory of the main source code tree) but those parts +** that could reasonable be separated out are moved into this file. +** +** To compile with the user-authentication feature, append this file to +** end of an SQLite amalgamation, then add the SQLITE_USER_AUTHENTICATION +** compile-time option. See the user-auth.txt file in the same source +** directory as this file for additional information. +*/ +#ifdef SQLITE_USER_AUTHENTICATION +#ifndef _SQLITEINT_H_ +# include "sqliteInt.h" +#endif + +/* +** Prepare an SQL statement for use by the user authentication logic. +** Return a pointer to the prepared statement on success. Return a +** NULL pointer if there is an error of any kind. +*/ +static sqlite3_stmt *sqlite3UserAuthPrepare( + sqlite3 *db, + const char *zFormat, + ... +){ + sqlite3_stmt *pStmt; + char *zSql; + int rc; + va_list ap; + int savedFlags = db->flags; + + va_start(ap, zFormat); + zSql = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + if( zSql==0 ) return 0; + db->flags |= SQLITE_WriteSchema; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + db->flags = savedFlags; + sqlite3_free(zSql); + if( rc ){ + sqlite3_finalize(pStmt); + pStmt = 0; + } + return pStmt; +} + +/* +** Check to see if the sqlite_user table exists in database zDb. +*/ +static int userTableExists(sqlite3 *db, const char *zDb){ + int rc; + sqlite3_mutex_enter(db->mutex); + sqlite3BtreeEnterAll(db); + if( db->init.busy==0 ){ + char *zErr = 0; + sqlite3Init(db, &zErr); + sqlite3DbFree(db, zErr); + } + rc = sqlite3FindTable(db, "sqlite_user", zDb)!=0; + sqlite3BtreeLeaveAll(db); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/* +** Check to see if database zDb has a "sqlite_user" table and if it does +** whether that table can authenticate zUser with nPw,zPw. Write one of +** the UAUTH_* user authorization level codes into *peAuth and return a +** result code. +*/ +static int userAuthCheckLogin( + sqlite3 *db, /* The database connection to check */ + const char *zDb, /* Name of specific database to check */ + u8 *peAuth /* OUT: One of UAUTH_* constants */ +){ + sqlite3_stmt *pStmt; + int rc; + + *peAuth = UAUTH_Unknown; + if( !userTableExists(db, "main") ){ + *peAuth = UAUTH_Admin; /* No sqlite_user table. Everybody is admin. */ + return SQLITE_OK; + } + if( db->auth.zAuthUser==0 ){ + *peAuth = UAUTH_Fail; + return SQLITE_OK; + } + pStmt = sqlite3UserAuthPrepare(db, + "SELECT pw=sqlite_crypt(?1,pw), isAdmin FROM \"%w\".sqlite_user" + " WHERE uname=?2", zDb); + if( pStmt==0 ) return SQLITE_NOMEM; + sqlite3_bind_blob(pStmt, 1, db->auth.zAuthPW, db->auth.nAuthPW,SQLITE_STATIC); + sqlite3_bind_text(pStmt, 2, db->auth.zAuthUser, -1, SQLITE_STATIC); + rc = sqlite3_step(pStmt); + if( rc==SQLITE_ROW && sqlite3_column_int(pStmt,0) ){ + *peAuth = sqlite3_column_int(pStmt, 1) + UAUTH_User; + }else{ + *peAuth = UAUTH_Fail; + } + return sqlite3_finalize(pStmt); +} +int sqlite3UserAuthCheckLogin( + sqlite3 *db, /* The database connection to check */ + const char *zDb, /* Name of specific database to check */ + u8 *peAuth /* OUT: One of UAUTH_* constants */ +){ + int rc; + u8 savedAuthLevel; + assert( zDb!=0 ); + assert( peAuth!=0 ); + savedAuthLevel = db->auth.authLevel; + db->auth.authLevel = UAUTH_Admin; + rc = userAuthCheckLogin(db, zDb, peAuth); + db->auth.authLevel = savedAuthLevel; + return rc; +} + +/* +** If the current authLevel is UAUTH_Unknown, the take actions to figure +** out what authLevel should be +*/ +void sqlite3UserAuthInit(sqlite3 *db){ + if( db->auth.authLevel==UAUTH_Unknown ){ + u8 authLevel = UAUTH_Fail; + sqlite3UserAuthCheckLogin(db, "main", &authLevel); + db->auth.authLevel = authLevel; + if( authLevelflags &= ~SQLITE_WriteSchema; + } +} + +/* +** Implementation of the sqlite_crypt(X,Y) function. +** +** If Y is NULL then generate a new hash for password X and return that +** hash. If Y is not null, then generate a hash for password X using the +** same salt as the previous hash Y and return the new hash. +*/ +void sqlite3CryptFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + const char *zIn; + int nIn, ii; + u8 *zOut; + char zSalt[8]; + zIn = sqlite3_value_blob(argv[0]); + nIn = sqlite3_value_bytes(argv[0]); + if( sqlite3_value_type(argv[1])==SQLITE_BLOB + && sqlite3_value_bytes(argv[1])==nIn+sizeof(zSalt) + ){ + memcpy(zSalt, sqlite3_value_blob(argv[1]), sizeof(zSalt)); + }else{ + sqlite3_randomness(sizeof(zSalt), zSalt); + } + zOut = sqlite3_malloc( nIn+sizeof(zSalt) ); + if( zOut==0 ){ + sqlite3_result_error_nomem(context); + }else{ + memcpy(zOut, zSalt, sizeof(zSalt)); + for(ii=0; iiauth.authLevel = UAUTH_Unknown; + sqlite3_free(db->auth.zAuthUser); + sqlite3_free(db->auth.zAuthPW); + memset(&db->auth, 0, sizeof(db->auth)); + db->auth.zAuthUser = sqlite3_mprintf("%s", zUsername); + if( db->auth.zAuthUser==0 ) return SQLITE_NOMEM; + db->auth.zAuthPW = sqlite3_malloc( nPW+1 ); + if( db->auth.zAuthPW==0 ) return SQLITE_NOMEM; + memcpy(db->auth.zAuthPW,zPW,nPW); + db->auth.nAuthPW = nPW; + rc = sqlite3UserAuthCheckLogin(db, "main", &authLevel); + db->auth.authLevel = authLevel; + sqlite3ExpirePreparedStatements(db); + if( rc ){ + return rc; /* OOM error, I/O error, etc. */ + } + if( authLevelauth.authLevelauth.zAuthUser==0 ){ + assert( isAdmin!=0 ); + sqlite3_user_authenticate(db, zUsername, aPW, nPW); + } + return SQLITE_OK; +} + +/* +** The sqlite3_user_change() interface can be used to change a users +** login credentials or admin privilege. Any user can change their own +** login credentials. Only an admin user can change another users login +** credentials or admin privilege setting. No user may change their own +** admin privilege setting. +*/ +int sqlite3_user_change( + sqlite3 *db, /* Database connection */ + const char *zUsername, /* Username to change */ + const char *aPW, /* Modified password or credentials */ + int nPW, /* Number of bytes in aPW[] */ + int isAdmin /* Modified admin privilege for the user */ +){ + sqlite3_stmt *pStmt; + int rc; + u8 authLevel; + + authLevel = db->auth.authLevel; + if( authLevelauth.zAuthUser, zUsername)!=0 ){ + if( db->auth.authLevelauth.authLevel = UAUTH_Admin; + if( !userTableExists(db, "main") ){ + /* This routine is a no-op if the user to be modified does not exist */ + }else{ + pStmt = sqlite3UserAuthPrepare(db, + "UPDATE sqlite_user SET isAdmin=%d, pw=sqlite_crypt(?1,NULL)" + " WHERE uname=%Q", isAdmin, zUsername); + if( pStmt==0 ){ + rc = SQLITE_NOMEM; + }else{ + sqlite3_bind_blob(pStmt, 1, aPW, nPW, SQLITE_STATIC); + sqlite3_step(pStmt); + rc = sqlite3_finalize(pStmt); + } + } + db->auth.authLevel = authLevel; + return rc; +} + +/* +** The sqlite3_user_delete() interface can be used (by an admin user only) +** to delete a user. The currently logged-in user cannot be deleted, +** which guarantees that there is always an admin user and hence that +** the database cannot be converted into a no-authentication-required +** database. +*/ +int sqlite3_user_delete( + sqlite3 *db, /* Database connection */ + const char *zUsername /* Username to remove */ +){ + sqlite3_stmt *pStmt; + if( db->auth.authLevelauth.zAuthUser, zUsername)==0 ){ + /* Cannot delete self */ + return SQLITE_AUTH; + } + if( !userTableExists(db, "main") ){ + /* This routine is a no-op if the user to be deleted does not exist */ + return SQLITE_OK; + } + pStmt = sqlite3UserAuthPrepare(db, + "DELETE FROM sqlite_user WHERE uname=%Q", zUsername); + if( pStmt==0 ) return SQLITE_NOMEM; + sqlite3_step(pStmt); + return sqlite3_finalize(pStmt); +} + +#endif /* SQLITE_USER_AUTHENTICATION */ diff --git a/main.mk b/main.mk index 7a26313b12..85255b8b97 100644 --- a/main.mk +++ b/main.mk @@ -46,7 +46,7 @@ # TCCX = $(TCC) $(OPTS) -I. -I$(TOP)/src -I$(TOP) TCCX += -I$(TOP)/ext/rtree -I$(TOP)/ext/icu -I$(TOP)/ext/fts3 -TCCX += -I$(TOP)/ext/async +TCCX += -I$(TOP)/ext/async -I$(TOP)/ext/userauth TCCX += -I$(TOP)/ext/fts5 # Object files for the SQLite library. @@ -67,8 +67,8 @@ LIBOBJ+= vdbe.o parse.o \ notify.o opcodes.o os.o os_unix.o os_win.o \ pager.o pcache.o pcache1.o pragma.o prepare.o printf.o \ random.o resolve.o rowset.o rtree.o select.o status.o \ - table.o tokenize.o trigger.o \ - update.o util.o vacuum.o \ + table.o threads.o tokenize.o trigger.o \ + update.o userauth.o util.o vacuum.o \ vdbeapi.o vdbeaux.o vdbeblob.o vdbemem.o vdbesort.o \ vdbetrace.o wal.o walker.o where.o utf.o vtab.o @@ -159,6 +159,7 @@ SRC = \ $(TOP)/src/sqliteLimit.h \ $(TOP)/src/table.c \ $(TOP)/src/tclsqlite.c \ + $(TOP)/src/threads.c \ $(TOP)/src/tokenize.c \ $(TOP)/src/trigger.c \ $(TOP)/src/utf.c \ @@ -226,7 +227,9 @@ SRC += \ $(TOP)/ext/rtree/sqlite3rtree.h \ $(TOP)/ext/rtree/rtree.h \ $(TOP)/ext/rtree/rtree.c - +SRC += \ + $(TOP)/ext/userauth/userauth.c \ + $(TOP)/ext/userauth/sqlite3userauth.h SRC += \ $(TOP)/ext/fts5/fts5.h \ $(TOP)/ext/fts5/fts5Int.h \ @@ -270,6 +273,7 @@ TESTSRC = \ $(TOP)/src/test_autoext.c \ $(TOP)/src/test_async.c \ $(TOP)/src/test_backup.c \ + $(TOP)/src/test_blob.c \ $(TOP)/src/test_btree.c \ $(TOP)/src/test_config.c \ $(TOP)/src/test_demovfs.c \ @@ -304,6 +308,7 @@ TESTSRC = \ TESTSRC += \ $(TOP)/ext/misc/amatch.c \ $(TOP)/ext/misc/closure.c \ + $(TOP)/ext/misc/eval.c \ $(TOP)/ext/misc/fileio.c \ $(TOP)/ext/misc/fuzzer.c \ $(TOP)/ext/misc/ieee754.c \ @@ -343,6 +348,7 @@ TESTSRC2 = \ $(TOP)/src/pcache.c \ $(TOP)/src/pcache1.c \ $(TOP)/src/select.c \ + $(TOP)/src/threads.c \ $(TOP)/src/tokenize.c \ $(TOP)/src/utf.c \ $(TOP)/src/util.c \ @@ -406,6 +412,8 @@ EXTHDR += \ EXTHDR += \ $(TOP)/ext/fts5/fts5Int.h \ $(TOP)/ext/fts5/fts5.h +EXTHDR += \ + $(TOP)/ext/userauth/sqlite3userauth.h # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. @@ -590,7 +598,6 @@ fts5.o: $(TOP)/ext/fts5/fts5.c $(HDR) $(EXTHDR) rtree.o: $(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c - # FTS5 things # fts5_aux.o: $(TOP)/ext/fts5/fts5_aux.c $(HDR) $(EXTHDR) @@ -628,6 +635,10 @@ fts5parse.c: $(TOP)/ext/fts5/fts5parse.y lemon cat fts5parse.c.orig | sed 's/yy/fts5yy/g' | sed 's/YY/fts5YY/g' > fts5parse.c +userauth.o: $(TOP)/ext/userauth/userauth.c $(HDR) $(EXTHDR) + $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/userauth/userauth.c + + # Rules for building test programs and for running tests # tclsqlite3: $(TOP)/src/tclsqlite.c libsqlite3.a @@ -685,9 +696,15 @@ test: testfixture$(EXE) sqlite3$(EXE) # threadtest runs a few thread-safety tests that are implemented in C. This # target is invoked by the releasetest.tcl script. # -threadtest3$(EXE): sqlite3.o $(TOP)/test/threadtest3.c $(TOP)/test/tt3_checkpoint.c - $(TCCX) -O2 sqlite3.o $(TOP)/test/threadtest3.c \ - -o threadtest3$(EXE) $(THREADLIB) +THREADTEST3_SRC = $(TOP)/test/threadtest3.c \ + $(TOP)/test/tt3_checkpoint.c \ + $(TOP)/test/tt3_index.c \ + $(TOP)/test/tt3_vacuum.c \ + $(TOP)/test/tt3_stress.c \ + $(TOP)/test/tt3_lookaside1.c + +threadtest3$(EXE): sqlite3.o $(THREADTEST3_SRC) + $(TCCX) $(TOP)/test/threadtest3.c sqlite3.o -o $@ $(THREADLIB) threadtest: threadtest3$(EXE) ./threadtest3$(EXE) @@ -699,9 +716,29 @@ $(TEST_EXTENSION): $(TOP)/src/test_loadext.c extensiontest: testfixture$(EXE) $(TEST_EXTENSION) ./testfixture$(EXE) $(TOP)/test/loadext.test -showdb$(EXE): $(TOP)/tool/showdb.c sqlite3.c +showdb$(EXE): $(TOP)/tool/showdb.c sqlite3.o $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o showdb$(EXE) \ - $(TOP)/tool/showdb.c sqlite3.c + $(TOP)/tool/showdb.c sqlite3.o $(THREADLIB) + +showstat4$(EXE): $(TOP)/tool/showstat4.c sqlite3.o + $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o showstat4$(EXE) \ + $(TOP)/tool/showstat4.c sqlite3.o $(THREADLIB) + +showjournal$(EXE): $(TOP)/tool/showjournal.c sqlite3.o + $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o showjournal$(EXE) \ + $(TOP)/tool/showjournal.c sqlite3.o $(THREADLIB) + +showwal$(EXE): $(TOP)/tool/showwal.c sqlite3.o + $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o showwal$(EXE) \ + $(TOP)/tool/showwal.c sqlite3.o $(THREADLIB) + +fts3view$(EXE): $(TOP)/ext/fts3/tool/fts3view.c sqlite3.o + $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o fts3view$(EXE) \ + $(TOP)/ext/fts3/tool/fts3view.c sqlite3.o $(THREADLIB) + +rollback-test$(EXE): $(TOP)/tool/rollback-test.c sqlite3.o + $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o rollback-test$(EXE) \ + $(TOP)/tool/rollback-test.c sqlite3.o $(THREADLIB) LogEst$(EXE): $(TOP)/tool/logest.c sqlite3.h $(TCC) -o LogEst$(EXE) $(TOP)/tool/logest.c @@ -749,10 +786,18 @@ clean: rm -f fts3-testfixture fts3-testfixture.exe rm -f testfixture testfixture.exe rm -f threadtest3 threadtest3.exe + rm -f LogEst LogEst.exe + rm -f fts3view fts3view.exe + rm -f rollback-test rollback-test.exe + rm -f showdb showdb.exe + rm -f showjournal showjournal.exe + rm -f showstat4 showstat4.exe + rm -f showwal showwal.exe + rm -f speedtest1 speedtest1.exe + rm -f wordcount wordcount.exe rm -f sqlite3.c sqlite3-*.c fts?amal.c tclsqlite3.c rm -f sqlite3rc.h rm -f shell.c sqlite3ext.h rm -f sqlite3_analyzer sqlite3_analyzer.exe sqlite3_analyzer.c rm -f sqlite-*-output.vsix rm -f mptester mptester.exe - rm -f showdb diff --git a/manifest b/manifest index 49fabbb73c..73bbea9d02 100644 --- a/manifest +++ b/manifest @@ -1,12 +1,12 @@ -C Add\sa\sversion\sof\sthe\sunicode61\stokenizer\sto\sfts5. -D 2015-01-01T16:46:10.851 +C Merge\slatest\strunk\schanges\swith\sthis\sbranch. +D 2015-01-01T18:03:49.016 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5 +F Makefile.in 7cd23e4fc91004a6bd081623e1bc6932e44828c0 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc b2d3ac211b479fa5a6c9aaf3b9c49fdea900d228 +F Makefile.msc b363b90fe1bfc3b87d190f2f728a126c00d9ce09 F Makefile.vxworks 034289efa9d591b04b1a73598623119c306cbba0 -F README.md 64f270c43c38c46de749e419c22f0ae2f4499fe8 -F VERSION 9f823c026c6a32fc5f84d212a8aae0a221dba45c +F README.md d58e3bebc0a4145e0f2a87994015fdb575a8e866 +F VERSION d846487aff892625eb8e75960234e7285f0462fe F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F addopcodes.awk 9eb448a552d5c0185cf62c463f9c173cedae3811 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 @@ -23,22 +23,22 @@ F autoconf/depcomp 0b26f101e3bc9fd1ff0be1da9fb4a82371142f92 x F autoconf/install-sh 06ee6336e63bb845c8439d777c32eb2eccc4fbf1 x F autoconf/ltmain.sh 7a658a24028f02331c1d2446562758083c5eadd1 F autoconf/missing d7c9981a81af13370d4ed152b24c0a82b7028585 x -F autoconf/tea/Makefile.in 5c3b0bdfb66c20d55ebff59d1718864461570ca9 +F autoconf/tea/Makefile.in d55bcc63832caf0309c2ff80358756116618cfca F autoconf/tea/README 3e9a3c060f29a44344ab50aec506f4db903fb873 F autoconf/tea/aclocal.m4 52c47aac44ce0ddb1f918b6993e8beb8eee88f43 -F autoconf/tea/configure.in e0466b881b53f31f5a4a69e7a91ad130902fb359 +F autoconf/tea/configure.in 93d43c79e936fb16556e22498177d7e8571efa04 F autoconf/tea/doc/sqlite3.n e1fe45d4f5286ee3d0ccc877aca2a0def488e9bb F autoconf/tea/license.terms 13bd403c9610fd2b76ece0ab50c4c5eda933d523 F autoconf/tea/pkgIndex.tcl.in 3ef61715cf1c7bdcff56947ffadb26bc991ca39d F autoconf/tea/tclconfig/install-sh bdd5e293591621ae60d9824d86a4b1c5f22c3d00 -F autoconf/tea/tclconfig/tcl.m4 f035b86539a5ab30689e997a11ae9e7fd2e65570 +F autoconf/tea/tclconfig/tcl.m4 66ddf0a5d5e4b1d29bff472c0985fd7fa89d0fb5 F autoconf/tea/win/makefile.vc f89d0184d0eee5f7e356ea407964dcd139939928 F autoconf/tea/win/nmakehlp.c 2070e086f39866b353a482d3a14dedaf26196506 F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977 F config.h.in 0921066a13130082764ab4ab6456f7b5bebe56de F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55 -F configure 5de98fac90452f876511fee8a35565a55dfb191f x +F configure 4343c810cc772571210af75d1a8f7c2eb711d75a x F configure.ac 4cf9f60785143fa141b10962ccc885d973792e9a F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/lemon.html 334dbf6621b8fb8790297ec1abf3cfa4621709d1 @@ -78,15 +78,15 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 20bc65862cfcea0a39bb64a819f8fe92a8e144c1 +F ext/fts3/fts3.c 8b6cceb3e0be22da26d83a3cec0e0e337e6b8ec6 F ext/fts3/fts3.h 62a77d880cf06a2865052726f8325c8fabcecad7 -F ext/fts3/fts3Int.h 16cddf2d7b0e5f3681615ae1d8ca0e45fca44918 +F ext/fts3/fts3Int.h 53d4eca1fb23eab00681fb028fb82eb5705c1e21 F ext/fts3/fts3_aux.c 5c211e17a64885faeb16b9ba7772f9d5445c2365 -F ext/fts3/fts3_expr.c 351395fad6fcb16ecfc61db0861008a70101330c +F ext/fts3/fts3_expr.c 40123785eaa3ebd4c45c9b23407cc44ac0c49905 F ext/fts3/fts3_hash.c 29b986e43f4e9dd40110eafa377dc0d63c422c60 F ext/fts3/fts3_hash.h 39cf6874dc239d6b4e30479b1975fe5b22a3caaf F ext/fts3/fts3_icu.c e319e108661147bcca8dd511cd562f33a1ba81b5 -F ext/fts3/fts3_porter.c 7f8b4bf5af7c0f20f73b8e87e14fa9298f52e290 +F ext/fts3/fts3_porter.c 3565faf04b626cddf85f03825e86056a4562c009 F ext/fts3/fts3_snippet.c 51beb5c1498176fd9caccaf1c75b55cb803a985a F ext/fts3/fts3_term.c a521f75132f9a495bdca1bdd45949b3191c52763 F ext/fts3/fts3_test.c 8a3a78c4458b2d7c631fcf4b152a5cd656fa7038 @@ -94,15 +94,15 @@ F ext/fts3/fts3_tokenize_vtab.c 011170fe9eba5ff062f1a31d3188e00267716706 F ext/fts3/fts3_tokenizer.c bbdc731bc91338050675c6d1da9ab82147391e16 F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 -F ext/fts3/fts3_unicode.c 92391b4b4fb043564c6539ea9b8661e3bcba47b9 -F ext/fts3/fts3_unicode2.c 0113d3acf13429e6dc38e0647d1bc71211c31a4d +F ext/fts3/fts3_unicode.c a93f5edc0aff44ef8b06d7cb55b52026541ca145 +F ext/fts3/fts3_unicode2.c c3d01968d497bd7001e7dc774ba75b372738c057 F ext/fts3/fts3_write.c 8260388626516a7005d06a9dce94f9e55c6c2a41 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 -F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197 +F ext/fts3/tool/fts3view.c 3986531f2fc0ceca0c89c31ec7d0589b6adb19d6 F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 -F ext/fts3/unicode/mkunicode.tcl 2fa92b916b17ee0fc94129d36969972d463bc016 +F ext/fts3/unicode/mkunicode.tcl 4199cb887040ee3c3cd59a5171ddb0566904586e F ext/fts5/extract_api_docs.tcl 6320db4a1d0722a4e2069e661381ad75e9889786 F ext/fts5/fts5.c 37e124e24e5860f9842e5f3ee22129a786c0fd74 F ext/fts5/fts5.h 4f9d2c477c0ee1907164642471329a82cb6b203b @@ -143,27 +143,29 @@ F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 F ext/misc/amatch.c 678056a4bfcd83c4e82dea81d37543cd1d6dbee1 F ext/misc/closure.c 636024302cde41b2bf0c542f81c40c624cfb7012 F ext/misc/compress.c 76e45655f4046e756064ab10c62e18f2eb846b9f -F ext/misc/fileio.c beea82bb5055b6590cffe2d2e6d922905894f691 +F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 +F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 136533c53cfce0957f0b48fa11dba27e21c5c01d F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc F ext/misc/rot13.c 1ac6f95f99b575907b9b09c81a349114cf9be45a -F ext/misc/spellfix.c 93f3961074cebe63c31fcefe62ca2a032ee8dfed +F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 +F ext/misc/spellfix.c 56739fab8c2ed6a9e2dac5592a88d281a999c43b F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512 F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95 F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c 6f70db93e0e42c369325c5cddcf2024c5a87ca43 +F ext/rtree/rtree.c 57bec53e1a677ab74217fe1f20a58c3a47261d6b F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e -F ext/rtree/rtree1.test e2da4aaa426918d27122d1a1066c6ecf8409a514 +F ext/rtree/rtree1.test 541bbcab74613907fea08b2ecdcdd5b7aa724cc9 F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba F ext/rtree/rtree3.test a494da55c30ee0bc9b01a91c80c81b387b22d2dc F ext/rtree/rtree4.test c8fe384f60ebd49540a5fecc990041bf452eb6e0 F ext/rtree/rtree5.test 6a510494f12454bf57ef28f45bc7764ea279431e -F ext/rtree/rtree6.test 0cfbdf27ee086bf16a3da2c6f2d5b3d6473cb27e +F ext/rtree/rtree6.test 773a90db2dce6a8353dd0d5b64bca69b29761196 F ext/rtree/rtree7.test 1fa710b9e6bf997a0c1a537b81be7bb6fded1971 F ext/rtree/rtree8.test db79c812f9e4a11f9b1f3f9934007884610a713a F ext/rtree/rtree9.test d86ebf08ff6328895613ed577dd8a2a37c472c34 @@ -172,15 +174,19 @@ F ext/rtree/rtreeB.test c85f9ce78766c4e68b8b89fbf2979ee9cfa82b4e F ext/rtree/rtreeC.test df158dcc81f1a43ce7eef361af03c48ec91f1e06 F ext/rtree/rtreeD.test 636630357638f5983701550b37f0f5867130d2ca F ext/rtree/rtreeE.test 388c1c8602c3ce55c15f03b509e9cf545fb7c41f +F ext/rtree/rtreeF.test 66deb9fd1611c7ca2e374adba63debdc2dbb12b4 F ext/rtree/rtree_perf.tcl 6c18c1f23cd48e0f948930c98dfdd37dfccb5195 F ext/rtree/rtree_util.tcl 06aab2ed5b826545bf215fff90ecb9255a8647ea F ext/rtree/sqlite3rtree.h 83349d519fe5f518b3ea025d18dd1fe51b1684bd F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 +F ext/userauth/sqlite3userauth.h 19cb6f0e31316d0ee4afdfb7a85ef9da3333a220 +F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 +F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 602303f3596d10237f25da030ee1d96065e2e5a8 +F main.mk c2e1899c04ef129d5d3c947b1b313f0c97358d72 F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea F mkopcodeh.awk c6b3fa301db6ef7ac916b14c60868aeaec1337b5 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 @@ -188,85 +194,85 @@ F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test cce8e306d8596d5a2e497e27112dae1f6e5e3538 F mptest/crash02.subtest f4ef05adcd15d60e5d2bd654204f2c008b519df8 -F mptest/mptest.c 499a74af4be293b7c1c7c3d40f332b67227dd739 +F mptest/mptest.c 24c5f72415df2eab7088ef8c9f99f163aed590c8 F mptest/multiwrite01.test 499ad0310da8dff8e8f98d2e272fc2a8aa741b2e F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b -F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 +F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a -F src/alter.c b00900877f766f116f9e16116f1ccacdc21d82f1 -F src/analyze.c e8c8a9d20beb2ad156321330e8f4fea002d8deee -F src/attach.c 3801129015ef59d76bf23c95ef9b0069d18a0c52 -F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 -F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 +F src/alter.c ba266a779bc7ce10e52e59e7d3dc79fa342e8fdb +F src/analyze.c 7a2986e6ea8247e5f21aca3d0b584598f58d84fe +F src/attach.c 7f6b3fafa2290b407e4a94dcf1afda7ec0fe394b +F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 +F src/backup.c 7ddee9c7d505e07e959a575b18498f17c71e53ea F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb -F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 70c60a3807b2072982f184d9614e020d2953f89c -F src/btree.h 4245a349bfe09611d7ff887dbc3a80cee8b7955a -F src/btreeInt.h cf180d86b2e9e418f638d65baa425c4c69c0e0e3 -F src/build.c 927e39b6aaf872c7b28f154f6acfeb9a05a72442 -F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 -F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac -F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a -F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 -F src/delete.c bcf8f72126cea80fc3d5bc5494cf19b3f8935aaf -F src/expr.c 4f9e497c66e2f25a4d139357a778c84d5713207c +F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 +F src/btree.c 4c098bb6e8678e4596983862abf78f7a0fcb807e +F src/btree.h 94277c1d30c0b75705974bcc8b0c05e79c03d474 +F src/btreeInt.h a3d0ae1d511365e1a2b76ad10960dbe55c286f34 +F src/build.c f5cfd7b32216f695b995bbc7c1a395f6d451d11f +F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 +F src/complete.c 198a0066ba60ab06fc00fba1998d870a4d575463 +F src/ctime.c df19848891c8a553c80e6f5a035e768280952d1a +F src/date.c 93594514aae68de117ca4a2a0d6cc63eddf26744 +F src/delete.c 0750b1eb4d96cd3fb2c798599a3a7c85e92f1417 +F src/expr.c 00da3072f362b06f39ce4052baa1d4ce2bb36d1c F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb -F src/fkey.c a549cff9fe8b736cdae21650ea0af6de29b77619 -F src/func.c 3bc223ea36cd29a91c481485343d0ee4257ab8dc -F src/global.c 1e4bd956dc2f608f87d2a929abc4a20db65f30e4 -F src/hash.c d139319967164f139c8d1bb8a11b14db9c4ba3cd -F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 +F src/fkey.c e0444b61bed271a76840cbe6182df93a9baa3f12 +F src/func.c 6d3c4ebd72aa7923ce9b110a7dc15f9b8c548430 +F src/global.c 6ded36dda9466fc1c9a3c5492ded81d79bf3977d +F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 +F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 -F src/insert.c a038daeadfb818aaadafa854f40f5623b6929d84 +F src/insert.c 5b9243a33726008cc4132897d2be371db12a13be F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d -F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 -F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b -F src/loadext.c 867c7b330b740c6c917af9956b13b81d0a048303 -F src/main.c afc0ae834a8abca9079908f6193b3886564164a0 -F src/malloc.c 0203ebce9152c6a0e5de520140b8ba65187350be +F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e +F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 +F src/loadext.c 86bd4e2fccd520b748cba52492ab60c4a770f660 +F src/main.c cb121abd60a4ec512ad3ad7273f649e691d252ef +F src/malloc.c 740db54387204c9a2eb67c6d98e68b08e9ef4eab F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 -F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b -F src/mem2.c dce31758da87ec2cfa52ba4c5df1aed6e07d8e8f +F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f +F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 F src/mem3.c 61c9d47b792908c532ca3a62b999cf21795c6534 -F src/mem5.c 74670012946c4adc8a6ad84d03acc80959c3e529 -F src/memjournal.c 0683aac6cab6ec2b5374c0db37c0deb2436a3785 -F src/mutex.c d3b66a569368015e0fcb1ac15f81c119f504d3bc -F src/mutex.h 5bc526e19dccc412b7ff04642f6fdad3fdfdabea -F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553 -F src/mutex_unix.c c3a4e00f96ba068a8dbef34084465979aaf369cc -F src/mutex_w32.c 9bcab6e699fdb20eae784237c36b299830fb3316 -F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 -F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace -F src/os.h 60d419395e32a8029fa380a80a3da2e9030f635e +F src/mem5.c 61eeb90134f9a5be6c2e68d8daae7628b25953fb +F src/memjournal.c 3eb2c0b51adbd869cb6a44780323f05fa904dc85 +F src/mutex.c 19bf9acba69ca2f367c3761080f8a9f0cf4670a8 +F src/mutex.h 779d588e3b7756ec3ecf7d78cde1d84aba414f85 +F src/mutex_noop.c f3f09fd7a2eb4287cfc799753ffc30380e7b71a1 +F src/mutex_unix.c 551e2f25f0fa0ee8fd7a43f50fc3d8be00e95dde +F src/mutex_w32.c df48fe07562a45c5c927c45b8d5873a27f98bbb0 +F src/notify.c 9711a7575036f0d3040ba61bc6e217f13a9888e7 +F src/os.c 8fd25588eeba74068d41102d26810e216999b6c8 +F src/os.h 3e57a24e2794a94d3cf2342c6d9a884888cd96bf F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa -F src/os_unix.c a7baf1b30f3c58ba20b813e01aab23b18ae44f85 -F src/os_win.c 8dbf6c11780fe2eb96c1f289e664d0c7b2911d37 -F src/os_win.h 057344a6720b4c8405d9bd98f58cb37a6ee46c25 -F src/pager.c f6bb1fa6cdf2062f2d8aec3e64db302bca519ab8 -F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 -F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 -F src/pcache.c d8eafac28290d4bb80332005435db44991d07fc2 -F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 -F src/pcache1.c 102e6f5a2fbc646154463eb856d1fd716867b64c -F src/pragma.c 810ef31ccfaa233201dcf100637a9777cc24e897 -F src/prepare.c 677521ab7132615a8a26107a1d1c3132f44ae337 -F src/printf.c af06f66927919730f03479fed6ae9854f73419f4 -F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece -F src/resolve.c 5fc110baeacf120a73fe34e103f052632ff11a02 -F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be -F src/select.c 6762c62e11b504aa014edceab8886495165e3a77 -F src/shell.c 058940e8a7198ea15ae90f60ffb0e2d67287b46c -F src/sqlite.h.in a98eb3e8c86c934ea6f5bcfc6b69653dde2f4ed4 -F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e -F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h fccdc735c27b3dc12322fec7cdad8bc76be8d00b +F src/os_unix.c 08c0346d2ea5e5ffd5b1a796f9becf1976d648d7 +F src/os_win.c 91d3d08e33ec0258d180d4c8255492f47d15e007 +F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 +F src/pager.c 4120a49ecd37697e28f5ed807f470b9c0b88410c +F src/pager.h c3476e7c89cdf1c6914e50a11f3714e30b4e0a77 +F src/parse.y 5dfead8aed90cb0c7c1115898ee2266804daff45 +F src/pcache.c b83d160ce81ca101f98f0d27498e6d6bd49f1599 +F src/pcache.h b44658c9c932d203510279439d891a2a83e12ba8 +F src/pcache1.c 1e77432b40b7d3288327d9cdf399dcdfd2b6d3bf +F src/pragma.c bd33aa24456f043bb6f6d32a918bbeed41d8c591 +F src/prepare.c 173a5a499138451b2561614ecb87d78f9f4644b9 +F src/printf.c 9e75a6a0b55bf61cfff7d7e19d89834a1b938236 +F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 +F src/resolve.c f6c46d3434439ab2084618d603e6d6dbeb0d6ada +F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e +F src/select.c f377fb8a5c73c10678ea74f3400f7913943e3d75 +F src/shell.c 45d9c9bd7cde07845af957f2d849933b990773cf +F src/sqlite.h.in 47cb601ed2b2ea7f01119e2763185c809d8e82fa +F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad +F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d +F src/sqliteInt.h d36da9a07130cae13cbfee0986bf20028cb01465 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d -F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 -F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e -F src/tclsqlite.c 3a274c56cfc66b1f957afef201547213fc2ccecc -F src/test1.c 3c8bc491d2f8de5adbbf306533cefc343c733927 +F src/status.c 81712116e826b0089bb221b018929536b2b5406f +F src/table.c e7a09215315a978057fb42c640f890160dbcc45e +F src/tclsqlite.c 36d0331f5084b3ffef003a5f98151088c1f72d8a +F src/test1.c 041c4edf2f9c49a329add297e26ee86a83852f51 F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -278,21 +284,22 @@ F src/test9.c bea1e8cf52aa93695487badedd6e1886c321ea60 F src/test_async.c 21e11293a2f72080eda70e1124e9102044531cd8 F src/test_autoext.c dea8a01a7153b9adc97bd26161e4226329546e12 F src/test_backup.c 3875e899222b651e18b662f86e0e50daa946344e +F src/test_blob.c 1f2e3e25255b731c4fcf15ee7990d06347cb6c09 F src/test_btree.c 2e9978eca99a9a4bfa8cae949efb00886860a64f -F src/test_config.c 9acba5c44c1562159104096e6e2ed5d293d4b86d +F src/test_config.c c07a375b1eb4e56ed9fc70bc210649c8ff01718d F src/test_demovfs.c 69b2085076654ebc18014cbc6386f04409c959a9 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c ced436e3d4b8e4681328409b8081051ce614e28f -F src/test_func.c d3013ce36f19ac72a99c73864930fd1fa41832f8 +F src/test_func.c 14e543ae4d905ee31dc322b2f8d31bfac1769d45 F src/test_hexio.c abfdecb6fa58c354623978efceb088ca18e379cd F src/test_init.c 66b33120ffe9cd853b5a905ec850d51151337b32 -F src/test_intarray.c 87847c71c3c36889c0bcc9c4baf9d31881665d61 -F src/test_intarray.h 2ece66438cfd177b78d1bfda7a4180cd3a10844d +F src/test_intarray.c 6c610a21ab8edde85a3a2c7f2b069244ecf4d834 +F src/test_intarray.h 9dc57417fb65bc7835cc18548852cc08cc062202 F src/test_journal.c f5c0a05b7b3d5930db769b5ee6c3766dc2221a64 F src/test_loadext.c a5251f956ab6af21e138dc1f9c0399394a510cb4 -F src/test_malloc.c 1ff5b1243d96124c9a180f3b89424820a1f337f3 -F src/test_multiplex.c 9f304bf04170c91c0318238d512df2da039eb1c8 -F src/test_multiplex.h 110a8c4d356e0aa464ca8730375608a9a0b61ae1 +F src/test_malloc.c ba34143f941a9d74b30bbffc8818389bb73a1ca2 +F src/test_multiplex.c caadb62cc777268b4f8fb94d5b27b80156c8f7c0 +F src/test_multiplex.h c08e4e8f8651f0c5e0509b138ff4d5b43ed1f5d3 F src/test_mutex.c 293042d623ebba969160f471a82aa1551626454f F src/test_onefile.c 0396f220561f3b4eedc450cef26d40c593c69a25 F src/test_osinst.c 3d0340bc31a9f3d8a3547e0272373e80f78dde25 @@ -300,7 +307,7 @@ F src/test_pcache.c a5cd24730cb43c5b18629043314548c9169abb00 F src/test_quota.c 65f6348fec0f2b3020c907247fb47556b214abb9 F src/test_quota.h 2a8ad1952d1d2ca9af0ce0465e56e6c023b5e15d F src/test_rtree.c fdd8d29ca5165c7857987a2ba263fac5c69e231f -F src/test_schema.c cd12a2223c3a394f4d07bb93bdf6d344c5c121b6 +F src/test_schema.c 2bdba21b82f601da69793e1f1d11bf481a79b091 F src/test_server.c a2615049954cbb9cfb4a62e18e2f0616e4dc38fe F src/test_sqllog.c c1c1bbedbcaf82b93d83e4f9dd990e62476a680e F src/test_stat.c 9898687a6c2beca733b0dd6fe19163d987826d31 @@ -311,48 +318,52 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c f84075a388527892ff184988f43b69ce69b8083c F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/tokenize.c 6da2de6e12218ccb0aea5184b56727d011f4bee7 -F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb -F src/update.c 01564b3c430f6c7b0a35afaf7aba7987206fa3a5 -F src/utf.c a0314e637768a030e6e84a957d0c4f6ba910cc05 -F src/util.c 049fe1d3c0e2209c1bee107aec2fcff6285f909f -F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 9bfe6becfc094382ae213656fbe511055ad83a54 -F src/vdbe.h 394464909ed682334aa3d5831aae0c2fe2abef94 -F src/vdbeInt.h e6d83e5bfd62fc6685ba1ed6153f7099f82de9f7 -F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 -F src/vdbeaux.c e493f38758c4b8f4ca2007cf6a700bd405d192f3 -F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac -F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 44441d73b08b3a638dcdb725afffb87c6574ad27 -F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 -F src/vtab.c b05a86bed6ee2c7f79edb2cb57b951b220867caf -F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a +F src/threads.c 6de09362b657f19ba83e5fa521ee715787ce9fee +F src/tokenize.c e00458c9938072b0ea711c850b8dcf4ddcb5fe18 +F src/trigger.c 25571661fdeae8c7f975ff40ffec205520a3f92f +F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 +F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c +F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 +F src/vacuum.c 9b30ec729337dd012ed88d4c292922c8ef9cf00c +F src/vdbe.c 1a9e671c9cfc259e4d2affc71f7df4a4c00a842c +F src/vdbe.h 6fc69d9c5e146302c56e163cb4b31d1ee64a18c3 +F src/vdbeInt.h 9bb69ff2447c34b6ccc58b34ec35b615f86ead78 +F src/vdbeapi.c 4bc511a46b9839392ae0e90844a71dc96d9dbd71 +F src/vdbeaux.c 07ef87c6d4b5abdf13ff33babb10205702fdab0a +F src/vdbeblob.c 4af4bfb71f6df7778397b4a0ebc1879793276778 +F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f +F src/vdbesort.c 80e40d889ebb536cb7a5ac4c12fa2a4662bc9181 +F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 +F src/vtab.c ec672a0b5d507f24b9294c0696dbd2fd2ce9b122 +F src/wal.c 85353539f2d9d0c91ebd057c32525b1e1aa3335e F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 -F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 7b9e13cff91a2f14ac61e6a1bc3a83b5113e6298 -F src/whereInt.h 929c1349b5355fd44f22cee5c14d72b3329c58a6 +F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 +F src/where.c d46de821bc604a4fd36fa3928c086950e91aafb1 +F src/whereInt.h d3633e9b592103241b74b0ec76185f3e5b8b62e0 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 -F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 +F test/aggnested.test b35b4cd69fc913f90d39a575e171e1116c3a4bb7 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 F test/all.test 6ff7b43c2b4b905c74dc4a813d201d0fa64c5783 F test/alter.test 547dc2d292644301ac9a7dda22b319b74f9c08d2 F test/alter2.test 7ea05c7d92ac99349a802ef7ada17294dd647060 F test/alter3.test 49c9d9fba2b8fcdce2dedeca97bbf1f369cc548d -F test/alter4.test d6c011fa0d6227abba762498cafbb607c9609e93 +F test/alter4.test c461150723ac957f3b2214aa0b11552cd72023ec F test/altermalloc.test e81ac9657ed25c6c5bb09bebfa5a047cd8e4acfc F test/amatch1.test b5ae7065f042b7f4c1c922933f4700add50cdb9f F test/analyze.test 1772936d66471c65221e437b6d1999c3a03166c4 F test/analyze3.test bf41f0f680dd1e0d44eed5e769531e93a5320275 F test/analyze4.test eff2df19b8dd84529966420f29ea52edc6b56213 F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4 -F test/analyze6.test d31defa011a561b938b4608d3538c1b4e0b5e92c +F test/analyze6.test f1c552ce39cca4ec922a7e4e0e5d0203d6b3281f F test/analyze7.test bb1409afc9e8629e414387ef048b8e0e3e0bdc4f -F test/analyze8.test 093d15c1c888eed5034304a98c992f7360130b88 -F test/analyze9.test 623e02a99a78fa12fe5def2fd559032d5d887e0f -F test/analyzeA.test 1a5c40079894847976d983ca39c707aaa44b6944 +F test/analyze8.test c05a461d0a6b05991106467d0c47480f2e709c82 +F test/analyze9.test 72795c8113604b5dcd47a1498a61d6d7fb5d041a +F test/analyzeA.test 3335697f6700c7052295cfd0067fc5b2aacddf9a F test/analyzeB.test 8bf35ee0a548aea831bf56762cb8e7fdb1db083d +F test/analyzeC.test 555a6cc388b9818b6eda6df816f01ce0a75d3a93 +F test/analyzeD.test 08f9d0bee4e118a66fff3a32d02dbe0ee0a2b594 +F test/analyzeE.test 8684e8ac5722fb97c251887ad97e5d496a98af1d F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b F test/async2.test c0a9bd20816d7d6a2ceca7b8c03d3d69c28ffb8b F test/async3.test d73a062002376d7edc1fe3edff493edbec1fc2f7 @@ -364,20 +375,22 @@ F test/attach2.test 0ec5defa340363de6cd50fd595046465e9aaba2d F test/attach3.test 359eb65d00102cdfcef6fa4e81dc1648f8f80b27 F test/attach4.test 53bf502f17647c6d6c5add46dda6bac8b6f4665c F test/attachmalloc.test 3a4bfca9545bfe906a8d2e622de10fbac5b711b0 -F test/auth.test 5bdf154eb28c0e4bbc0473f335858c0d96171768 -F test/auth2.test c3b415b76c033bedb81292118fb7c01f5f10cbcd -F test/auth3.test a4755e6a2a2fea547ffe63c874eb569e60a28eb5 +F test/auth.test 855233ef26eb3601b6886567ea4e326c72959360 +F test/auth2.test 264c6af53cad9aba5218c68bbe18036e39007bfa +F test/auth3.test 5cfa94ed90c6617c42b7ba4b133fd79678b251c7 F test/autoinc.test c58912526998a39e11f66b533e23cfabea7f25b7 -F test/autoindex1.test 762ff3f8e25d852aae55c6462ca166a80c0cde61 -F test/autoindex2.test 60d2fc6f38364308ce73a9beb01b47ded38697de -F test/autoindex3.test 8254f689c3241081fad52b7bea18ba53e07e14a2 +F test/autoindex1.test 6ff78b94f43a59616c06c11c55b12935173506d7 +F test/autoindex2.test af7e595c6864cc6ef5fc38d5db579a3e34940cb8 +F test/autoindex3.test a3be0d1a53a7d2edff208a5e442312957047e972 +F test/autoindex4.test fc807f9efd158bec60f5dfdf34ebe46fb274612d F test/autovacuum.test 941892505d2c0f410a0cb5970dfa1c7c4e5f6e74 F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4 F test/avtrans.test 0252654f4295ddda3b2cce0e894812259e655a85 F test/backcompat.test 19a1f337c68419b020a7481dd272a472c4ad8ef4 -F test/backup.test c9cdd23a495864b9edf75a9fa66f5cb7e10fcf62 +F test/backup.test b79299a536a4c6d919094786595b95be56d02014 F test/backup2.test 34986ef926ea522911a51dfdb2f8e99b7b75ebcf F test/backup4.test 2a2e4a64388090b152de753fd9e123f28f6a3bd4 +F test/backup5.test ee5da6d7fe5082f5b9b0bbfa31d016f52412a2e4 F test/backup_ioerr.test 4c3c7147cee85b024ecf6e150e090c32fdbb5135 F test/backup_malloc.test 7162d604ec2b4683c4b3799a48657fb8b5e2d450 F test/badutf.test d5360fc31f643d37a973ab0d8b4fb85799c3169f @@ -387,6 +400,7 @@ F test/between.test 34d375fb5ce1ae283ffe82b6b233e9f38e84fc6c F test/bigfile.test aa74f4e5db51c8e54a1d9de9fa65d01d1eb20b59 F test/bigfile2.test 1b489a3a39ae90c7f027b79110d6b4e1dbc71bfc F test/bigrow.test f0aeb7573dcb8caaafea76454be3ade29b7fc747 +F test/bigsort.test 8299fa9298f4f1e02fc7d2712e8b77d6cd60e5a2 F test/bind.test 3c7b320969000c441a70952b0b15938fbb66237c F test/bindxfer.test efecd12c580c14df5f4ad3b3e83c667744a4f7e0 F test/bitvec.test 75894a880520164d73b1305c1c3f96882615e142 @@ -399,14 +413,15 @@ F test/boundary3.tcl 23361e108a125dca9c4080c2feb884fe54d69243 F test/boundary3.test 56ef82096b4329aca2be74fa1e2b0f762ea0eb45 F test/boundary4.tcl 0bb4b1a94f4fc5ae59b79b9a2b7a140c405e2983 F test/boundary4.test 89e02fa66397b8a325d5eb102b5806f961f8ec4b +F test/btree01.test e08b3613540145b353f20c81cb18ead54ff12e0f F test/btreefault.test c2bcb542685eea44621275cfedbd8a13f65201e3 F test/busy.test 76b4887f8b9160ba903c1ac22e8ff406ad6ae2f0 F test/cache.test 13bc046b26210471ca6f2889aceb1ea52dc717de F test/capi2.test 011c16da245fdc0106a2785035de6b242c05e738 -F test/capi3.test 71bcf2fbd36a9732f617766dfd752552c8e491b5 +F test/capi3.test f0718f4f90d0efdc980119bfbdf1d7f1541ee5ef F test/capi3b.test efb2b9cfd127efa84433cd7a2d72ce0454ae0dc4 -F test/capi3c.test a21869e4d50d5dbb7e566e328fc0bc7c2efa6a32 -F test/capi3d.test 6d0fc0a86d73f42dd19a7d8b7761ab9bc02277d0 +F test/capi3c.test fdc0d67a2cb8e8fc400d5b7735e330161ea057a2 +F test/capi3d.test a82b6321c50a1cfc848e386fa2c851893606f68c F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 F test/check.test 5831ddb6f2c687782eaf2e1a07b6e17f24c4f763 @@ -423,7 +438,7 @@ F test/collate7.test 8ec29d98f3ee4ccebce6e16ce3863fb6b8c7b868 F test/collate8.test df26649cfcbddf109c04122b340301616d3a88f6 F test/collate9.test 3adcc799229545940df2f25308dd1ad65869145a F test/collateA.test b8218ab90d1fa5c59dcf156efabb1b2599c580d6 -F test/colmeta.test 087c42997754b8c648819832241daf724f813322 +F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1 F test/colname.test 08948a4809d22817e0e5de89c7c0a8bd90cb551b F test/conflict.test 841bcf7cabbfca39c577eb8411ea8601843b46a8 F test/conflict2.test 0d3af4fb534fa1bd020c79960bb56e4d52655f09 @@ -445,8 +460,8 @@ F test/corruptD.test b3c205fac7952b1de645ce44bb02335cd9e3e040 F test/corruptE.test 193b4ca4e927e77c1d5f4f56203ddc998432a7ee F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804 -F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb -F test/corruptI.test b3e4203d420490fc3d3062711597bc1dea06a789 +F test/corruptH.test 5dd4fa98c6c1ed33b178f9e8a48c4fdd3cfc9067 +F test/corruptI.test 221ad8b7f0a9ac6b80fc577e73b5ad8cdea31243 F test/cost.test 19d314526616ce4473eb4e4e450fcb94499ce318 F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 @@ -465,7 +480,7 @@ F test/ctime.test 7bd009071e242aac4f18521581536b652b789a47 F test/date.test 42973251b9429f2c41b77eb98a7b0b0ba2d3b2c0 F test/dbstatus.test 8de104bb5606f19537d23cd553b41349b5ab1204 F test/dbstatus2.test 10418e62b3db5dca070f0c3eef3ea13946f339c2 -F test/default.test 792c3c70836f1901e2a8cb34fa0880ed71e2c1a9 +F test/default.test 0cb49b1c315a0d81c81d775e407f66906a2a604d F test/delete.test a065b05d2ebf60fd16639c579a4adfb7c381c701 F test/delete2.test 3a03f2cca1f9a67ec469915cb8babd6485db43fa F test/delete3.test 555e84a00a99230b7d049d477a324a631126a6ab @@ -473,13 +488,18 @@ F test/descidx1.test 6d03b44c8538fe0eb4924e19fba10cdd8f3c9240 F test/descidx2.test 9f1a0c83fd57f8667c82310ca21b30a350888b5d F test/descidx3.test 09ddbe3f5295f482d2f8b687cf6db8bad7acd9a2 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e -F test/distinct.test 086e70c765f172e8974e9f83b9ac5ca03c154e77 +F test/distinct.test 175d49ee783febaf368192dfe7f5afbc68910230 F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 -F test/e_createtable.test ed82efcedc4b3656b27a5fcd12335cdb7e20eeee +F test/e_blobbytes.test 9bea1d3e2b20f3010b04abba58f6ba172301f49f +F test/e_blobclose.test df756753f571bc30e42e3a6cba2807576e49e716 +F test/e_blobopen.test 234f960d90235a9b51ec3ca1e062e8541dd558d8 +F test/e_blobwrite.test 615b405f29feb2cfb5a1f03dab7933258294fa26 +F test/e_changes.test fd66105385153dbf21fdb35eb8ef6c3e1eade579 +F test/e_createtable.test c7e67b49e6cf92473c8fb30ab26143e9e2128cf7 F test/e_delete.test d5186e2f5478b659f16a2c8b66c09892823e542a F test/e_droptrigger.test 3cd080807622c13e5bbb61fc9a57bd7754da2412 F test/e_dropview.test 0c9f7f60989164a70a67a9d9c26d1083bc808306 -F test/e_expr.test 5c71d183fbf519a4769fd2e2124afdc70b5b1f42 +F test/e_expr.test 8f5fdd7261e2d746813b0c6a1c0e34824ad3c5ad F test/e_fkey.test a1783fe1f759e1990e6a11adfcf0702dac4d0707 F test/e_fts3.test 5c02288842e4f941896fd44afdef564dd5fc1459 F test/e_insert.test 7b2fa9cd1456f83474d6c5d27db3abaeb8be2023 @@ -487,32 +507,38 @@ F test/e_reindex.test 396b7b4f0a66863b4e95116a67d93b227193e589 F test/e_resolve.test dcce9308fb13b934ce29591105d031d3e14fbba6 F test/e_select.test 52692ff3849541e828ad4661fe3773a9b8711763 F test/e_select2.test aceb80ab927d46fba5ce7586ebabf23e2bb0604f +F test/e_totalchanges.test b12ee5809d3e63aeb83238dd501a7bca7fd72c10 F test/e_update.test 312cb8f5ccfe41515a6bb092f8ea562a9bd54d52 -F test/e_uri.test a2c92d80093a7efdcfbb11093651cbea87097b6b +F test/e_uri.test 5ae33760fb2039c61aa2d90886f1664664173585 F test/e_vacuum.test 5bfbdc21b65c0abf24398d0ba31dc88d93ca77a9 +F test/e_wal.test 0967f0b8f1dfda871dc7b9b5574198f1f4f7d69a +F test/e_walauto.test d2dfc6681aade1f1306a8a336f5a258d8b62becd +F test/e_walckpt.test 65e29b6631e51f210f83e4ff11571e647ba93608 +F test/e_walhook.test da3ea8b3483d1af72190337bda50155a91a4b664 F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea F test/enc2.test 83437a79ba1545a55fb549309175c683fb334473 F test/enc3.test 90683ad0e6ea587b9d5542ca93568af9a9858c40 F test/enc4.test c8f1ce3618508fd0909945beb8b8831feef2c020 F test/eqp.test 85873fa5816c48915c82c4e74cb5c35a5b48160f F test/errmsg.test f31592a594b44ee121371d25ddd5d63497bb3401 -F test/eval.test bc269c365ba877554948441e91ad5373f9f91be3 +F test/eval.test a64c9105d6ff163df7cf09d6ac29cdad5922078c F test/exclusive.test c7ebbc756eacf544c108b15eed64d7d4e5f86b75 F test/exclusive2.test 32798111aae78a5deec980eee383213f189df308 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 F test/exists.test 8f7b27b61c2fbe5822f0a1f899c715d14e416e30 -F test/expr.test 67c9fd6f8f829e239dc8b0f4a08a73c08b09196d +F test/expr.test c4b9bf0cc60b26862475e19999fbd2609ca8259c F test/extension01.test 00d13cec817f331a687a243e0e5a2d87b0e358c9 F test/fallocate.test 3e979af17dfa7e5e9dda5eba1a696c04fa9d47f7 F test/filectrl.test 14fa712e42c4cb791e09dfd58a6a03efb47ef13a F test/filefmt.test cb34663f126cbc2d358af552dcaf5c72769b0146 F test/fkey1.test e1d1fa84cde579185ea01358436839703e415a5b -F test/fkey2.test 32ca728bcb854feed72d1406ea375fe423eebff2 +F test/fkey2.test 1db212cda86b0d3ce72714001f7b6381c321341c F test/fkey3.test 76d475c80b84ee7a5d062e56ccb6ea68882e2b49 F test/fkey4.test 86446017011273aad8f9a99c1a65019e7bd9ca9d -F test/fkey5.test 8a1fde4e7721ae00b05b3178888833726ca2df8d +F test/fkey5.test 488601fbda8350619b3029487e56830447056fd2 F test/fkey6.test abb59f866c1b44926fd02d1fdd217d831fe04f48 -F test/fkey7.test e31d0e71a41c1d29349a16448d6c420e2c53a8fc +F test/fkey7.test 72e915890ee4a005daaf3002cb208e8fe973ac13 +F test/fkey8.test 8f08203458321e6c19a263829de4cfc936274ab0 F test/fkey_malloc.test 594a7ea1fbab553c036c70813cd8bd9407d63749 F test/format4.test 1f0cac8ff3895e9359ed87e41aaabee982a812eb F test/fts-9fd058691.test 78b887e30ae6816df0e1fed6259de4b5a64ad33c @@ -587,13 +613,13 @@ F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851 F test/fts3expr.test 3401d47b229c4504424caf362cc4ff704cad4162 F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a F test/fts3expr3.test 9e91b8edbcb197bf2e92161aa7696446d96dce5f -F test/fts3expr4.test 0713d94ab951ed88a8c3629a4889a48c55c4067c +F test/fts3expr4.test e1be1248566f43c252d4404d52914f1fc4bfa065 F test/fts3fault.test cb72dccb0a3b9f730f16c5240f3fcb9303eb1660 F test/fts3fault2.test 3198eef2804deea7cac8403e771d9cbcb752d887 F test/fts3first.test dbdedd20914c8d539aa3206c9b34a23775644641 F test/fts3join.test 53e66a0c21eb568580674a43b21c059acb26f499 F test/fts3malloc.test b0e4c133b8d61d4f6d112d8110f8320e9e453ef6 -F test/fts3matchinfo.test ff423e73faab8fc6d7adeefedf74dd8e2b0b14e0 +F test/fts3matchinfo.test 58544fa4d254000fa4e7f494b0a832f7ba61d45e F test/fts3near.test 7e3354d46f155a822b59c0e957fd2a70c1d7e905 F test/fts3prefix.test b36d4f00b128a51e7b386cc013a874246d9d7dc1 F test/fts3prefix2.test e1f0a822ca661dced7f12ce392e14eaf65609dce @@ -605,7 +631,7 @@ F test/fts3sort.test ed34c716a11cc2009a35210e84ad5f9c102362ca F test/fts3tok1.test c551043de056b0b1582a54e878991f57bad074bc F test/fts3tok_err.test 52273cd193b9036282f7bacb43da78c6be87418d F test/fts3varint.test 752c08ed5d32c5d7dc211b056f4ed68a76b7e36e -F test/fts4aa.test 0c3152322c7f0b548cc942ad763eaba0da87ccca +F test/fts4aa.test 10aac8e9d62c7357590acfabe3fad01e9a9ce1cb F test/fts4check.test 74d77f6cdb768ac49df5afda575cef14ae3d239a F test/fts4content.test 2e7252557d6d24afa101d9ba1de710d6140e6d06 F test/fts4docid.test e33c383cfbdff0284685604d256f347a18fdbf01 @@ -623,7 +649,7 @@ F test/fts5_common.tcl 2488117cd80b7a4de7c20054b89f082b77b4189c F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d F test/func.test ae97561957aba6ca9e3a7b8a13aac41830d701ef F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f -F test/func3.test e82d16b13739f1a0c5efb20048583c79b767b625 +F test/func3.test d202a7606d23f90988a664e88e268aed1087c11c F test/func4.test 6beacdfcb0e18c358e6c2dcacf1b65d1fa80955f F test/func5.test cdd224400bc3e48d891827cc913a57051a426fa4 F test/fuzz-oss1.test 4912e528ec9cf2f42134456933659d371c9e0d74 @@ -635,18 +661,19 @@ F test/fuzz_malloc.test 328f70aaca63adf29b4c6f06505ed0cf57ca7c26 F test/fuzzer1.test d4c52aaf3ef923da293a2653cfab33d02f718a36 F test/fuzzerfault.test 8792cd77fd5bce765b05d0c8e01b9edcf8af8536 F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 +F test/hexlit.test f9ecde8145bfc2341573473256c74ae37a200497 F test/hook.test 162d7cef7a2d2b04839fe14402934e6a1b79442f F test/icu.test 70df4faca133254c042d02ae342c0a141f2663f4 F test/in.test 047c4671328e9032ab95666a67021adbbd36e98e F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0 -F test/in4.test 41c1c031aa46b1eb4411df2687ed2ed498da23b5 -F test/in5.test 99f9a40af01711b06d2d614ecfe96129f334fba3 +F test/in4.test d2b38cba404bc4320f4fe1b595b3d163f212c068 +F test/in5.test 1de657472fa9ac2924be25c2c959ac5ca1aae554 F test/incrblob.test e81846d214f3637622620fbde7cd526781cfe328 F test/incrblob2.test bf4d549aa4a466d7fbe3e3a3693d3861263d5600 F test/incrblob3.test d8d036fde015d4a159cd3cbae9d29003b37227a4 F test/incrblob4.test f26502a5697893e5acea268c910f16478c2f0fab -F test/incrblob_err.test d2562d2771ebffd4b3af89ef64c140dd44371597 +F test/incrblob_err.test af1f12ba60d220c9752073ff2bda2ad59e88960d F test/incrblobfault.test 280474078f6da9e732cd2a215d3d854969014b6e F test/incrvacuum.test d2a6ddf5e429720b5fe502766af747915ccf6c32 F test/incrvacuum2.test 379eeb8740b0ef60c372c439ad4cbea20b34bb9b @@ -656,9 +683,9 @@ F test/index.test 4d990005a67a36984e4f1a5f1bdccea8d08da4ee F test/index2.test ee83c6b5e3173a3d7137140d945d9a5d4fdfb9d6 F test/index3.test 55a90cff99834305e8141df7afaef39674b57062 F test/index4.test ab92e736d5946840236cd61ac3191f91a7856bf6 -F test/index5.test fc07c14193c0430814e7a08b5da46888ee795c33 +F test/index5.test 25b0b451aceed4ac5f7d49f856f6de7257470b3e F test/index6.test fb370966ac3cd0989053dd5385757b5c3e24ab6a -F test/index7.test a3baf9a625bda7fd49471e99aeae04095fbfeecf +F test/index7.test 917cf1e1c7439bb155abbeabec511b28945e157b F test/indexedby.test b2f22f3e693a53813aa3f50b812eb609ba6df1ec F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 @@ -673,16 +700,16 @@ F test/interrupt.test dfe9a67a94b0b2d8f70545ba1a6cca10780d71cc F test/intpkey.test 7506090fc08e028712a8bf47e5f54111947e3844 F test/io.test 3a7abcef18727cc0f2399e04b0e8903eccae50f8 F test/ioerr.test 2a24bd6ed5a8b062e64bfe1f6cf94fb25e92210d -F test/ioerr2.test 9d71166f8466eda510f1af6137bdabaa82b5408d +F test/ioerr2.test 2593563599e2cc6b6b4fcf5878b177bdd5d8df26 F test/ioerr3.test d3cec5e1a11ad6d27527d0d38573fbff14c71bdd F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b -F test/join.test 559b81eb56ae350246f9c25986aa52c81c725c7e +F test/join.test 52d4d49f86d0cf46926672878c4eaf0da399104a F test/join2.test f2171c265e57ee298a27e57e7051d22962f9f324 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 -F test/join5.test 86675fc2919269aa923c84dd00ee4249b97990fe +F test/join5.test 5df23eba184f159ed9705a954957e765a10c141d F test/join6.test cfe6503791ceb0cbb509966740286ec423cbf10b F test/journal1.test 69abc726c51b4a0409189f9a85191205297c0577 F test/journal2.test ae06f566c28552c313ded3fee79a6c69e6d049b1 @@ -702,14 +729,14 @@ F test/lock.test 87af515b0c4cf928576d0f89946d67d7c265dfb4 F test/lock2.test 5242d8ac4e2d59c403aebff606af449b455aceff F test/lock3.test f271375930711ae044080f4fe6d6eda930870d00 F test/lock4.test e175ae13865bc87680607563bafba21f31a26f12 -F test/lock5.test 5ad6a1f536036ff1be915cfdd41481aeafda3273 +F test/lock5.test c6c5e0ebcb21c61a572870cc86c0cb9f14cede38 F test/lock6.test ad5b387a3a8096afd3c68a55b9535056431b0cf5 F test/lock7.test 49f1eaff1cdc491cc5dee3669f3c671d9f172431 F test/lock_common.tcl 0c270b121d40959fa2f3add382200c27045b3d95 F test/lookaside.test 93f07bac140c5bb1d49f3892d2684decafdc7af2 F test/main.test 39c4bb8a157f57298ed1659d6df89d9f35aaf2c8 F test/make-where7.tcl 05c16b5d4f5d6512881dfec560cb793915932ef9 -F test/malloc.test fd368e31fe98d4779ed80442f311ed9f03bcd1f7 +F test/malloc.test 96939d2d1a6f39667bbebe5bc27c6525f2ab614e F test/malloc3.test e3b32c724b5a124b57cb0ed177f675249ad0c66a F test/malloc4.test 957337613002b7058a85116493a262f679f3a261 F test/malloc5.test fafce0aa9157060445cd1a56ad50fc79d82f28c3 @@ -717,7 +744,7 @@ F test/malloc6.test 2f039d9821927eacae43e1831f815e157659a151 F test/malloc7.test 7c68a32942858bc715284856c5507446bba88c3a F test/malloc8.test 9b7a3f8cb9cf0b12fff566e80a980b1767bd961d F test/malloc9.test 2307c6ee3703b0a21391f3ea92388b4b73f9105e -F test/mallocA.test 1ba0367fb5434e7bc2fa4afcb30b14174d91b160 +F test/mallocA.test 672cd7dedb63771bade3a6f557f851a4ad161d4a F test/mallocAll.test 98f1be74bc9f49a858bc4f361fc58e26486798be F test/mallocB.test bc475ab850cda896142ab935bbfbc74c24e51ed6 F test/mallocC.test 3dffe16532f109293ce1ccecd0c31dca55ef08c4 @@ -728,18 +755,19 @@ F test/mallocG.test 0ff91b65c50bdaba680fb75d87fe4ad35bb7934f F test/mallocH.test 79b65aed612c9b3ed2dcdaa727c85895fd1bfbdb F test/mallocI.test a88c2b9627c8506bf4703d8397420043a786cdb6 F test/mallocJ.test b5d1839da331d96223e5f458856f8ffe1366f62e -F test/mallocK.test d79968641d1b70d88f6c01bdb9a7eb4a55582cc9 -F test/malloc_common.tcl dc07e2bbbfc3f8416f4cbddcc139d77eb360d9af +F test/mallocK.test 3cff7c0f64735f6883bacdd294e45a6ed5714817 +F test/mallocL.test 252ddc7eb4fbf75364eab17b938816085ff1fc17 +F test/malloc_common.tcl a644f12e2da20ddfabb8bd077ec610a44113450e F test/manydb.test 28385ae2087967aa05c38624cec7d96ec74feb3e F test/mem5.test c6460fba403c5703141348cd90de1c294188c68f F test/memdb.test fcb5297b321b562084fc79d64d5a12a1cd2b639b F test/memleak.test 10b9c6c57e19fc68c32941495e9ba1c50123f6e2 -F test/memsubsys1.test f97cfd0b30e85c2f1ed16d642e7ac58006be84b2 +F test/memsubsys1.test 690d142525a7d31efb638b0d232e25fac3afeb1a F test/memsubsys2.test 3a1c1a9de48e5726faa85108b02459fae8cb9ee9 F test/minmax.test 42fbad0e81afaa6e0de41c960329f2b2c3526efd F test/minmax2.test b44bae787fc7b227597b01b0ca5575c7cb54d3bc F test/minmax3.test cc1e8b010136db0d01a6f2a29ba5a9f321034354 -F test/minmax4.test 536a3360470633a177e42fbc19660d146b51daef +F test/minmax4.test 936941484ebdceb8adec7c86b6cd9b6e5e897c1f F test/misc1.test 1201a037c24f982cc0e956cdaa34fcaf6439c417 F test/misc2.test 00d7de54eda90e237fc9a38b9e5ccc769ebf6d4d F test/misc3.test cf3dda47d5dda3e53fc5804a100d3c82be736c9d @@ -747,26 +775,28 @@ F test/misc4.test 9c078510fbfff05a9869a0b6d8b86a623ad2c4f6 F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test edd0b63e2ee29a256900b0514f6fff27e19e9bb2 +F test/misc8.test fc2754d38892f7dac30c22db3616c2764f117d66 F test/misuse.test 3c34719944ba045cc6c188a4852ba04680728912 -F test/mmap1.test 93d167b328255cbe6679fe1e1a23be1b1197d07b +F test/mmap1.test 1bfd611b9841eafb44f7d83c0788e146d84a33c9 F test/mmap2.test 9d6dd9ddb4ad2379f29cc78f38ce1e63ed418022 F test/mmap3.test c92273e16eb8d23c1d55c9815b446bb72ef0512e F test/mmapfault.test d4c9eff9cd8c2dc14bc43e71e042f175b0a26fe3 -F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 +F test/multiplex.test efd015ca0b5b4a57dc9535b8feb1273eebeadb60 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a F test/multiplex3.test d228f59eac91839a977eac19f21d053f03e4d101 +F test/multiplex4.test d3e8a5a522c51cbf3ed1c5b0bd496be02c29d7b1 F test/mutex1.test 78b2b9bb320e51d156c4efdb71b99b051e7a4b41 F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660 F test/nan.test e9648b9d007c7045242af35e11a984d4b169443a F test/nolock.test 0540dd96f39b8876e3ffdd8814fad0ea425efeee F test/notify1.test 669b2b743618efdc18ca4b02f45423d5d2304abf -F test/notify2.test ce23eb522c9e1fff6443f96376fe67872202061c +F test/notify2.test 2ecabaa1305083856b7c39cf32816b612740c161 F test/notify3.test 10ff25cde502e72a92053a2f215d64bece4ef934 F test/notnull.test f8fcf58669ddba79274daa2770d61dfad8274f62 F test/null.test a8b09b8ed87852742343b33441a9240022108993 F test/numcast.test 5d126f7f581432e86a90d1e35cac625164aec4a1 F test/openv2.test 0d3040974bf402e19b7df4b783e447289d7ab394 -F test/orderby1.test 12426f99518cde45f34215ca6a0ebc0e9bc5c77a +F test/orderby1.test eb246e377612b21a418fbea57047ba8ea88aaa6b F test/orderby2.test bc11009f7cd99d96b1b11e57b199b00633eb5b04 F test/orderby3.test 8619d06a3debdcd80a27c0fdea5c40b468854b99 F test/orderby4.test 4d39bfbaaa3ae64d026ca2ff166353d2edca4ba4 @@ -774,6 +804,7 @@ F test/orderby5.test 8f08a54836d21fb7c70245360751aedd1c2286fb F test/orderby6.test 8b38138ab0972588240b3fca0985d2e400432859 F test/orderby7.test 3d1383d52ade5b9eb3a173b3147fdd296f0202da F test/oserror.test 50417780d0e0d7cd23cf12a8277bb44024765df3 +F test/ovfl.test 4f7ca651cba5c059a12d8c67dddd49bec5747799 F test/pager1.test 1acbdb14c5952a72dd43129cabdbf69aaa3ed1fa F test/pager2.test 67b8f40ae98112bcdba1f2b2d03ea83266418c71 F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f @@ -785,12 +816,13 @@ F test/pageropt.test 6b8f6a123a5572c195ad4ae40f2987007923bbd6 F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 -F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54 -F test/permutations.test f26508e576234a5875041dd1632c5eed9fc495d9 -F test/pragma.test adb21a90875bc54a880fa939c4d7c46598905aa0 +F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff +F test/permutations.test b18313202c6a706cc25feaf0726ff333694cc9e1 +F test/pragma.test aa16dedfe01c02c8895169012f7dfde9c163f0d5 F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 +F test/pragma3.test 6f849ccffeee7e496d2f2b5e74152306c0b8757c F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 -F test/printf2.test bed79b4c3e5da08ba88ad637c0bf62586843cfb1 +F test/printf2.test b4acd4bf8734243257f01ddefa17c4fb090acc8a F test/progress.test a282973d1d17f08071bc58a77d6b80f2a81c354d F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc F test/queryonly.test 5f653159e0f552f0552d43259890c1089391dcca @@ -804,21 +836,23 @@ F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rdonly.test dd30a4858d8e0fbad2304c2bd74a33d4df36412a F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8 F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 -F test/releasetest.mk 2eced2f9ae701fd0a29e714a241760503ccba25a -F test/releasetest.tcl 06d289d8255794073a58d2850742f627924545ce +F test/releasetest.tcl 9440b21870bc25c9303e7a2adc0708257249295a F test/resolver01.test 33abf37ff8335e6bf98f2b45a0af3e06996ccd9a -F test/rollback.test e9504a009a202c3ed711da2e6879ff60c5a4669c +F test/rollback.test 458fe73eb3ffdfdf9f6ba3e9b7350a6220414dea +F test/rollback2.test fc14cf6d1a2b250d2735ef16124b971bce152f14 +F test/rollbackfault.test 6a004f71087cc399296cffbb5429ea6da655ae65 F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 -F test/rowid.test b78b30afb9537a73788ca1233a23a32190a3bb1f +F test/rowid.test 742b5741584a8a44fd83e856cc2896688401d645 F test/rtree.test 0c8d9dd458d6824e59683c19ab2ffa9ef946f798 F test/run-wordcount.sh 891e89c4c2d16e629cd45951d4ed899ad12afc09 -F test/savepoint.test 6c53f76dffe5df0dd87646efe3e7aa159c36e07b +F test/savepoint.test c671fdbd34cd3bfe1518a777526ada595180cf8d F test/savepoint2.test 9b8543940572a2f01a18298c3135ad0c9f4f67d7 F test/savepoint3.test e328085853b14898d78ceea00dfe7db18bb6a9ec F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 -F test/savepoint7.test fbf319a7b2dda089ec5be30a424a0e95f121d423 +F test/savepoint7.test db3db281486c925095f305aad09fe806e5188ff3 +F test/scanstatus.test 5253c219e331318a437f436268e0e82345700285 F test/schema.test 8f7999be894260f151adf15c2c7540f1c6d6a481 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38 @@ -851,22 +885,31 @@ F test/shared7.test a81e99f83e6c51b02ac99c96fb3a2a7b5978c956 F test/shared8.test 00a07bf5e1337ecf72e94542bdefdc330d7a2538 F test/shared9.test 5f2a8f79b4d6c7d107a01ffa1ed05ae7e6333e21 F test/sharedA.test 0cdf1a76dfa00e6beee66af5b534b1e8df2720f5 -F test/shared_err.test 0079c05c97d88cfa03989b7c20a8b266983087aa +F test/sharedB.test 16cc7178e20965d75278f410943109b77b2e645e +F test/shared_err.test 2f2aee20db294b9924e81f6ccbe60f19e21e8506 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 -F test/shell1.test fb080d67c81e8a80a79ea04b36f127209b5bd112 -F test/shell2.test c57da3a381c099b02c813ba156298d5c2f5c93a3 +F test/shell1.test ab6025d941f9c84c5b83412c6b4d8b57f78dfa3a +F test/shell2.test 12b8bf901b0e3a8ac58cf5c0c63a0a388d4d1862 F test/shell3.test 5e8545ec72c4413a0e8d4c6be56496e3c257ca29 F test/shell4.test 8a9c08976291e6c6c808b4d718f4a8b299f339f5 -F test/shell5.test ef0c52952a4a96dc1d9ec3b1fa81ec897ca48154 +F test/shell5.test 15a419cc1df21c892ed64f5596ae7a501f2816f2 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5 F test/shrink.test 8c70f62b6e8eb4d54533de6d65bd06b1b9a17868 F test/sidedelete.test f0ad71abe6233e3b153100f3b8d679b19a488329 -F test/skipscan1.test 28c7faa41a0d7265040ecb0a0abd90c0904270b2 +F test/skipscan1.test 2ddfe5d168462170c4487f534e2a99fb006b2076 F test/skipscan2.test d1d1450952b7275f0b0a3a981f0230532743951a +F test/skipscan3.test ec5bab3f81c7038b43450e7b3062e04a198bdbb5 +F test/skipscan5.test 67817a4b6857c47e0e33ba3e506da6f23ef68de2 +F test/skipscan6.test 5866039d03a56f5bd0b3d172a012074a1d90a15b F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 -F test/sort.test 0e4456e729e5a92a625907c63dcdedfbe72c5dc5 +F test/sort.test c4400e7533748f6bd7413851ff148645e82b9e2d +F test/sort2.test 84a92eebf697feee9c6697746918c7d67373eea1 +F test/sort3.test 6178ade30810ac9166fcdf14b7065e49c0f534e2 +F test/sort4.test 6c37d85f7cd28d50cce222fcab84ccd771e105cb +F test/sort5.test a448240a42b49239edc00f85d6d7ac7a1b261e1f +F test/sortfault.test b8e35177f97438b930ee87c9419ca2599e8073e1 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb F test/speed1p.test b180e98609c7677382cf618c0ec9b69f789033a8 @@ -875,27 +918,27 @@ F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b -F test/speedtest1.c d29c8048beb7ea9254191f3fde9414709166a920 -F test/spellfix.test 61309f5efbec53603b3f86457d34a504f80abafe -F test/sqllimits1.test b1aae27cc98eceb845e7f7adf918561256e31298 +F test/speedtest1.c e4e2aa37ff66bad9f414a50a8cb9edfaac65c9e5 +F test/spellfix.test 24f676831acddd2f4056a598fd731a72c6311f49 +F test/sqllimits1.test 9014524e7ab16e2a4976b13397db4c29cc29c6d9 F test/stat.test 76fd746b85459e812a0193410fb599f0531f22de F test/stmt.test 25d64e3dbf9a3ce89558667d7f39d966fe2a71b9 F test/subquery.test 666fdecceac258f5fd84bed09a64e49d9f37edd9 -F test/subquery2.test 91e1e364072aeff431d1f9689b15147e421d88c7 +F test/subquery2.test 438f8a7da1457277b22e4176510f7659b286995f F test/subselect.test d24fd8757daf97dafd2e889c73ea4c4272dcf4e4 F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a F test/superlock.test 1cde669f68d2dd37d6c9bd35eee1d95491ae3fc2 F test/sync.test a34cd43e98b7fb84eabbf38f7ed8f7349b3f3d85 F test/syscall.test d2fdaad713f103ac611fe7ef9b724c7b69f8149c F test/sysfault.test fa776e60bf46bdd3ae69f0b73e46ee3977a58ae6 -F test/table.test 580d23530187026d4502fae74a490f0408cf2cc7 +F test/table.test 06271d61eb13871490d38168433c1ef3dd82bb2a F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 F test/tclsqlite.test 37a61c2da7e3bfe3b8c1a2867199f6b860df5d43 F test/tempdb.test 19d0f66e2e3eeffd68661a11c83ba5e6ace9128c F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1 -F test/tester.tcl 7d2c97b43e51abde7a35f9d3bc57e886c8503e55 +F test/tester.tcl a9cb43af36b13ec12587e3579bc13eda98cfb6b2 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -906,7 +949,8 @@ F test/thread2.test f35d2106452b77523b3a2b7d1dcde2e5ee8f9e46 F test/thread_common.tcl 334639cadcb9f912bf82aa73f49efd5282e6cadd F test/threadtest1.c 6029d9c5567db28e6dc908a0c63099c3ba6c383b F test/threadtest2.c ace893054fa134af3fc8d6e7cfecddb8e3acefb9 -F test/threadtest3.c 0ed13e09690f6204d7455fac3b0e8ece490f6eef +F test/threadtest3.c 9ab4b168681c3a6f70f6c833ba08e0d48dd4af9b +F test/threadtest4.c c1e67136ceb6c7ec8184e56ac61db28f96bd2925 F test/tkt-02a8e81d44.test 6c80d9c7514e2a42d4918bf87bf6bc54f379110c F test/tkt-26ff0c2d1e.test 888324e751512972c6e0d1a09df740d8f5aaf660 F test/tkt-2a5629202f.test 0521bd25658428baa26665aa53ffed9367d33af2 @@ -933,12 +977,13 @@ F test/tkt-78e04e52ea.test 813779f8888f3ca226df656c4eef078f9635f3c9 F test/tkt-7a31705a7e6.test e75a2bba4eec801b92c8040eb22096ac6d35e844 F test/tkt-7bbfb7d442.test 7b2cd79c7a17ae6750e75ec1a7846712a69c9d18 F test/tkt-80ba201079.test 105a721e6aad0ae3c5946d7615d1e4d03f6145b8 -F test/tkt-80e031a00f.test 9a154173461a4dbe2de49cda73963e04842d52f7 +F test/tkt-80e031a00f.test f50046f474ecf67ad5c50cd9200da04ff887d7cd F test/tkt-8454a207b9.test c583a9f814a82a2b5ba95207f55001c9f0cd816c F test/tkt-868145d012.test a5f941107ece6a64410ca4755c6329b7eb57a356 F test/tkt-8c63ff0ec.test 258b7fc8d7e4e1cb5362c7d65c143528b9c4cbed F test/tkt-91e2e8ba6f.test 08c4f94ae07696b05c9b822da0b4e5337a2f54c5 F test/tkt-94c04eaadb.test f738c57c7f68ab8be1c054415af7774617cb6223 +F test/tkt-9a8b09f8e6.test b2ef151d0984b2ebf237760dbeaa50724e5a0667 F test/tkt-9d68c883.test 458f7d82a523d7644b54b497c986378a7d8c8b67 F test/tkt-9f2eb3abac.test 85bc63e749f050e6a61c8f9207f1eee65c9d3395 F test/tkt-a7b7803e.test 159ef554234fa1f9fb318c751b284bd1cf858da4 @@ -947,6 +992,7 @@ F test/tkt-b1d3a2e531.test 8f7576e41ca179289ee1a8fee28386fd8e4b0550 F test/tkt-b351d95f9.test d14a503c414c5c58fdde3e80f9a3cfef986498c0 F test/tkt-b72787b1.test a95e8cdad0b98af1853ac7f0afd4ab27b77bf5f3 F test/tkt-b75a9ca6b0.test 97cc2d5eeaf82799eb42138c0a1ff64370238ce4 +F test/tkt-ba7cbfaedc.test e76d88e572e489ee0d64fe4caf4af18b3d1dc688 F test/tkt-bd484a090c.test 60460bf946f79a79712b71f202eda501ca99b898 F test/tkt-bdc6bbbb38.test fc38bb09bdd440e3513a1f5f98fc60a075182d7d F test/tkt-c48d99d690.test ba61977d62ab612fc515b3c488a6fbd6464a2447 @@ -956,7 +1002,7 @@ F test/tkt-d635236375.test 9d37e988b47d87505bc9445be0ca447002df5d09 F test/tkt-d82e3f3721.test bcc0dfba658d15bab30fd4a9320c9e35d214ce30 F test/tkt-f3e5abed55.test d5a0126118142d13e27f6ce9f4c47096e9321c00 F test/tkt-f67b41381a.test a23bc124c981662db712167bacd0ed8ad11abac9 -F test/tkt-f777251dc7a.test af6531446c64bfd268416f07b4df7be7f9c749d2 +F test/tkt-f777251dc7a.test d1a8fc3eefb7a9e64d19ff24d5c8c94c34a632fb F test/tkt-f7b4edec.test d998a08ff2b18b7f62edce8e3044317c45efe6c7 F test/tkt-f973c7ac31.test 28ef85c7f015477916795246d8286aeda39d4ead F test/tkt-fa7bf5ec.test 9102dfea58aa371d78969da735f9392c57e2e035 @@ -1047,11 +1093,11 @@ F test/tkt3997.test a335fa41ca3985660a139df7b734a26ef53284bd F test/tkt4018.test 7c2c9ba4df489c676a0a7a0e809a1fb9b2185bd1 F test/tokenize.test ce430a7aed48fc98301611429595883fdfcab5d7 F test/tpch01.test 04adbf8d8300fa60a222f28d901abd76e7be6dd4 -F test/trace.test 4b36a41a3e9c7842151af6da5998f5080cdad9e5 +F test/trace.test 73a5508100f7fccfbc3f8018d5f6963ed478eea0 F test/trace2.test 93b47ca6996c66b47f57224cfb146f34e07df382 F test/trans.test 6e1b4c6a42dba31bd65f8fa5e61a2708e08ddde6 F test/trans2.test 62bd045bfc7a1c14c5ba83ba64d21ade31583f76 -F test/trans3.test 373ac5183cc56be69f48ae44090e7f672939f732 +F test/trans3.test 91a100e5412b488e22a655fe423a14c26403ab94 F test/transitive1.test 03f532954f46cdf5608f7766bff0b0c52bf2a7cd F test/trigger1.test dc47573ac79ffe0ee3eecaa517d70d8dacbccd03 F test/trigger2.test 5cd7d69a7ba1143ee045e4ae2963ff32ae4c87a6 @@ -1061,21 +1107,27 @@ F test/trigger5.test 619391a3e9fc194081d22cefd830d811e7badf83 F test/trigger6.test 0e411654f122552da6590f0b4e6f781048a4a9b9 F test/trigger7.test b39e6dee1debe0ff9c2ef66326668f149f07c9c4 F test/trigger8.test 30cb0530bd7c4728055420e3f739aa00412eafa4 -F test/trigger9.test 5b0789f1c5c4600961f8e68511b825b87be53e31 +F test/trigger9.test 2226ec795a33b0460ab5cf8891e9054cc7edef41 F test/triggerA.test fe5597f47ee21bacb4936dc827994ed94161e332 F test/triggerB.test 56780c031b454abac2340dbb3b71ac5c56c3d7fe F test/triggerC.test a68980c5955d62ee24be6f97129d824f199f9a4c F test/triggerD.test 8e7f3921a92a5797d472732108109e44575fa650 F test/triggerE.test 355e9c5cbaed5cd039a60baad1fb2197caeb8e52 -F test/tt3_checkpoint.c 415eccce672d681b297485fc20f44cdf0eac93af +F test/tt3_checkpoint.c 9e75cf7c1c364f52e1c47fd0f14c4340a9db0fe1 +F test/tt3_index.c 39eec10a35f57672225be4d182862152896dee4a +F test/tt3_lookaside1.c 0377e202c3c2a50d688cb65ba203afeda6fafeb9 +F test/tt3_stress.c c57d804716165811d979d4a719e05baccd79277f +F test/tt3_vacuum.c 1753f45917699c9c1f66b64c717a717c9379f776 F test/types.test bf816ce73c7dfcfe26b700c19f97ef4050d194ff F test/types2.test 3555aacf8ed8dc883356e59efc314707e6247a84 F test/types3.test 99e009491a54f4dc02c06bdbc0c5eea56ae3e25a F test/unique.test 93f8b2ef5ea51b9495f8d6493429b1fd0f465264 +F test/unique2.test 41e7f83c6827605991160a31380148a9fc5f1339 F test/unixexcl.test cd6c765f75e50e8e2c2ba763149e5d340ea19825 F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8 -F test/update.test 1b6c488a8f993d090b7ee9ad0e234faa161b3aeb +F test/update.test 6c68446b8a0a33d522a7c72b320934596a2d7d32 F test/uri.test 23662b7b61958b0f0e47082de7d06341ccf85d5b +F test/userauth01.test e740a2697a7b40d7c5003a7d7edaee16acd349a9 F test/utf16align.test 54cd35a27c005a9b6e7815d887718780b6a462ae F test/vacuum.test ce91c39f7f91a4273bf620efad21086b5aa6ef1d F test/vacuum2.test af432e6e3bfc0ea20a80cb86a03c7d9876d38324 @@ -1084,9 +1136,9 @@ F test/vacuum4.test d3f8ecff345f166911568f397d2432c16d2867d9 F test/varint.test ab7b110089a08b9926ed7390e7e97bdefeb74102 F test/veryquick.test 57ab846bacf7b90cf4e9a672721ea5c5b669b661 F test/view.test f311691d696a5cc27e3c1b875cec1b0866b4ccd9 -F test/vtab1.test b631d147b198cfd7903ab5fed028eb2a3d321dc6 +F test/vtab1.test 1cef14310144718812351a61c5cfb4ba8494a171 F test/vtab2.test 7bcffc050da5c68f4f312e49e443063e2d391c0d -F test/vtab3.test baad99fd27217f5d6db10660522e0b7192446de1 +F test/vtab3.test b45f47d20f225ccc9c28dc915d92740c2dee311e F test/vtab4.test 942f8b8280b3ea8a41dae20e7822d065ca1cb275 F test/vtab5.test 889f444970393c73f1e077e2bdc5d845e157a391 F test/vtab6.test 5f5380c425e52993560ab4763db4f826d2ba7b09 @@ -1106,7 +1158,7 @@ F test/wal.test 885f32b2b390b30b4aa3dbb0e568f8f78d40f5cc F test/wal2.test 1f841d2048080d32f552942e333fd99ce541dada F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c -F test/wal5.test 8f888b50f66b78821e61ed0e233ded5de378224b +F test/wal5.test 11b8658dd4d5448f4604124bebd9b68be5bc3e66 F test/wal6.test 527581f5527bf9c24394991e2be83000aace5f9e F test/wal64k.test 163655ecd2cb8afef4737cac2a40fdd2eeaf20b8 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd @@ -1129,7 +1181,7 @@ F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test e7be6d9888f83aa5d3d3c7c08aa9b5c28b93609a F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e F test/where.test 28b64e93428961b07b0d486778d63fd672948f6b -F test/where2.test 455a2eb2666e66c1e84e2cb5815173a85e6237db +F test/where2.test 23fdb5d8e756554aad4ca7ae03de9dd8367a2c6e F test/where3.test 1ad55ba900bd7747f98b6082e65bd3e442c5004e F test/where4.test d8420ceeb8323a41ceff1f1841fc528e824e1ecf F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2 @@ -1137,7 +1189,7 @@ F test/where6.test 5da5a98cec820d488e82708301b96cb8c18a258b F test/where7.test 5a4b0abc207d71da4deecd734ad8579e8dd40aa8 F test/where8.test 806f1dcec4088be2b826b33f757fe6e17c3236a1 F test/where8m.test da346596e19d54f0aba35ebade032a7c47d79739 -F test/where9.test 4f3eab951353a3ae164befc521c777dfa903e46c +F test/where9.test 729c3ba9b47e8f9f1aab96bae7dad2a524f1d1a2 F test/whereA.test 4d253178d135ec46d1671e440cd8f2b916aa6e6b F test/whereB.test 0def95db3bdec220a731c7e4bec5930327c1d8c5 F test/whereC.test d6f4ecd4fa2d9429681a5b22a25d2bda8e86ab8a @@ -1147,7 +1199,7 @@ F test/whereF.test 5b2ba0dbe8074aa13e416b37c753991f0a2492d7 F test/whereG.test 69f5ec4b15760a8c860f80e2d55525669390aab3 F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2 F test/whereI.test 1d89199697919d4930be05a71e7fe620f114e622 -F test/whereJ.test 7dde28284d20f358b559ca592e294db03e1d7103 +F test/whereJ.test 55a3221706a7ab706293f17cc8f96da563bf0767 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c @@ -1156,11 +1208,12 @@ F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d F test/with1.test 268081a6b14817a262ced4d0ee34d4d2a1dd2068 F test/with2.test ee227a663586aa09771cafd4fa269c5217eaf775 F test/withM.test e97f2a8c506ab3ea9eab94e6f6072f6cc924c991 -F test/without_rowid1.test e00a0a9dc9f0be651f011d61e8a32b7add5afb30 +F test/without_rowid1.test 7862e605753c8d25329f665fa09072e842183151 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99 -F test/without_rowid3.test eac3d5c8a1924725b58503a368f2cbd24fd6c8a0 +F test/without_rowid3.test 1081aabf60a1e1123b7f9a8f6ae19954351843b0 F test/without_rowid4.test 4e08bcbaee0399f35d58b5581881e7a6243d458a -F test/without_rowid5.test b4a639a367f04d382d20e8f44fc1be4f2d57d107 +F test/without_rowid5.test 61256715b686359df48ca1742db50cc7e3e7b862 +F test/without_rowid6.test deddb78ef539c355bddec00cdfaea6c56efd8b3f F test/wordcount.c 9915e06cb33d8ca8109b8700791afe80d305afda F test/zeroblob.test caaecfb4f908f7bc086ed238668049f96774d688 F test/zerodamage.test cf6748bad89553cc1632be51a6f54e487e4039ac @@ -1178,13 +1231,13 @@ F tool/lemon.c 3ff0fec22f92dfb54e62eeb48772eddffdbeb0d6 F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc F tool/loadfts.c 76b6589ab5efcdc9cfe16d43ab5a6c2618e44bd4 F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 -F tool/mkautoconfamal.sh f8d8dbf7d62f409ebed5134998bf5b51d7266383 +F tool/mkautoconfamal.sh 5dc5010e2e748a9e1bba67baca5956a2c2deda7b F tool/mkkeywordhash.c dfff09dbbfaf950e89af294f48f902181b144670 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e -F tool/mkpragmatab.tcl 78a77b2c554d534c6f2dc903130186ed15715460 +F tool/mkpragmatab.tcl 07a5124cf2dbafa1b375eefcf8ac4227028b0f8b F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 -F tool/mksqlite3c-noext.tcl 1712d3d71256ca1f297046619c89e77a4d7c8f6d -F tool/mksqlite3c.tcl 5be4e6b3ecf563c1ec6d579dc03ea2839e7fbc8b +F tool/mksqlite3c-noext.tcl 88a1e3b0c769773fb7a9ebb363ffc603a4ac21d8 +F tool/mksqlite3c.tcl 18fc267758af3cd00ffce9346f9dc807c14c5507 F tool/mksqlite3h.tcl ba24038056f51fde07c0079c41885ab85e2cff12 F tool/mksqlite3internalh.tcl b6514145a7d5321b47e64e19b8116cc44f973eb1 F tool/mkvsix.tcl 52a4c613707ac34ae9c226e5ccc69cb948556105 @@ -1194,9 +1247,10 @@ F tool/opcodeDoc.awk b3a2a3d5d3075b8bd90b7afe24283efdd586659c F tool/pagesig.c ff0ca355fd3c2398e933da5e22439bbff89b803b F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5 -F tool/showdb.c b018a8a69d07050fc0fe9afcf17313cdef0cc599 -F tool/showjournal.c b62cecaab86a4053d944c276bb5232e4d17ece02 -F tool/showwal.c 3f7f7da5ec0cba51b1449a75f700493377da57b5 +F tool/showdb.c bd073a78bce714a0e42d92ea474b3eb8cb53be5d +F tool/showjournal.c 053eb1cc774710c6890b7dd6293300cc297b16a5 +F tool/showstat4.c 9515faa8ec176599d4a8288293ba8ec61f7b728a +F tool/showwal.c 85cb36d4fe3e93e2fbd63e786e0d1ce42d0c4fad F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe F tool/space_used.tcl f714c41a59e326b8b9042f415b628b561bafa06b F tool/spaceanal.tcl 8e50b217c56a6a086a1b47eac9d09c5cd65b996f @@ -1210,12 +1264,13 @@ F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d F tool/symbols.sh fec58532668296d7c7dc48be9c87f75ccdb5814f F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 -F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 +F tool/varint.c 5d94cb5003db9dbbcbcc5df08d66f16071aee003 +F tool/vdbe-compress.tcl 5926c71f9c12d2ab73ef35c29376e756eb68361c F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7f148edb30103c5f4fee20cd08e38537f9615bf2 -R d01caf1e8e04bd7c1b6e26fb465c90b6 +P d09f7800cf14f73ea86d037107ef80295b2c173a 66269d0d8e49eb3dc7f508714753584f648bb022 +R 547c6b40048a436817ff50668d0f1e7c U dan -Z 5c3f4d7bf4502327dfa6eb630b5a26ec +Z dab366ead758ca084f8be9b0cb1bbb1d diff --git a/manifest.uuid b/manifest.uuid index f67937770d..4878666f57 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d09f7800cf14f73ea86d037107ef80295b2c173a \ No newline at end of file +4b3651677e7132c4c45605bc1f216fc08ef31198 \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c index 059ae102fa..7b56b61902 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -1395,7 +1395,7 @@ int main(int argc, char **argv){ maybeClose(g.pLog); maybeClose(g.pErrLog); if( iClient==0 ){ - printf("Summary: %d errors in %d tests\n", g.nError, g.nTest); + printf("Summary: %d errors out of %d tests\n", g.nError, g.nTest); } return g.nError>0; } diff --git a/sqlite3.1 b/sqlite3.1 index 02df7f44c7..80353b0eec 100644 --- a/sqlite3.1 +++ b/sqlite3.1 @@ -2,7 +2,7 @@ .\" First parameter, NAME, should be all caps .\" Second parameter, SECTION, should be 1-8, maybe w/ subsection .\" other parameters are allowed: see man(7), man(1) -.TH SQLITE3 1 "Mon Jan 31 11:14:00 2014" +.TH SQLITE3 1 "Fri Oct 31 10:41:31 EDT 2014" .\" Please adjust this date whenever revising the manpage. .\" .\" Some roff macros, for reference: @@ -49,7 +49,7 @@ a table named "memos" and insert a couple of records into that table: $ .B sqlite3 mydata.db .br -SQLite version 3.8.3 +SQLite version 3.8.8 .br Enter ".help" for instructions .br @@ -107,26 +107,29 @@ the '.help' command. For example: sqlite> .B .help .nf -.cc | -.backup ?DB? FILE Backup DB (default "main") to FILE -.bail ON|OFF Stop after hitting an error. Default OFF -.databases List names and files of attached databases -.dump ?TABLE? ... Dump the database in an SQL text format +.tr %. +%backup ?DB? FILE Backup DB (default "main") to FILE +%bail on|off Stop after hitting an error. Default OFF +%clone NEWDB Clone data into NEWDB from the existing database +%databases List names and files of attached databases +%dump ?TABLE? ... Dump the database in an SQL text format If TABLE specified, only dump tables matching LIKE pattern TABLE. -.echo ON|OFF Turn command echo on or off -.exit Exit this program -.explain ?ON|OFF? Turn output mode suitable for EXPLAIN on or off. +%echo on|off Turn command echo on or off +%eqp on|off Enable or disable automatic EXPLAIN QUERY PLAN +%exit Exit this program +%explain ?on|off? Turn output mode suitable for EXPLAIN on or off. With no args, it turns EXPLAIN on. -.header(s) ON|OFF Turn display of headers on or off -.help Show this message -.import FILE TABLE Import data from FILE into TABLE -.indices ?TABLE? Show names of all indices +%fullschema Show schema and the content of sqlite_stat tables +%headers on|off Turn display of headers on or off +%help Show this message +%import FILE TABLE Import data from FILE into TABLE +%indices ?TABLE? Show names of all indices If TABLE specified, only show indices for tables matching LIKE pattern TABLE. -.load FILE ?ENTRY? Load an extension library -.log FILE|off Turn logging on or off. FILE can be stderr/stdout -.mode MODE ?TABLE? Set output mode where MODE is one of: +%load FILE ?ENTRY? Load an extension library +%log FILE|off Turn logging on or off. FILE can be stderr/stdout +%mode MODE ?TABLE? Set output mode where MODE is one of: csv Comma-separated values column Left-aligned columns. (See .width) html HTML code @@ -135,31 +138,35 @@ sqlite> list Values delimited by .separator string tabs Tab-separated values tcl TCL list elements -.nullvalue STRING Use STRING in place of NULL values -.open ?FILENAME? Close existing database and reopen FILENAME -.output FILENAME Send output to FILENAME -.output stdout Send output to the screen -.print STRING... Print literal STRING -.prompt MAIN CONTINUE Replace the standard prompts -.quit Exit this program -.read FILENAME Execute SQL in FILENAME -.restore ?DB? FILE Restore content of DB (default "main") from FILE -.schema ?TABLE? Show the CREATE statements +%nullvalue STRING Use STRING in place of NULL values +%once FILENAME Output for the next SQL command only to FILENAME +%open ?FILENAME? Close existing database and reopen FILENAME +%output ?FILENAME? Send output to FILENAME or stdout +%print STRING... Print literal STRING +%prompt MAIN CONTINUE Replace the standard prompts +%quit Exit this program +%read FILENAME Execute SQL in FILENAME +%restore ?DB? FILE Restore content of DB (default "main") from FILE +%save FILE Write in-memory database into FILE +%schema ?TABLE? Show the CREATE statements If TABLE specified, only show tables matching LIKE pattern TABLE. -.separator STRING Change separator used by output mode and .import -.show Show the current values for various settings -.stats ON|OFF Turn stats on or off -.tables ?TABLE? List names of tables +%separator STRING ?NL? Change separator used by output mode and .import + NL is the end-of-line mark for CSV +%shell CMD ARGS... Run CMD ARGS... in a system shell +%show Show the current values for various settings +%stats on|off Turn stats on or off +%system CMD ARGS... Run CMD ARGS... in a system shell +%tables ?TABLE? List names of tables If TABLE specified, only list tables matching LIKE pattern TABLE. -.timeout MS Try opening locked tables for MS milliseconds -.trace FILE|off Output each SQL statement as it is run -.vfsname ?AUX? Print the name of the VFS stack -.width NUM1 NUM2 ... Set column widths for "column" mode -.timer ON|OFF Turn the CPU timer measurement on or off +%timeout MS Try opening locked tables for MS milliseconds +%timer on|off Turn SQL timer on or off +%trace FILE|off Output each SQL statement as it is run +%vfsname ?AUX? Print the name of the VFS stack +%width NUM1 NUM2 ... Set column widths for "column" mode + Negative values right-justify sqlite> -|cc . .sp .fi .SH OPTIONS @@ -269,7 +276,7 @@ o If the -init option is present, the specified file is processed. o All other command line options are processed. .SH SEE ALSO -http://www.sqlite.org/ +http://www.sqlite.org/cli.html .br The sqlite3-doc package. .SH AUTHOR diff --git a/src/alter.c b/src/alter.c index 64204b7b24..dd060248b8 100644 --- a/src/alter.c +++ b/src/alter.c @@ -174,8 +174,8 @@ static void renameTriggerFunc( UNUSED_PARAMETER(NotUsed); /* The principle used to locate the table name in the CREATE TRIGGER - ** statement is that the table name is the first token that is immediatedly - ** preceded by either TK_ON or TK_DOT and immediatedly followed by one + ** statement is that the table name is the first token that is immediately + ** preceded by either TK_ON or TK_DOT and immediately followed by one ** of TK_WHEN, TK_BEGIN or TK_FOR. */ if( zSql ){ diff --git a/src/analyze.c b/src/analyze.c index 4dcd7e8b8f..e483807116 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -35,7 +35,7 @@ ** not possible to enable both STAT3 and STAT4 at the same time. If they ** are both enabled, then STAT4 takes precedence. ** -** For most applications, sqlite_stat1 provides all the statisics required +** For most applications, sqlite_stat1 provides all the statistics required ** for the query planner to make good choices. ** ** Format of sqlite_stat1: @@ -246,6 +246,7 @@ static void openStatTable( assert( i1 ); /* >1 because it includes the rowid column */ + assert( nCol>0 ); nColUp = sizeof(tRowcnt)<8 ? (nCol+1)&~1 : nCol; + nKeyCol = sqlite3_value_int(argv[1]); + assert( nKeyCol<=nCol ); + assert( nKeyCol>0 ); /* Allocate the space required for the Stat4Accum object */ n = sizeof(*p) @@ -415,6 +435,7 @@ static void statInit( p->db = db; p->nRow = 0; p->nCol = nCol; + p->nKeyCol = nKeyCol; p->current.anDLt = (tRowcnt*)&p[1]; p->current.anEq = &p->current.anDLt[nColUp]; @@ -425,9 +446,9 @@ static void statInit( p->iGet = -1; p->mxSample = mxSample; - p->nPSample = (tRowcnt)(sqlite3_value_int64(argv[1])/(mxSample/3+1) + 1); + p->nPSample = (tRowcnt)(sqlite3_value_int64(argv[2])/(mxSample/3+1) + 1); p->current.anLt = &p->current.anEq[nColUp]; - p->iPrn = nCol*0x689e962d ^ sqlite3_value_int(argv[1])*0xd0944565; + p->iPrn = nCol*0x689e962d ^ sqlite3_value_int(argv[2])*0xd0944565; /* Set up the Stat4Accum.a[] and aBest[] arrays */ p->a = (struct Stat4Sample*)&p->current.anLt[nColUp]; @@ -446,11 +467,14 @@ static void statInit( } #endif - /* Return a pointer to the allocated object to the caller */ - sqlite3_result_blob(context, p, sizeof(p), stat4Destructor); + /* Return a pointer to the allocated object to the caller. Note that + ** only the pointer (the 2nd parameter) matters. The size of the object + ** (given by the 3rd parameter) is never used and can be any positive + ** value. */ + sqlite3_result_blob(context, p, sizeof(*p), stat4Destructor); } static const FuncDef statInitFuncdef = { - 1+IsStat34, /* nArg */ + 2+IsStat34, /* nArg */ SQLITE_UTF8, /* funcFlags */ 0, /* pUserData */ 0, /* pNext */ @@ -674,7 +698,10 @@ static void samplePushPrevious(Stat4Accum *p, int iChng){ ** R Rowid for the current row. Might be a key record for ** WITHOUT ROWID tables. ** -** The SQL function always returns NULL. +** This SQL function always returns NULL. It's purpose it to accumulate +** statistical data and/or samples in the Stat4Accum object about the +** index being analyzed. The stat_get() SQL function will later be used to +** extract relevant information for constructing the sqlite_statN tables. ** ** The R parameter is only used for STAT3 and STAT4 */ @@ -691,7 +718,7 @@ static void statPush( UNUSED_PARAMETER( argc ); UNUSED_PARAMETER( context ); - assert( p->nCol>1 ); /* Includes rowid field */ + assert( p->nCol>0 ); assert( iChngnCol ); if( p->nRow==0 ){ @@ -768,7 +795,10 @@ static const FuncDef statPushFuncdef = { /* ** Implementation of the stat_get(P,J) SQL function. This routine is -** used to query the results. Content is returned for parameter J +** used to query statistical information that has been gathered into +** the Stat4Accum object by prior calls to stat_push(). The P parameter +** has type BLOB but it is really just a pointer to the Stat4Accum object. +** The content to returned is determined by the parameter J ** which is one of the STAT_GET_xxxx values defined above. ** ** If neither STAT3 nor STAT4 are enabled, then J is always @@ -819,7 +849,7 @@ static void statGet( char *z; int i; - char *zRet = sqlite3MallocZero(p->nCol * 25); + char *zRet = sqlite3MallocZero( (p->nKeyCol+1)*25 ); if( zRet==0 ){ sqlite3_result_error_nomem(context); return; @@ -827,7 +857,7 @@ static void statGet( sqlite3_snprintf(24, zRet, "%llu", (u64)p->nRow); z = zRet + sqlite3Strlen30(zRet); - for(i=0; i<(p->nCol-1); i++){ + for(i=0; inKeyCol; i++){ u64 nDistinct = p->current.anDLt[i] + 1; u64 iVal = (p->nRow + nDistinct - 1) / nDistinct; sqlite3_snprintf(24, z, " %llu", iVal); @@ -987,27 +1017,27 @@ static void analyzeOneTable( sqlite3VdbeAddOp4(v, OP_String8, 0, regTabname, 0, pTab->zName, 0); for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - int nCol; /* Number of columns indexed by pIdx */ - int *aGotoChng; /* Array of jump instruction addresses */ + int nCol; /* Number of columns in pIdx. "N" */ int addrRewind; /* Address of "OP_Rewind iIdxCur" */ - int addrGotoChng0; /* Address of "Goto addr_chng_0" */ int addrNextRow; /* Address of "next_row:" */ const char *zIdxName; /* Name of the index */ + int nColTest; /* Number of columns to test for changes */ if( pOnlyIdx && pOnlyIdx!=pIdx ) continue; if( pIdx->pPartIdxWhere==0 ) needTableCnt = 0; - VdbeNoopComment((v, "Begin analysis of %s", pIdx->zName)); - nCol = pIdx->nKeyCol; - aGotoChng = sqlite3DbMallocRaw(db, sizeof(int)*(nCol+1)); - if( aGotoChng==0 ) continue; + if( !HasRowid(pTab) && IsPrimaryKeyIndex(pIdx) ){ + nCol = pIdx->nKeyCol; + zIdxName = pTab->zName; + nColTest = nCol - 1; + }else{ + nCol = pIdx->nColumn; + zIdxName = pIdx->zName; + nColTest = pIdx->uniqNotNull ? pIdx->nKeyCol-1 : nCol-1; + } /* Populate the register containing the index name. */ - if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) ){ - zIdxName = pTab->zName; - }else{ - zIdxName = pIdx->zName; - } sqlite3VdbeAddOp4(v, OP_String8, 0, regIdxname, 0, zIdxName, 0); + VdbeComment((v, "Analysis for %s.%s", pTab->zName, zIdxName)); /* ** Pseudo-code for loop that calls stat_push(): @@ -1032,7 +1062,7 @@ static void analyzeOneTable( ** regPrev(1) = idx(1) ** ... ** - ** chng_addr_N: + ** endDistinctTest: ** regRowid = idx(rowid) ** stat_push(P, regChng, regRowid) ** Next csr @@ -1045,7 +1075,7 @@ static void analyzeOneTable( ** the regPrev array and a trailing rowid (the rowid slot is required ** when building a record to insert into the sample column of ** the sqlite_stat4 table. */ - pParse->nMem = MAX(pParse->nMem, regPrev+nCol); + pParse->nMem = MAX(pParse->nMem, regPrev+nColTest); /* Open a read-only cursor on the index being analyzed. */ assert( iDb==sqlite3SchemaToIndex(db, pIdx->pSchema) ); @@ -1055,18 +1085,22 @@ static void analyzeOneTable( /* Invoke the stat_init() function. The arguments are: ** - ** (1) the number of columns in the index including the rowid, - ** (2) the number of rows in the index, + ** (1) the number of columns in the index including the rowid + ** (or for a WITHOUT ROWID table, the number of PK columns), + ** (2) the number of columns in the key without the rowid/pk + ** (3) the number of rows in the index, ** - ** The second argument is only used for STAT3 and STAT4 + ** + ** The third argument is only used for STAT3 and STAT4 */ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regStat4+2); + sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regStat4+3); #endif - sqlite3VdbeAddOp2(v, OP_Integer, nCol+1, regStat4+1); + sqlite3VdbeAddOp2(v, OP_Integer, nCol, regStat4+1); + sqlite3VdbeAddOp2(v, OP_Integer, pIdx->nKeyCol, regStat4+2); sqlite3VdbeAddOp3(v, OP_Function, 0, regStat4+1, regStat4); sqlite3VdbeChangeP4(v, -1, (char*)&statInitFuncdef, P4_FUNCDEF); - sqlite3VdbeChangeP5(v, 1+IsStat34); + sqlite3VdbeChangeP5(v, 2+IsStat34); /* Implementation of the following: ** @@ -1079,44 +1113,62 @@ static void analyzeOneTable( addrRewind = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Integer, 0, regChng); - addrGotoChng0 = sqlite3VdbeAddOp0(v, OP_Goto); - - /* - ** next_row: - ** regChng = 0 - ** if( idx(0) != regPrev(0) ) goto chng_addr_0 - ** regChng = 1 - ** if( idx(1) != regPrev(1) ) goto chng_addr_1 - ** ... - ** regChng = N - ** goto chng_addr_N - */ addrNextRow = sqlite3VdbeCurrentAddr(v); - for(i=0; iazColl[i]); - sqlite3VdbeAddOp2(v, OP_Integer, i, regChng); - sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, i, regTemp); - aGotoChng[i] = - sqlite3VdbeAddOp4(v, OP_Ne, regTemp, 0, regPrev+i, pColl, P4_COLLSEQ); - sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); - VdbeCoverage(v); - } - sqlite3VdbeAddOp2(v, OP_Integer, nCol, regChng); - aGotoChng[nCol] = sqlite3VdbeAddOp0(v, OP_Goto); - /* - ** chng_addr_0: - ** regPrev(0) = idx(0) - ** chng_addr_1: - ** regPrev(1) = idx(1) - ** ... - */ - sqlite3VdbeJumpHere(v, addrGotoChng0); - for(i=0; i0 ){ + int endDistinctTest = sqlite3VdbeMakeLabel(v); + int *aGotoChng; /* Array of jump instruction addresses */ + aGotoChng = sqlite3DbMallocRaw(db, sizeof(int)*nColTest); + if( aGotoChng==0 ) continue; + /* + ** next_row: + ** regChng = 0 + ** if( idx(0) != regPrev(0) ) goto chng_addr_0 + ** regChng = 1 + ** if( idx(1) != regPrev(1) ) goto chng_addr_1 + ** ... + ** regChng = N + ** goto endDistinctTest + */ + sqlite3VdbeAddOp0(v, OP_Goto); + addrNextRow = sqlite3VdbeCurrentAddr(v); + if( nColTest==1 && pIdx->nKeyCol==1 && IsUniqueIndex(pIdx) ){ + /* For a single-column UNIQUE index, once we have found a non-NULL + ** row, we know that all the rest will be distinct, so skip + ** subsequent distinctness tests. */ + sqlite3VdbeAddOp2(v, OP_NotNull, regPrev, endDistinctTest); + VdbeCoverage(v); + } + for(i=0; iazColl[i]); + sqlite3VdbeAddOp2(v, OP_Integer, i, regChng); + sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, i, regTemp); + aGotoChng[i] = + sqlite3VdbeAddOp4(v, OP_Ne, regTemp, 0, regPrev+i, pColl, P4_COLLSEQ); + sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); + VdbeCoverage(v); + } + sqlite3VdbeAddOp2(v, OP_Integer, nColTest, regChng); + sqlite3VdbeAddOp2(v, OP_Goto, 0, endDistinctTest); + + + /* + ** chng_addr_0: + ** regPrev(0) = idx(0) + ** chng_addr_1: + ** regPrev(1) = idx(1) + ** ... + */ + sqlite3VdbeJumpHere(v, addrNextRow-1); + for(i=0; inMem = MAX(pParse->nMem, regCol+nCol+1); + pParse->nMem = MAX(pParse->nMem, regCol+nCol); addrNext = sqlite3VdbeCurrentAddr(v); callStatGet(v, regStat4, STAT_GET_ROWID, regSampleRowid); @@ -1190,7 +1242,7 @@ static void analyzeOneTable( i16 iCol = pIdx->aiColumn[i]; sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, iCol, regCol+i); } - sqlite3VdbeAddOp3(v, OP_MakeRecord, regCol, nCol+1, regSample); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regCol, nCol, regSample); #endif sqlite3VdbeAddOp3(v, OP_MakeRecord, regTabname, 6, regTemp); sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur+1, regNewRowid); @@ -1202,7 +1254,6 @@ static void analyzeOneTable( /* End of analysis */ sqlite3VdbeJumpHere(v, addrRewind); - sqlite3DbFree(db, aGotoChng); } @@ -1214,7 +1265,8 @@ static void analyzeOneTable( sqlite3VdbeAddOp2(v, OP_Count, iTabCur, regStat1); jZeroRows = sqlite3VdbeAddOp1(v, OP_IfNot, regStat1); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Null, 0, regIdxname); - sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regTemp, "aaa", 0); + assert( "BBB"[0]==SQLITE_AFF_TEXT ); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regTemp, "BBB", 0); sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regNewRowid); sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regTemp, regNewRowid); sqlite3VdbeChangeP5(v, OPFLAG_APPEND); @@ -1303,6 +1355,7 @@ void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ Table *pTab; Index *pIdx; Token *pTableName; + Vdbe *v; /* Read the database schema. If an error occurs, leave an error message ** and code in pParse and return NULL. */ @@ -1350,6 +1403,8 @@ void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ } } } + v = sqlite3GetVdbe(pParse); + if( v ) sqlite3VdbeAddOp0(v, OP_Expire); } /* @@ -1382,7 +1437,7 @@ static void decodeIntArray( #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 if( z==0 ) z = ""; #else - if( NEVER(z==0) ) z = ""; + assert( z!=0 ); #endif for(i=0; *z && ibUnordered = 1; - }else if( sqlite3_strglob("sz=[0-9]*", z)==0 ){ - int v32 = 0; - sqlite3GetInt32(z+3, &v32); - pIndex->szIdxRow = sqlite3LogEst(v32); + pIndex->bUnordered = 0; + pIndex->noSkipScan = 0; + while( z[0] ){ + if( sqlite3_strglob("unordered*", z)==0 ){ + pIndex->bUnordered = 1; + }else if( sqlite3_strglob("sz=[0-9]*", z)==0 ){ + pIndex->szIdxRow = sqlite3LogEst(sqlite3Atoi(z+3)); + }else if( sqlite3_strglob("noskipscan*", z)==0 ){ + pIndex->noSkipScan = 1; + } +#ifdef SQLITE_ENABLE_COSTMULT + else if( sqlite3_strglob("costmult=[0-9]*",z)==0 ){ + pIndex->pTable->costMult = sqlite3LogEst(sqlite3Atoi(z+9)); + } +#endif + while( z[0]!=0 && z[0]!=' ' ) z++; + while( z[0]==' ' ) z++; } } } @@ -1456,11 +1519,24 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ z = argv[2]; if( pIndex ){ - decodeIntArray((char*)z, pIndex->nKeyCol+1, 0, pIndex->aiRowLogEst, pIndex); + int nCol = pIndex->nKeyCol+1; +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + tRowcnt * const aiRowEst = pIndex->aiRowEst = (tRowcnt*)sqlite3MallocZero( + sizeof(tRowcnt) * nCol + ); + if( aiRowEst==0 ) pInfo->db->mallocFailed = 1; +#else + tRowcnt * const aiRowEst = 0; +#endif + pIndex->bUnordered = 0; + decodeIntArray((char*)z, nCol, aiRowEst, pIndex->aiRowLogEst, pIndex); if( pIndex->pPartIdxWhere==0 ) pTable->nRowLogEst = pIndex->aiRowLogEst[0]; }else{ Index fakeIdx; fakeIdx.szIdxRow = pTable->szTabRow; +#ifdef SQLITE_ENABLE_COSTMULT + fakeIdx.pTable = pTable; +#endif decodeIntArray((char*)z, 1, 0, &pTable->nRowLogEst, &fakeIdx); pTable->szTabRow = fakeIdx.szIdxRow; } @@ -1502,30 +1578,52 @@ static void initAvgEq(Index *pIdx){ IndexSample *aSample = pIdx->aSample; IndexSample *pFinal = &aSample[pIdx->nSample-1]; int iCol; - for(iCol=0; iColnKeyCol; iCol++){ + int nCol = 1; + if( pIdx->nSampleCol>1 ){ + /* If this is stat4 data, then calculate aAvgEq[] values for all + ** sample columns except the last. The last is always set to 1, as + ** once the trailing PK fields are considered all index keys are + ** unique. */ + nCol = pIdx->nSampleCol-1; + pIdx->aAvgEq[nCol] = 1; + } + for(iCol=0; iColnSample; int i; /* Used to iterate through samples */ tRowcnt sumEq = 0; /* Sum of the nEq values */ - tRowcnt nSum = 0; /* Number of terms contributing to sumEq */ tRowcnt avgEq = 0; - tRowcnt nDLt = pFinal->anDLt[iCol]; + tRowcnt nRow; /* Number of rows in index */ + i64 nSum100 = 0; /* Number of terms contributing to sumEq */ + i64 nDist100; /* Number of distinct values in index */ + + if( !pIdx->aiRowEst || iCol>=pIdx->nKeyCol || pIdx->aiRowEst[iCol+1]==0 ){ + nRow = pFinal->anLt[iCol]; + nDist100 = (i64)100 * pFinal->anDLt[iCol]; + nSample--; + }else{ + nRow = pIdx->aiRowEst[0]; + nDist100 = ((i64)100 * pIdx->aiRowEst[0]) / pIdx->aiRowEst[iCol+1]; + } + pIdx->nRowEst0 = nRow; /* Set nSum to the number of distinct (iCol+1) field prefixes that - ** occur in the stat4 table for this index before pFinal. Set - ** sumEq to the sum of the nEq values for column iCol for the same - ** set (adding the value only once where there exist dupicate - ** prefixes). */ - for(i=0; i<(pIdx->nSample-1); i++){ - if( aSample[i].anDLt[iCol]!=aSample[i+1].anDLt[iCol] ){ + ** occur in the stat4 table for this index. Set sumEq to the sum of + ** the nEq values for column iCol for the same set (adding the value + ** only once where there exist duplicate prefixes). */ + for(i=0; inSample-1) + || aSample[i].anDLt[iCol]!=aSample[i+1].anDLt[iCol] + ){ sumEq += aSample[i].anEq[iCol]; - nSum++; + nSum100 += 100; } } - if( nDLt>nSum ){ - avgEq = (pFinal->anLt[iCol] - sumEq)/(nDLt - nSum); + + if( nDist100>nSum100 ){ + avgEq = ((i64)100 * (nRow - sumEq))/(nDist100 - nSum100); } if( avgEq==0 ) avgEq = 1; pIdx->aAvgEq[iCol] = avgEq; - if( pIdx->nSampleCol==1 ) break; } } } @@ -1584,7 +1682,6 @@ static int loadStatTbl( while( sqlite3_step(pStmt)==SQLITE_ROW ){ int nIdxCol = 1; /* Number of columns in stat4 records */ - int nAvgCol = 1; /* Number of entries in Index.aAvgEq */ char *zIndex; /* Index name */ Index *pIdx; /* Pointer to the index object */ @@ -1602,13 +1699,17 @@ static int loadStatTbl( ** loaded from the stat4 table. In this case ignore stat3 data. */ if( pIdx==0 || pIdx->nSample ) continue; if( bStat3==0 ){ - nIdxCol = pIdx->nKeyCol+1; - nAvgCol = pIdx->nKeyCol; + assert( !HasRowid(pIdx->pTable) || pIdx->nColumn==pIdx->nKeyCol+1 ); + if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){ + nIdxCol = pIdx->nKeyCol; + }else{ + nIdxCol = pIdx->nColumn; + } } pIdx->nSampleCol = nIdxCol; nByte = sizeof(IndexSample) * nSample; nByte += sizeof(tRowcnt) * nIdxCol * 3 * nSample; - nByte += nAvgCol * sizeof(tRowcnt); /* Space for Index.aAvgEq[] */ + nByte += nIdxCol * sizeof(tRowcnt); /* Space for Index.aAvgEq[] */ pIdx->aSample = sqlite3DbMallocZero(db, nByte); if( pIdx->aSample==0 ){ @@ -1616,7 +1717,7 @@ static int loadStatTbl( return SQLITE_NOMEM; } pSpace = (tRowcnt*)&pIdx->aSample[nSample]; - pIdx->aAvgEq = pSpace; pSpace += nAvgCol; + pIdx->aAvgEq = pSpace; pSpace += nIdxCol; for(i=0; iaSample[i].anEq = pSpace; pSpace += nIdxCol; pIdx->aSample[i].anLt = pSpace; pSpace += nIdxCol; @@ -1765,12 +1866,17 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ /* Load the statistics from the sqlite_stat4 table. */ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && OptimizationEnabled(db, SQLITE_Stat34) ){ int lookasideEnabled = db->lookaside.bEnabled; db->lookaside.bEnabled = 0; rc = loadStat4(db, sInfo.zDatabase); db->lookaside.bEnabled = lookasideEnabled; } + for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ + Index *pIdx = sqliteHashData(i); + sqlite3_free(pIdx->aiRowEst); + pIdx->aiRowEst = 0; + } #endif if( rc==SQLITE_NOMEM ){ diff --git a/src/attach.c b/src/attach.c index 89050fd9dc..de8742938b 100644 --- a/src/attach.c +++ b/src/attach.c @@ -150,6 +150,7 @@ static void attachFunc( "attached databases must use the same text encoding as main database"); rc = SQLITE_ERROR; } + sqlite3BtreeEnter(aNew->pBt); pPager = sqlite3BtreePager(aNew->pBt); sqlite3PagerLockingMode(pPager, db->dfltLockMode); sqlite3BtreeSecureDelete(aNew->pBt, @@ -157,6 +158,7 @@ static void attachFunc( #ifndef SQLITE_OMIT_PAGER_PRAGMAS sqlite3BtreeSetPagerFlags(aNew->pBt, 3 | (db->flags & PAGER_FLAGS_MASK)); #endif + sqlite3BtreeLeave(aNew->pBt); } aNew->safety_level = 3; aNew->zName = sqlite3DbStrDup(db, zName); @@ -207,6 +209,15 @@ static void attachFunc( rc = sqlite3Init(db, &zErrDyn); sqlite3BtreeLeaveAll(db); } +#ifdef SQLITE_USER_AUTHENTICATION + if( rc==SQLITE_OK ){ + u8 newAuth = 0; + rc = sqlite3UserAuthCheckLogin(db, zName, &newAuth); + if( newAuthauth.authLevel ){ + rc = SQLITE_AUTH_USER; + } + } +#endif if( rc ){ int iDb = db->nDb - 1; assert( iDb>=2 ); diff --git a/src/auth.c b/src/auth.c index d38bb836a7..9768fc2fc0 100644 --- a/src/auth.c +++ b/src/auth.c @@ -72,8 +72,11 @@ int sqlite3_set_authorizer( int (*xAuth)(void*,int,const char*,const char*,const char*,const char*), void *pArg ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); - db->xAuth = xAuth; + db->xAuth = (sqlite3_xauth)xAuth; db->pAuthArg = pArg; sqlite3ExpirePreparedStatements(db); sqlite3_mutex_leave(db->mutex); @@ -108,7 +111,11 @@ int sqlite3AuthReadCol( char *zDb = db->aDb[iDb].zName; /* Name of attached database */ int rc; /* Auth callback return code */ - rc = db->xAuth(db->pAuthArg, SQLITE_READ, zTab,zCol,zDb,pParse->zAuthContext); + rc = db->xAuth(db->pAuthArg, SQLITE_READ, zTab,zCol,zDb,pParse->zAuthContext +#ifdef SQLITE_USER_AUTHENTICATION + ,db->auth.zAuthUser +#endif + ); if( rc==SQLITE_DENY ){ if( db->nDb>2 || iDb!=0 ){ sqlite3ErrorMsg(pParse, "access to %s.%s.%s is prohibited",zDb,zTab,zCol); @@ -208,7 +215,11 @@ int sqlite3AuthCheck( if( db->xAuth==0 ){ return SQLITE_OK; } - rc = db->xAuth(db->pAuthArg, code, zArg1, zArg2, zArg3, pParse->zAuthContext); + rc = db->xAuth(db->pAuthArg, code, zArg1, zArg2, zArg3, pParse->zAuthContext +#ifdef SQLITE_USER_AUTHENTICATION + ,db->auth.zAuthUser +#endif + ); if( rc==SQLITE_DENY ){ sqlite3ErrorMsg(pParse, "not authorized"); pParse->rc = SQLITE_AUTH; diff --git a/src/backup.c b/src/backup.c index 4a6bc7493c..e3f869035e 100644 --- a/src/backup.c +++ b/src/backup.c @@ -87,12 +87,12 @@ static Btree *findBtree(sqlite3 *pErrorDb, sqlite3 *pDb, const char *zDb){ int rc = 0; pParse = sqlite3StackAllocZero(pErrorDb, sizeof(*pParse)); if( pParse==0 ){ - sqlite3Error(pErrorDb, SQLITE_NOMEM, "out of memory"); + sqlite3ErrorWithMsg(pErrorDb, SQLITE_NOMEM, "out of memory"); rc = SQLITE_NOMEM; }else{ pParse->db = pDb; if( sqlite3OpenTempDatabase(pParse) ){ - sqlite3Error(pErrorDb, pParse->rc, "%s", pParse->zErrMsg); + sqlite3ErrorWithMsg(pErrorDb, pParse->rc, "%s", pParse->zErrMsg); rc = SQLITE_ERROR; } sqlite3DbFree(pErrorDb, pParse->zErrMsg); @@ -105,7 +105,7 @@ static Btree *findBtree(sqlite3 *pErrorDb, sqlite3 *pDb, const char *zDb){ } if( i<0 ){ - sqlite3Error(pErrorDb, SQLITE_ERROR, "unknown database %s", zDb); + sqlite3ErrorWithMsg(pErrorDb, SQLITE_ERROR, "unknown database %s", zDb); return 0; } @@ -122,6 +122,20 @@ static int setDestPgsz(sqlite3_backup *p){ return rc; } +/* +** Check that there is no open read-transaction on the b-tree passed as the +** second argument. If there is not, return SQLITE_OK. Otherwise, if there +** is an open read-transaction, return SQLITE_ERROR and leave an error +** message in database handle db. +*/ +static int checkReadTransaction(sqlite3 *db, Btree *p){ + if( sqlite3BtreeIsInReadTrans(p) ){ + sqlite3ErrorWithMsg(db, SQLITE_ERROR, "destination database is in use"); + return SQLITE_ERROR; + } + return SQLITE_OK; +} + /* ** Create an sqlite3_backup process to copy the contents of zSrcDb from ** connection handle pSrcDb to zDestDb in pDestDb. If successful, return @@ -138,6 +152,13 @@ sqlite3_backup *sqlite3_backup_init( ){ sqlite3_backup *p; /* Value to return */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(pSrcDb)||!sqlite3SafetyCheckOk(pDestDb) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + /* Lock the source database handle. The destination database ** handle is not locked in this routine, but it is locked in ** sqlite3_backup_step(). The user is required to ensure that no @@ -150,7 +171,7 @@ sqlite3_backup *sqlite3_backup_init( sqlite3_mutex_enter(pDestDb->mutex); if( pSrcDb==pDestDb ){ - sqlite3Error( + sqlite3ErrorWithMsg( pDestDb, SQLITE_ERROR, "source and destination must be distinct" ); p = 0; @@ -161,7 +182,7 @@ sqlite3_backup *sqlite3_backup_init( ** sqlite3_backup_finish(). */ p = (sqlite3_backup *)sqlite3MallocZero(sizeof(sqlite3_backup)); if( !p ){ - sqlite3Error(pDestDb, SQLITE_NOMEM, 0); + sqlite3Error(pDestDb, SQLITE_NOMEM); } } @@ -174,12 +195,15 @@ sqlite3_backup *sqlite3_backup_init( p->iNext = 1; p->isAttached = 0; - if( 0==p->pSrc || 0==p->pDest || setDestPgsz(p)==SQLITE_NOMEM ){ + if( 0==p->pSrc || 0==p->pDest + || setDestPgsz(p)==SQLITE_NOMEM + || checkReadTransaction(pDestDb, p->pDest)!=SQLITE_OK + ){ /* One (or both) of the named databases did not exist or an OOM - ** error was hit. The error has already been written into the - ** pDestDb handle. All that is left to do here is free the - ** sqlite3_backup structure. - */ + ** error was hit. Or there is a transaction open on the destination + ** database. The error has already been written into the pDestDb + ** handle. All that is left to do here is free the sqlite3_backup + ** structure. */ sqlite3_free(p); p = 0; } @@ -334,6 +358,9 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){ int pgszSrc = 0; /* Source page size */ int pgszDest = 0; /* Destination page size */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( p==0 ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(p->pSrcDb->mutex); sqlite3BtreeEnter(p->pSrc); if( p->pDestDb ){ @@ -597,12 +624,12 @@ int sqlite3_backup_finish(sqlite3_backup *p){ } /* If a transaction is still open on the Btree, roll it back. */ - sqlite3BtreeRollback(p->pDest, SQLITE_OK); + sqlite3BtreeRollback(p->pDest, SQLITE_OK, 0); /* Set the error code of the destination database handle. */ rc = (p->rc==SQLITE_DONE) ? SQLITE_OK : p->rc; if( p->pDestDb ){ - sqlite3Error(p->pDestDb, rc, 0); + sqlite3Error(p->pDestDb, rc); /* Exit the mutexes and free the backup context structure. */ sqlite3LeaveMutexAndCloseZombie(p->pDestDb); @@ -623,6 +650,12 @@ int sqlite3_backup_finish(sqlite3_backup *p){ ** call to sqlite3_backup_step(). */ int sqlite3_backup_remaining(sqlite3_backup *p){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( p==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif return p->nRemaining; } @@ -631,6 +664,12 @@ int sqlite3_backup_remaining(sqlite3_backup *p){ ** recent call to sqlite3_backup_step(). */ int sqlite3_backup_pagecount(sqlite3_backup *p){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( p==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif return p->nPagecount; } diff --git a/src/btmutex.c b/src/btmutex.c index d87d4d5fee..f9fe5b3dde 100644 --- a/src/btmutex.c +++ b/src/btmutex.c @@ -38,7 +38,7 @@ static void lockBtreeMutex(Btree *p){ ** Release the BtShared mutex associated with B-Tree handle p and ** clear the p->locked boolean. */ -static void unlockBtreeMutex(Btree *p){ +static void SQLITE_NOINLINE unlockBtreeMutex(Btree *p){ BtShared *pBt = p->pBt; assert( p->locked==1 ); assert( sqlite3_mutex_held(pBt->mutex) ); @@ -49,6 +49,9 @@ static void unlockBtreeMutex(Btree *p){ p->locked = 0; } +/* Forward reference */ +static void SQLITE_NOINLINE btreeLockCarefully(Btree *p); + /* ** Enter a mutex on the given BTree object. ** @@ -66,8 +69,6 @@ static void unlockBtreeMutex(Btree *p){ ** subsequent Btrees that desire a lock. */ void sqlite3BtreeEnter(Btree *p){ - Btree *pLater; - /* Some basic sanity checking on the Btree. The list of Btrees ** connected by pNext and pPrev should be in sorted order by ** Btree.pBt value. All elements of the list should belong to @@ -92,9 +93,20 @@ void sqlite3BtreeEnter(Btree *p){ if( !p->sharable ) return; p->wantToLock++; if( p->locked ) return; + btreeLockCarefully(p); +} + +/* This is a helper function for sqlite3BtreeLock(). By moving +** complex, but seldom used logic, out of sqlite3BtreeLock() and +** into this routine, we avoid unnecessary stack pointer changes +** and thus help the sqlite3BtreeLock() routine to run much faster +** in the common case. +*/ +static void SQLITE_NOINLINE btreeLockCarefully(Btree *p){ + Btree *pLater; /* In most cases, we should be able to acquire the lock we - ** want without having to go throught the ascending lock + ** want without having to go through the ascending lock ** procedure that follows. Just be sure not to block. */ if( sqlite3_mutex_try(p->pBt->mutex)==SQLITE_OK ){ @@ -124,6 +136,7 @@ void sqlite3BtreeEnter(Btree *p){ } } + /* ** Exit the recursive mutex on a Btree. */ diff --git a/src/btree.c b/src/btree.c index 9032a9304c..f9f76c2ebb 100644 --- a/src/btree.c +++ b/src/btree.c @@ -9,7 +9,7 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** This file implements a external (disk-based) database using BTrees. +** This file implements an external (disk-based) database using BTrees. ** See the header comment on "btreeInt.h" for additional information. ** Including a description of file format and an overview of operation. */ @@ -162,7 +162,7 @@ static int hasSharedCacheTableLock( ** the correct locks are held. So do not bother - just return true. ** This case does not come up very often anyhow. */ - if( isIndex && (!pSchema || (pSchema->flags&DB_SchemaLoaded)==0) ){ + if( isIndex && (!pSchema || (pSchema->schemaFlags&DB_SchemaLoaded)==0) ){ return 1; } @@ -487,7 +487,9 @@ static void invalidateIncrblobCursors( BtShared *pBt = pBtree->pBt; assert( sqlite3BtreeHoldsMutex(pBtree) ); for(p=pBt->pCursor; p; p=p->pNext){ - if( (p->curFlags & BTCF_Incrblob)!=0 && (isClearTable || p->info.nKey==iRow) ){ + if( (p->curFlags & BTCF_Incrblob)!=0 + && (isClearTable || p->info.nKey==iRow) + ){ p->eState = CURSOR_INVALID; } } @@ -606,7 +608,7 @@ static int saveCursorPosition(BtCursor *pCur){ ** data. */ if( 0==pCur->apPage[0]->intKey ){ - void *pKey = sqlite3Malloc( (int)pCur->nKey ); + void *pKey = sqlite3Malloc( pCur->nKey ); if( pKey ){ rc = sqlite3BtreeKey(pCur, 0, (int)pCur->nKey, pKey); if( rc==SQLITE_OK ){ @@ -629,16 +631,42 @@ static int saveCursorPosition(BtCursor *pCur){ return rc; } +/* Forward reference */ +static int SQLITE_NOINLINE saveCursorsOnList(BtCursor*,Pgno,BtCursor*); + /* ** Save the positions of all cursors (except pExcept) that are open on -** the table with root-page iRoot. Usually, this is called just before cursor -** pExcept is used to modify the table (BtreeDelete() or BtreeInsert()). +** the table with root-page iRoot. "Saving the cursor position" means that +** the location in the btree is remembered in such a way that it can be +** moved back to the same spot after the btree has been modified. This +** routine is called just before cursor pExcept is used to modify the +** table, for example in BtreeDelete() or BtreeInsert(). +** +** Implementation note: This routine merely checks to see if any cursors +** need to be saved. It calls out to saveCursorsOnList() in the (unusual) +** event that cursors are in need to being saved. */ static int saveAllCursors(BtShared *pBt, Pgno iRoot, BtCursor *pExcept){ BtCursor *p; assert( sqlite3_mutex_held(pBt->mutex) ); assert( pExcept==0 || pExcept->pBt==pBt ); for(p=pBt->pCursor; p; p=p->pNext){ + if( p!=pExcept && (0==iRoot || p->pgnoRoot==iRoot) ) break; + } + return p ? saveCursorsOnList(p, iRoot, pExcept) : SQLITE_OK; +} + +/* This helper routine to saveAllCursors does the actual work of saving +** the cursors if and when a cursor is found that actually requires saving. +** The common case is that no cursors need to be saved, so this routine is +** broken out from its caller to avoid unnecessary stack pointer movement. +*/ +static int SQLITE_NOINLINE saveCursorsOnList( + BtCursor *p, /* The first cursor that needs saving */ + Pgno iRoot, /* Only save cursor with this iRoot. Save all if zero */ + BtCursor *pExcept /* Do not save this cursor */ +){ + do{ if( p!=pExcept && (0==iRoot || p->pgnoRoot==iRoot) ){ if( p->eState==CURSOR_VALID ){ int rc = saveCursorPosition(p); @@ -650,7 +678,8 @@ static int saveAllCursors(BtShared *pBt, Pgno iRoot, BtCursor *pExcept){ btreeReleaseAllCursorPages(p); } } - } + p = p->pNext; + }while( p ); return SQLITE_OK; } @@ -735,37 +764,48 @@ static int btreeRestoreCursorPosition(BtCursor *pCur){ SQLITE_OK) /* -** Determine whether or not a cursor has moved from the position it -** was last placed at. Cursors can move when the row they are pointing -** at is deleted out from under them. +** Determine whether or not a cursor has moved from the position where +** it was last placed, or has been invalidated for any other reason. +** Cursors can move when the row they are pointing at is deleted out +** from under them, for example. Cursor might also move if a btree +** is rebalanced. ** -** This routine returns an error code if something goes wrong. The -** integer *pHasMoved is set as follows: +** Calling this routine with a NULL cursor pointer returns false. ** -** 0: The cursor is unchanged -** 1: The cursor is still pointing at the same row, but the pointers -** returned by sqlite3BtreeKeyFetch() or sqlite3BtreeDataFetch() -** might now be invalid because of a balance() or other change to the -** b-tree. -** 2: The cursor is no longer pointing to the row. The row might have -** been deleted out from under the cursor. +** Use the separate sqlite3BtreeCursorRestore() routine to restore a cursor +** back to where it ought to be if this routine returns true. */ -int sqlite3BtreeCursorHasMoved(BtCursor *pCur, int *pHasMoved){ +int sqlite3BtreeCursorHasMoved(BtCursor *pCur){ + return pCur->eState!=CURSOR_VALID; +} + +/* +** This routine restores a cursor back to its original position after it +** has been moved by some outside activity (such as a btree rebalance or +** a row having been deleted out from under the cursor). +** +** On success, the *pDifferentRow parameter is false if the cursor is left +** pointing at exactly the same row. *pDifferntRow is the row the cursor +** was pointing to has been deleted, forcing the cursor to point to some +** nearby row. +** +** This routine should only be called for a cursor that just returned +** TRUE from sqlite3BtreeCursorHasMoved(). +*/ +int sqlite3BtreeCursorRestore(BtCursor *pCur, int *pDifferentRow){ int rc; - if( pCur->eState==CURSOR_VALID ){ - *pHasMoved = 0; - return SQLITE_OK; - } + assert( pCur!=0 ); + assert( pCur->eState!=CURSOR_VALID ); rc = restoreCursorPosition(pCur); if( rc ){ - *pHasMoved = 2; + *pDifferentRow = 1; return rc; } if( pCur->eState!=CURSOR_VALID || NEVER(pCur->skipNext!=0) ){ - *pHasMoved = 2; + *pDifferentRow = 1; }else{ - *pHasMoved = 1; + *pDifferentRow = 0; } return SQLITE_OK; } @@ -930,47 +970,44 @@ static u8 *findOverflowCell(MemPage *pPage, int iCell){ ** are two versions of this function. btreeParseCell() takes a ** cell index as the second argument and btreeParseCellPtr() ** takes a pointer to the body of the cell as its second argument. -** -** Within this file, the parseCell() macro can be called instead of -** btreeParseCellPtr(). Using some compilers, this will be faster. */ static void btreeParseCellPtr( MemPage *pPage, /* Page containing the cell */ u8 *pCell, /* Pointer to the cell text. */ CellInfo *pInfo /* Fill in this structure */ ){ - u16 n; /* Number bytes in cell content header */ + u8 *pIter; /* For scanning through pCell */ u32 nPayload; /* Number of bytes of cell payload */ assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - - pInfo->pCell = pCell; assert( pPage->leaf==0 || pPage->leaf==1 ); - n = pPage->childPtrSize; - assert( n==4-4*pPage->leaf ); - if( pPage->intKey ){ - if( pPage->hasData ){ - assert( n==0 ); - n = getVarint32(pCell, nPayload); - }else{ - nPayload = 0; - } - n += getVarint(&pCell[n], (u64*)&pInfo->nKey); - pInfo->nData = nPayload; + if( pPage->intKeyLeaf ){ + assert( pPage->childPtrSize==0 ); + pIter = pCell + getVarint32(pCell, nPayload); + pIter += getVarint(pIter, (u64*)&pInfo->nKey); + }else if( pPage->noPayload ){ + assert( pPage->childPtrSize==4 ); + pInfo->nSize = 4 + getVarint(&pCell[4], (u64*)&pInfo->nKey); + pInfo->nPayload = 0; + pInfo->nLocal = 0; + pInfo->iOverflow = 0; + pInfo->pPayload = 0; + return; }else{ - pInfo->nData = 0; - n += getVarint32(&pCell[n], nPayload); + pIter = pCell + pPage->childPtrSize; + pIter += getVarint32(pIter, nPayload); pInfo->nKey = nPayload; } pInfo->nPayload = nPayload; - pInfo->nHeader = n; + pInfo->pPayload = pIter; testcase( nPayload==pPage->maxLocal ); testcase( nPayload==pPage->maxLocal+1 ); - if( likely(nPayload<=pPage->maxLocal) ){ + if( nPayload<=pPage->maxLocal ){ /* This is the (easy) common case where the entire payload fits ** on the local page. No overflow is required. */ - if( (pInfo->nSize = (u16)(n+nPayload))<4 ) pInfo->nSize = 4; + pInfo->nSize = nPayload + (u16)(pIter - pCell); + if( pInfo->nSize<4 ) pInfo->nSize = 4; pInfo->nLocal = (u16)nPayload; pInfo->iOverflow = 0; }else{ @@ -997,18 +1034,16 @@ static void btreeParseCellPtr( }else{ pInfo->nLocal = (u16)minLocal; } - pInfo->iOverflow = (u16)(pInfo->nLocal + n); + pInfo->iOverflow = (u16)(&pInfo->pPayload[pInfo->nLocal] - pCell); pInfo->nSize = pInfo->iOverflow + 4; } } -#define parseCell(pPage, iCell, pInfo) \ - btreeParseCellPtr((pPage), findCell((pPage), (iCell)), (pInfo)) static void btreeParseCell( MemPage *pPage, /* Page containing the cell */ int iCell, /* The cell index. First cell is 0 */ CellInfo *pInfo /* Fill in this structure */ ){ - parseCell(pPage, iCell, pInfo); + btreeParseCellPtr(pPage, findCell(pPage, iCell), pInfo); } /* @@ -1018,8 +1053,9 @@ static void btreeParseCell( ** the space used by the cell pointer. */ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ - u8 *pIter = &pCell[pPage->childPtrSize]; - u32 nSize; + u8 *pIter = pCell + pPage->childPtrSize; /* For looping over bytes of pCell */ + u8 *pEnd; /* End mark for a varint */ + u32 nSize; /* Size value to return */ #ifdef SQLITE_DEBUG /* The value returned by this function should always be the same as @@ -1030,26 +1066,34 @@ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ btreeParseCellPtr(pPage, pCell, &debuginfo); #endif + if( pPage->noPayload ){ + pEnd = &pIter[9]; + while( (*pIter++)&0x80 && pIterchildPtrSize==4 ); + return (u16)(pIter - pCell); + } + nSize = *pIter; + if( nSize>=0x80 ){ + pEnd = &pIter[9]; + nSize &= 0x7f; + do{ + nSize = (nSize<<7) | (*++pIter & 0x7f); + }while( *(pIter)>=0x80 && pIterintKey ){ - u8 *pEnd; - if( pPage->hasData ){ - pIter += getVarint32(pIter, nSize); - }else{ - nSize = 0; - } - /* pIter now points at the 64-bit integer key value, a variable length ** integer. The following block moves pIter to point at the first byte ** past the end of the key value. */ pEnd = &pIter[9]; while( (*pIter++)&0x80 && pItermaxLocal ); testcase( nSize==pPage->maxLocal+1 ); - if( nSize>pPage->maxLocal ){ + if( nSize<=pPage->maxLocal ){ + nSize += (u32)(pIter - pCell); + if( nSize<4 ) nSize = 4; + }else{ int minLocal = pPage->minLocal; nSize = minLocal + (nSize - minLocal) % (pPage->pBt->usableSize - 4); testcase( nSize==pPage->maxLocal ); @@ -1057,16 +1101,9 @@ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ if( nSize>pPage->maxLocal ){ nSize = minLocal; } - nSize += 4; + nSize += 4 + (u16)(pIter - pCell); } - nSize += (u32)(pIter - pCell); - - /* The minimum size of any cell is 4 bytes. */ - if( nSize<4 ){ - nSize = 4; - } - - assert( nSize==debuginfo.nSize ); + assert( nSize==debuginfo.nSize || CORRUPT_DB ); return (u16)nSize; } @@ -1089,7 +1126,6 @@ static void ptrmapPutOvflPtr(MemPage *pPage, u8 *pCell, int *pRC){ if( *pRC ) return; assert( pCell!=0 ); btreeParseCellPtr(pPage, pCell, &info); - assert( (info.nData+(pPage->intKey?0:info.nKey))==info.nPayload ); if( info.iOverflow ){ Pgno ovfl = get4byte(&pCell[info.iOverflow]); ptrmapPut(pPage->pBt, ovfl, PTRMAP_OVERFLOW1, pPage->pgno, pRC); @@ -1103,10 +1139,15 @@ static void ptrmapPutOvflPtr(MemPage *pPage, u8 *pCell, int *pRC){ ** end of the page and all free space is collected into one ** big FreeBlk that occurs in between the header and cell ** pointer array and the cell content area. +** +** EVIDENCE-OF: R-44582-60138 SQLite may from time to time reorganize a +** b-tree page so that there are no freeblocks or fragment bytes, all +** unused bytes are contained in the unallocated space region, and all +** cells are packed tightly at the end of the page. */ static int defragmentPage(MemPage *pPage){ int i; /* Loop counter */ - int pc; /* Address of a i-th cell */ + int pc; /* Address of the i-th cell */ int hdr; /* Offset to the page header */ int size; /* Size of a cell */ int usableSize; /* Number of usable bytes on a page */ @@ -1115,6 +1156,7 @@ static int defragmentPage(MemPage *pPage){ int nCell; /* Number of cells on the page */ unsigned char *data; /* The page data */ unsigned char *temp; /* Temp area for cell content */ + unsigned char *src; /* Source of content */ int iCellFirst; /* First allowable cell index */ int iCellLast; /* Last possible cell index */ @@ -1124,15 +1166,13 @@ static int defragmentPage(MemPage *pPage){ assert( pPage->pBt->usableSize <= SQLITE_MAX_PAGE_SIZE ); assert( pPage->nOverflow==0 ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - temp = sqlite3PagerTempSpace(pPage->pBt->pPager); - data = pPage->aData; + temp = 0; + src = data = pPage->aData; hdr = pPage->hdrOffset; cellOffset = pPage->cellOffset; nCell = pPage->nCell; assert( nCell==get2byte(&data[hdr+3]) ); usableSize = pPage->pBt->usableSize; - cbrk = get2byte(&data[hdr+5]); - memcpy(&temp[cbrk], &data[cbrk], usableSize - cbrk); cbrk = usableSize; iCellFirst = cellOffset + 2*nCell; iCellLast = usableSize - 4; @@ -1151,7 +1191,7 @@ static int defragmentPage(MemPage *pPage){ } #endif assert( pc>=iCellFirst && pc<=iCellLast ); - size = cellSizePtr(pPage, &temp[pc]); + size = cellSizePtr(pPage, &src[pc]); cbrk -= size; #if defined(SQLITE_ENABLE_OVERSIZE_CELL_CHECK) if( cbrk=iCellFirst ); testcase( cbrk+size==usableSize ); testcase( pc+size==usableSize ); - memcpy(&data[cbrk], &temp[pc], size); put2byte(pAddr, cbrk); + if( temp==0 ){ + int x; + if( cbrk==pc ) continue; + temp = sqlite3PagerTempSpace(pPage->pBt->pPager); + x = get2byte(&data[hdr+5]); + memcpy(&temp[x], &data[x], (cbrk+size) - x); + src = temp; + } + memcpy(&data[cbrk], &src[pc], size); } assert( cbrk>=iCellFirst ); put2byte(&data[hdr+5], cbrk); @@ -1181,6 +1229,69 @@ static int defragmentPage(MemPage *pPage){ return SQLITE_OK; } +/* +** Search the free-list on page pPg for space to store a cell nByte bytes in +** size. If one can be found, return a pointer to the space and remove it +** from the free-list. +** +** If no suitable space can be found on the free-list, return NULL. +** +** This function may detect corruption within pPg. If corruption is +** detected then *pRc is set to SQLITE_CORRUPT and NULL is returned. +** +** If a slot of at least nByte bytes is found but cannot be used because +** there are already at least 60 fragmented bytes on the page, return NULL. +** In this case, if pbDefrag parameter is not NULL, set *pbDefrag to true. +*/ +static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc, int *pbDefrag){ + const int hdr = pPg->hdrOffset; + u8 * const aData = pPg->aData; + int iAddr; + int pc; + int usableSize = pPg->pBt->usableSize; + + for(iAddr=hdr+1; (pc = get2byte(&aData[iAddr]))>0; iAddr=pc){ + int size; /* Size of the free slot */ + /* EVIDENCE-OF: R-06866-39125 Freeblocks are always connected in order of + ** increasing offset. */ + if( pc>usableSize-4 || pc=nByte ){ + int x = size - nByte; + testcase( x==4 ); + testcase( x==3 ); + if( x<4 ){ + /* EVIDENCE-OF: R-11498-58022 In a well-formed b-tree page, the total + ** number of bytes in fragments may not exceed 60. */ + if( aData[hdr+7]>=60 ){ + if( pbDefrag ) *pbDefrag = 1; + return 0; + } + /* Remove the slot from the free-list. Update the number of + ** fragmented bytes within the page. */ + memcpy(&aData[iAddr], &aData[pc], 2); + aData[hdr+7] += (u8)x; + }else if( size+pc > usableSize ){ + *pRc = SQLITE_CORRUPT_BKPT; + return 0; + }else{ + /* The slot remains on the free-list. Reduce its size to account + ** for the portion used by the new allocation. */ + put2byte(&aData[pc+2], x); + } + return &aData[pc + x]; + } + } + + return 0; +} + /* ** Allocate nByte bytes of space from within the B-Tree page passed ** as the first argument. Write into *pIdx the index into pPage->aData[] @@ -1197,11 +1308,9 @@ static int defragmentPage(MemPage *pPage){ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ const int hdr = pPage->hdrOffset; /* Local cache of pPage->hdrOffset */ u8 * const data = pPage->aData; /* Local cache of pPage->aData */ - int nFrag; /* Number of fragmented bytes on pPage */ int top; /* First byte of cell content area */ + int rc = SQLITE_OK; /* Integer return code */ int gap; /* First byte of gap between cell pointers and cell content */ - int rc; /* Integer return code */ - int usableSize; /* Usable size of the page */ assert( sqlite3PagerIswriteable(pPage->pDbPage) ); assert( pPage->pBt ); @@ -1209,62 +1318,45 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ assert( nByte>=0 ); /* Minimum cell size is 4 */ assert( pPage->nFree>=nByte ); assert( pPage->nOverflow==0 ); - usableSize = pPage->pBt->usableSize; - assert( nByte < usableSize-8 ); + assert( nByte < (int)(pPage->pBt->usableSize-8) ); - nFrag = data[hdr+7]; assert( pPage->cellOffset == hdr + 12 - 4*pPage->leaf ); gap = pPage->cellOffset + 2*pPage->nCell; + assert( gap<=65536 ); + /* EVIDENCE-OF: R-29356-02391 If the database uses a 65536-byte page size + ** and the reserved space is zero (the usual value for reserved space) + ** then the cell content offset of an empty page wants to be 65536. + ** However, that integer is too large to be stored in a 2-byte unsigned + ** integer, so a value of 0 is used in its place. */ top = get2byteNotZero(&data[hdr+5]); if( gap>top ) return SQLITE_CORRUPT_BKPT; + + /* If there is enough space between gap and top for one more cell pointer + ** array entry offset, and if the freelist is not empty, then search the + ** freelist looking for a free slot big enough to satisfy the request. + */ testcase( gap+2==top ); testcase( gap+1==top ); testcase( gap==top ); - - if( nFrag>=60 ){ - /* Always defragment highly fragmented pages */ - rc = defragmentPage(pPage); + if( gap+2<=top && (data[hdr+1] || data[hdr+2]) ){ + int bDefrag = 0; + u8 *pSpace = pageFindSlot(pPage, nByte, &rc, &bDefrag); if( rc ) return rc; - top = get2byteNotZero(&data[hdr+5]); - }else if( gap+2<=top ){ - /* Search the freelist looking for a free slot big enough to satisfy - ** the request. The allocation is made from the first free slot in - ** the list that is large enough to accommodate it. - */ - int pc, addr; - for(addr=hdr+1; (pc = get2byte(&data[addr]))>0; addr=pc){ - int size; /* Size of the free slot */ - if( pc>usableSize-4 || pc=nByte ){ - int x = size - nByte; - testcase( x==4 ); - testcase( x==3 ); - if( x<4 ){ - /* Remove the slot from the free-list. Update the number of - ** fragmented bytes within the page. */ - memcpy(&data[addr], &data[pc], 2); - data[hdr+7] = (u8)(nFrag + x); - }else if( size+pc > usableSize ){ - return SQLITE_CORRUPT_BKPT; - }else{ - /* The slot remains on the free-list. Reduce its size to account - ** for the portion used by the new allocation. */ - put2byte(&data[pc+2], x); - } - *pIdx = pc + x; - return SQLITE_OK; - } + if( bDefrag ) goto defragment_page; + if( pSpace ){ + assert( pSpace>=data && (pSpace - data)<65536 ); + *pIdx = (int)(pSpace - data); + return SQLITE_OK; } } - /* Check to make sure there is enough space in the gap to satisfy - ** the allocation. If not, defragment. + /* The request could not be fulfilled using a freelist slot. Check + ** to see if defragmentation is necessary. */ testcase( gap+2+nByte==top ); if( gap+2+nByte>top ){ + defragment_page: + assert( pPage->nCell>0 || CORRUPT_DB ); rc = defragmentPage(pPage); if( rc ) return rc; top = get2byteNotZero(&data[hdr+5]); @@ -1287,90 +1379,100 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ /* ** Return a section of the pPage->aData to the freelist. -** The first byte of the new free block is pPage->aDisk[start] -** and the size of the block is "size" bytes. +** The first byte of the new free block is pPage->aData[iStart] +** and the size of the block is iSize bytes. ** -** Most of the effort here is involved in coalesing adjacent -** free blocks into a single big free block. +** Adjacent freeblocks are coalesced. +** +** Note that even though the freeblock list was checked by btreeInitPage(), +** that routine will not detect overlap between cells or freeblocks. Nor +** does it detect cells or freeblocks that encrouch into the reserved bytes +** at the end of the page. So do additional corruption checks inside this +** routine and return SQLITE_CORRUPT if any problems are found. */ -static int freeSpace(MemPage *pPage, int start, int size){ - int addr, pbegin, hdr; - int iLast; /* Largest possible freeblock offset */ - unsigned char *data = pPage->aData; +static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ + u16 iPtr; /* Address of ptr to next freeblock */ + u16 iFreeBlk; /* Address of the next freeblock */ + u8 hdr; /* Page header size. 0 or 100 */ + u8 nFrag = 0; /* Reduction in fragmentation */ + u16 iOrigSize = iSize; /* Original value of iSize */ + u32 iLast = pPage->pBt->usableSize-4; /* Largest possible freeblock offset */ + u32 iEnd = iStart + iSize; /* First byte past the iStart buffer */ + unsigned char *data = pPage->aData; /* Page content */ assert( pPage->pBt!=0 ); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); - assert( start>=pPage->hdrOffset+6+pPage->childPtrSize ); - assert( (start + size) <= (int)pPage->pBt->usableSize ); + assert( iStart>=pPage->hdrOffset+6+pPage->childPtrSize ); + assert( CORRUPT_DB || iEnd <= pPage->pBt->usableSize ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - assert( size>=0 ); /* Minimum cell size is 4 */ + assert( iSize>=4 ); /* Minimum cell size is 4 */ + assert( iStart<=iLast ); + /* Overwrite deleted information with zeros when the secure_delete + ** option is enabled */ if( pPage->pBt->btsFlags & BTS_SECURE_DELETE ){ - /* Overwrite deleted information with zeros when the secure_delete - ** option is enabled */ - memset(&data[start], 0, size); + memset(&data[iStart], 0, iSize); } - /* Add the space back into the linked list of freeblocks. Note that - ** even though the freeblock list was checked by btreeInitPage(), - ** btreeInitPage() did not detect overlapping cells or - ** freeblocks that overlapped cells. Nor does it detect when the - ** cell content area exceeds the value in the page header. If these - ** situations arise, then subsequent insert operations might corrupt - ** the freelist. So we do need to check for corruption while scanning - ** the freelist. + /* The list of freeblocks must be in ascending order. Find the + ** spot on the list where iStart should be inserted. */ hdr = pPage->hdrOffset; - addr = hdr + 1; - iLast = pPage->pBt->usableSize - 4; - assert( start<=iLast ); - while( (pbegin = get2byte(&data[addr]))0 ){ - if( pbegin0 && iFreeBlkiLast ){ - return SQLITE_CORRUPT_BKPT; - } - assert( pbegin>addr || pbegin==0 ); - put2byte(&data[addr], start); - put2byte(&data[start], pbegin); - put2byte(&data[start+2], size); - pPage->nFree = pPage->nFree + (u16)size; - - /* Coalesce adjacent free blocks */ - addr = hdr + 1; - while( (pbegin = get2byte(&data[addr]))>0 ){ - int pnext, psize, x; - assert( pbegin>addr ); - assert( pbegin <= (int)pPage->pBt->usableSize-4 ); - pnext = get2byte(&data[pbegin]); - psize = get2byte(&data[pbegin+2]); - if( pbegin + psize + 3 >= pnext && pnext>0 ){ - int frag = pnext - (pbegin+psize); - if( (frag<0) || (frag>(int)data[hdr+7]) ){ - return SQLITE_CORRUPT_BKPT; + if( iFreeBlk>iLast ) return SQLITE_CORRUPT_BKPT; + assert( iFreeBlk>iPtr || iFreeBlk==0 ); + + /* At this point: + ** iFreeBlk: First freeblock after iStart, or zero if none + ** iPtr: The address of a pointer iFreeBlk + ** + ** Check to see if iFreeBlk should be coalesced onto the end of iStart. + */ + if( iFreeBlk && iEnd+3>=iFreeBlk ){ + nFrag = iFreeBlk - iEnd; + if( iEnd>iFreeBlk ) return SQLITE_CORRUPT_BKPT; + iEnd = iFreeBlk + get2byte(&data[iFreeBlk+2]); + iSize = iEnd - iStart; + iFreeBlk = get2byte(&data[iFreeBlk]); + } + + /* If iPtr is another freeblock (that is, if iPtr is not the freelist + ** pointer in the page header) then check to see if iStart should be + ** coalesced onto the end of iPtr. + */ + if( iPtr>hdr+1 ){ + int iPtrEnd = iPtr + get2byte(&data[iPtr+2]); + if( iPtrEnd+3>=iStart ){ + if( iPtrEnd>iStart ) return SQLITE_CORRUPT_BKPT; + nFrag += iStart - iPtrEnd; + iSize = iEnd - iPtr; + iStart = iPtr; } - data[hdr+7] -= (u8)frag; - x = get2byte(&data[pnext]); - put2byte(&data[pbegin], x); - x = pnext + get2byte(&data[pnext+2]) - pbegin; - put2byte(&data[pbegin+2], x); - }else{ - addr = pbegin; } + if( nFrag>data[hdr+7] ) return SQLITE_CORRUPT_BKPT; + data[hdr+7] -= nFrag; } - - /* If the cell content area begins with a freeblock, remove it. */ - if( data[hdr+1]==data[hdr+5] && data[hdr+2]==data[hdr+6] ){ - int top; - pbegin = get2byte(&data[hdr+1]); - memcpy(&data[hdr+1], &data[pbegin], 2); - top = get2byte(&data[hdr+5]) + get2byte(&data[pbegin+2]); - put2byte(&data[hdr+5], top); + if( iStart==get2byte(&data[hdr+5]) ){ + /* The new freeblock is at the beginning of the cell content area, + ** so just extend the cell content area rather than create another + ** freelist entry */ + if( iPtr!=hdr+1 ) return SQLITE_CORRUPT_BKPT; + put2byte(&data[hdr+1], iFreeBlk); + put2byte(&data[hdr+5], iEnd); + }else{ + /* Insert the new freeblock into the freelist */ + put2byte(&data[iPtr], iStart); + put2byte(&data[iStart], iFreeBlk); + put2byte(&data[iStart+2], iSize); } - assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + pPage->nFree += iOrigSize; return SQLITE_OK; } @@ -1396,16 +1498,32 @@ static int decodeFlags(MemPage *pPage, int flagByte){ pPage->childPtrSize = 4-4*pPage->leaf; pBt = pPage->pBt; if( flagByte==(PTF_LEAFDATA | PTF_INTKEY) ){ + /* EVIDENCE-OF: R-03640-13415 A value of 5 means the page is an interior + ** table b-tree page. */ + assert( (PTF_LEAFDATA|PTF_INTKEY)==5 ); + /* EVIDENCE-OF: R-20501-61796 A value of 13 means the page is a leaf + ** table b-tree page. */ + assert( (PTF_LEAFDATA|PTF_INTKEY|PTF_LEAF)==13 ); pPage->intKey = 1; - pPage->hasData = pPage->leaf; + pPage->intKeyLeaf = pPage->leaf; + pPage->noPayload = !pPage->leaf; pPage->maxLocal = pBt->maxLeaf; pPage->minLocal = pBt->minLeaf; }else if( flagByte==PTF_ZERODATA ){ + /* EVIDENCE-OF: R-27225-53936 A value of 2 means the page is an interior + ** index b-tree page. */ + assert( (PTF_ZERODATA)==2 ); + /* EVIDENCE-OF: R-16571-11615 A value of 10 means the page is a leaf + ** index b-tree page. */ + assert( (PTF_ZERODATA|PTF_LEAF)==10 ); pPage->intKey = 0; - pPage->hasData = 0; + pPage->intKeyLeaf = 0; + pPage->noPayload = 0; pPage->maxLocal = pBt->maxLocal; pPage->minLocal = pBt->minLocal; }else{ + /* EVIDENCE-OF: R-47608-56469 Any other value for the b-tree page type is + ** an error. */ return SQLITE_CORRUPT_BKPT; } pPage->max1bytePayload = pBt->max1bytePayload; @@ -1445,21 +1563,33 @@ static int btreeInitPage(MemPage *pPage){ hdr = pPage->hdrOffset; data = pPage->aData; + /* EVIDENCE-OF: R-28594-02890 The one-byte flag at offset 0 indicating + ** the b-tree page type. */ if( decodeFlags(pPage, data[hdr]) ) return SQLITE_CORRUPT_BKPT; assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); pPage->maskPage = (u16)(pBt->pageSize - 1); pPage->nOverflow = 0; usableSize = pBt->usableSize; - pPage->cellOffset = cellOffset = hdr + 12 - 4*pPage->leaf; + pPage->cellOffset = cellOffset = hdr + 8 + pPage->childPtrSize; pPage->aDataEnd = &data[usableSize]; pPage->aCellIdx = &data[cellOffset]; + /* EVIDENCE-OF: R-58015-48175 The two-byte integer at offset 5 designates + ** the start of the cell content area. A zero value for this integer is + ** interpreted as 65536. */ top = get2byteNotZero(&data[hdr+5]); + /* EVIDENCE-OF: R-37002-32774 The two-byte integer at offset 3 gives the + ** number of cells on the page. */ pPage->nCell = get2byte(&data[hdr+3]); if( pPage->nCell>MX_CELL(pBt) ){ /* To many cells for a single page. The page must be corrupt */ return SQLITE_CORRUPT_BKPT; } testcase( pPage->nCell==MX_CELL(pBt) ); + /* EVIDENCE-OF: R-24089-57979 If a page contains no cells (which is only + ** possible for a root page of a table that contains no rows) then the + ** offset to the cell content area will equal the page size minus the + ** bytes of reserved space. */ + assert( pPage->nCell>0 || top==usableSize || CORRUPT_DB ); /* A malformed database page might cause us to read past the end ** of page when parsing a cell. @@ -1493,13 +1623,20 @@ static int btreeInitPage(MemPage *pPage){ } #endif - /* Compute the total free space on the page */ + /* Compute the total free space on the page + ** EVIDENCE-OF: R-23588-34450 The two-byte integer at offset 1 gives the + ** start of the first freeblock on the page, or is zero if there are no + ** freeblocks. */ pc = get2byte(&data[hdr+1]); - nFree = data[hdr+7] + top; + nFree = data[hdr+7] + top; /* Init nFree to non-freeblock free space */ while( pc>0 ){ u16 next, size; if( pciCellLast ){ - /* Start of free block is off the page */ + /* EVIDENCE-OF: R-55530-52930 In a well-formed b-tree page, there will + ** always be at least one cell before the first freeblock. + ** + ** Or, the freeblock is off the end of the page + */ return SQLITE_CORRUPT_BKPT; } next = get2byte(&data[pc]); @@ -1632,7 +1769,7 @@ static Pgno btreePagecount(BtShared *pBt){ u32 sqlite3BtreeLastPage(Btree *p){ assert( sqlite3BtreeHoldsMutex(p) ); assert( ((p->pBt->nPage)&0x8000000)==0 ); - return (int)btreePagecount(p->pBt); + return btreePagecount(p->pBt); } /* @@ -1905,6 +2042,9 @@ int sqlite3BtreeOpen( #ifdef SQLITE_SECURE_DELETE pBt->btsFlags |= BTS_SECURE_DELETE; #endif + /* EVIDENCE-OF: R-51873-39618 The page size for a database file is + ** determined by the 2-byte integer located at an offset of 16 bytes from + ** the beginning of the database file. */ pBt->pageSize = (zDbHeader[16]<<8) | (zDbHeader[17]<<16); if( pBt->pageSize<512 || pBt->pageSize>SQLITE_MAX_PAGE_SIZE || ((pBt->pageSize-1)&pBt->pageSize)!=0 ){ @@ -1923,6 +2063,9 @@ int sqlite3BtreeOpen( #endif nReserve = 0; }else{ + /* EVIDENCE-OF: R-37497-42412 The size of the reserved region is + ** determined by the one-byte unsigned integer found at an offset of 20 + ** into the database file header. */ nReserve = zDbHeader[20]; pBt->btsFlags |= BTS_PAGESIZE_FIXED; #ifndef SQLITE_OMIT_AUTOVACUUM @@ -2057,7 +2200,8 @@ static int removeFromSharingList(BtShared *pBt){ /* ** Make sure pBt->pTmpSpace points to an allocation of -** MX_CELL_SIZE(pBt) bytes. +** MX_CELL_SIZE(pBt) bytes with a 4-byte prefix for a left-child +** pointer. */ static void allocateTempSpace(BtShared *pBt){ if( !pBt->pTmpSpace ){ @@ -2072,8 +2216,16 @@ static void allocateTempSpace(BtShared *pBt){ ** it into a database page. This is not actually a problem, but it ** does cause a valgrind error when the 1 or 2 bytes of unitialized ** data is passed to system call write(). So to avoid this error, - ** zero the first 4 bytes of temp space here. */ - if( pBt->pTmpSpace ) memset(pBt->pTmpSpace, 0, 4); + ** zero the first 4 bytes of temp space here. + ** + ** Also: Provide four bytes of initialized space before the + ** beginning of pTmpSpace as an area available to prepend the + ** left-child pointer to the beginning of a cell. + */ + if( pBt->pTmpSpace ){ + memset(pBt->pTmpSpace, 0, 8); + pBt->pTmpSpace += 4; + } } } @@ -2081,8 +2233,11 @@ static void allocateTempSpace(BtShared *pBt){ ** Free the pBt->pTmpSpace allocation */ static void freeTempSpace(BtShared *pBt){ - sqlite3PageFree( pBt->pTmpSpace); - pBt->pTmpSpace = 0; + if( pBt->pTmpSpace ){ + pBt->pTmpSpace -= 4; + sqlite3PageFree(pBt->pTmpSpace); + pBt->pTmpSpace = 0; + } } /* @@ -2108,7 +2263,7 @@ int sqlite3BtreeClose(Btree *p){ ** The call to sqlite3BtreeRollback() drops any table-locks held by ** this handle. */ - sqlite3BtreeRollback(p, SQLITE_OK); + sqlite3BtreeRollback(p, SQLITE_OK, 0); sqlite3BtreeLeave(p); /* If there are still other outstanding references to the shared-btree @@ -2420,6 +2575,9 @@ static int lockBtree(BtShared *pBt){ u32 usableSize; u8 *page1 = pPage1->aData; rc = SQLITE_NOTADB; + /* EVIDENCE-OF: R-43737-39999 Every valid SQLite database file begins + ** with the following 16 bytes (in hex): 53 51 4c 69 74 65 20 66 6f 72 6d + ** 61 74 20 33 00. */ if( memcmp(page1, zMagicHeader, 16)!=0 ){ goto page1_init_failed; } @@ -2460,15 +2618,21 @@ static int lockBtree(BtShared *pBt){ } #endif - /* The maximum embedded fraction must be exactly 25%. And the minimum - ** embedded fraction must be 12.5% for both leaf-data and non-leaf-data. + /* EVIDENCE-OF: R-15465-20813 The maximum and minimum embedded payload + ** fractions and the leaf payload fraction values must be 64, 32, and 32. + ** ** The original design allowed these amounts to vary, but as of ** version 3.6.0, we require them to be fixed. */ if( memcmp(&page1[21], "\100\040\040",3)!=0 ){ goto page1_init_failed; } + /* EVIDENCE-OF: R-51873-39618 The page size for a database file is + ** determined by the 2-byte integer located at an offset of 16 bytes from + ** the beginning of the database file. */ pageSize = (page1[16]<<8) | (page1[17]<<16); + /* EVIDENCE-OF: R-25008-21688 The size of a page is a power of two + ** between 512 and 65536 inclusive. */ if( ((pageSize-1)&pageSize)!=0 || pageSize>SQLITE_MAX_PAGE_SIZE || pageSize<=256 @@ -2476,6 +2640,13 @@ static int lockBtree(BtShared *pBt){ goto page1_init_failed; } assert( (pageSize & 7)==0 ); + /* EVIDENCE-OF: R-59310-51205 The "reserved space" size in the 1-byte + ** integer at offset 20 is the number of bytes of space at the end of + ** each page to reserve for extensions. + ** + ** EVIDENCE-OF: R-37497-42412 The size of the reserved region is + ** determined by the one-byte unsigned integer found at an offset of 20 + ** into the database file header. */ usableSize = pageSize - page1[20]; if( (u32)pageSize!=pBt->pageSize ){ /* After reading the first page of the database assuming a page size @@ -2496,6 +2667,9 @@ static int lockBtree(BtShared *pBt){ rc = SQLITE_CORRUPT_BKPT; goto page1_init_failed; } + /* EVIDENCE-OF: R-28312-64704 However, the usable size is not allowed to + ** be less than 480. In other words, if the page size is 512, then the + ** reserved space size cannot exceed 32. */ if( usableSize<480 ){ goto page1_init_failed; } @@ -2550,7 +2724,7 @@ page1_init_failed: ** false then all cursors are counted. ** ** For the purposes of this routine, a cursor is any cursor that -** is capable of reading or writing to the databse. Cursors that +** is capable of reading or writing to the database. Cursors that ** have been tripped into the CURSOR_FAULT state are not counted. */ static int countValidCursors(BtShared *pBt, int wrOnly){ @@ -2576,11 +2750,11 @@ static void unlockBtreeIfUnused(BtShared *pBt){ assert( sqlite3_mutex_held(pBt->mutex) ); assert( countValidCursors(pBt,0)==0 || pBt->inTransaction>TRANS_NONE ); if( pBt->inTransaction==TRANS_NONE && pBt->pPage1!=0 ){ - assert( pBt->pPage1->aData ); + MemPage *pPage1 = pBt->pPage1; + assert( pPage1->aData ); assert( sqlite3PagerRefcount(pBt->pPager)==1 ); - assert( pBt->pPage1->aData ); - releasePage(pBt->pPage1); pBt->pPage1 = 0; + releasePage(pPage1); } } @@ -3014,7 +3188,7 @@ static int allocateBtreePage(BtShared *, MemPage **, Pgno *, Pgno, u8); ** calling this function again), return SQLITE_DONE. Or, if an error ** occurs, return some other error code. ** -** More specificly, this function attempts to re-organize the database so +** More specifically, this function attempts to re-organize the database so ** that the last page of the file currently in use is no longer in use. ** ** Parameter nFin is the number of pages that this database would contain @@ -3022,7 +3196,7 @@ static int allocateBtreePage(BtShared *, MemPage **, Pgno *, Pgno, u8); ** ** If the bCommit parameter is non-zero, this function assumes that the ** caller will keep calling incrVacuumStep() until it returns SQLITE_DONE -** or an error. bCommit is passed true for an auto-vacuum-on-commmit +** or an error. bCommit is passed true for an auto-vacuum-on-commit ** operation, or false for an incremental vacuum. */ static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg, int bCommit){ @@ -3376,6 +3550,7 @@ int sqlite3BtreeCommitPhaseTwo(Btree *p, int bCleanup){ sqlite3BtreeLeave(p); return rc; } + p->iDataVersion--; /* Compensate for pPager->iDataVersion++; */ pBt->inTransaction = TRANS_READ; btreeClearHasContent(pBt); } @@ -3401,60 +3576,91 @@ int sqlite3BtreeCommit(Btree *p){ /* ** This routine sets the state to CURSOR_FAULT and the error -** code to errCode for every cursor on BtShared that pBtree -** references. +** code to errCode for every cursor on any BtShared that pBtree +** references. Or if the writeOnly flag is set to 1, then only +** trip write cursors and leave read cursors unchanged. ** -** Every cursor is tripped, including cursors that belong -** to other database connections that happen to be sharing -** the cache with pBtree. +** Every cursor is a candidate to be tripped, including cursors +** that belong to other database connections that happen to be +** sharing the cache with pBtree. ** -** This routine gets called when a rollback occurs. -** All cursors using the same cache must be tripped -** to prevent them from trying to use the btree after -** the rollback. The rollback may have deleted tables -** or moved root pages, so it is not sufficient to -** save the state of the cursor. The cursor must be -** invalidated. +** This routine gets called when a rollback occurs. If the writeOnly +** flag is true, then only write-cursors need be tripped - read-only +** cursors save their current positions so that they may continue +** following the rollback. Or, if writeOnly is false, all cursors are +** tripped. In general, writeOnly is false if the transaction being +** rolled back modified the database schema. In this case b-tree root +** pages may be moved or deleted from the database altogether, making +** it unsafe for read cursors to continue. +** +** If the writeOnly flag is true and an error is encountered while +** saving the current position of a read-only cursor, all cursors, +** including all read-cursors are tripped. +** +** SQLITE_OK is returned if successful, or if an error occurs while +** saving a cursor position, an SQLite error code. */ -void sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode){ +int sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode, int writeOnly){ BtCursor *p; - if( pBtree==0 ) return; - sqlite3BtreeEnter(pBtree); - for(p=pBtree->pBt->pCursor; p; p=p->pNext){ - int i; - sqlite3BtreeClearCursor(p); - p->eState = CURSOR_FAULT; - p->skipNext = errCode; - for(i=0; i<=p->iPage; i++){ - releasePage(p->apPage[i]); - p->apPage[i] = 0; + int rc = SQLITE_OK; + + assert( (writeOnly==0 || writeOnly==1) && BTCF_WriteFlag==1 ); + if( pBtree ){ + sqlite3BtreeEnter(pBtree); + for(p=pBtree->pBt->pCursor; p; p=p->pNext){ + int i; + if( writeOnly && (p->curFlags & BTCF_WriteFlag)==0 ){ + if( p->eState==CURSOR_VALID ){ + rc = saveCursorPosition(p); + if( rc!=SQLITE_OK ){ + (void)sqlite3BtreeTripAllCursors(pBtree, rc, 0); + break; + } + } + }else{ + sqlite3BtreeClearCursor(p); + p->eState = CURSOR_FAULT; + p->skipNext = errCode; + } + for(i=0; i<=p->iPage; i++){ + releasePage(p->apPage[i]); + p->apPage[i] = 0; + } } + sqlite3BtreeLeave(pBtree); } - sqlite3BtreeLeave(pBtree); + return rc; } /* -** Rollback the transaction in progress. All cursors will be -** invalided by this operation. Any attempt to use a cursor -** that was open at the beginning of this operation will result -** in an error. +** Rollback the transaction in progress. +** +** If tripCode is not SQLITE_OK then cursors will be invalidated (tripped). +** Only write cursors are tripped if writeOnly is true but all cursors are +** tripped if writeOnly is false. Any attempt to use +** a tripped cursor will result in an error. ** ** This will release the write lock on the database file. If there ** are no active cursors, it also releases the read lock. */ -int sqlite3BtreeRollback(Btree *p, int tripCode){ +int sqlite3BtreeRollback(Btree *p, int tripCode, int writeOnly){ int rc; BtShared *pBt = p->pBt; MemPage *pPage1; + assert( writeOnly==1 || writeOnly==0 ); + assert( tripCode==SQLITE_ABORT_ROLLBACK || tripCode==SQLITE_OK ); sqlite3BtreeEnter(p); if( tripCode==SQLITE_OK ){ rc = tripCode = saveAllCursors(pBt, 0, 0); + if( rc ) writeOnly = 0; }else{ rc = SQLITE_OK; } if( tripCode ){ - sqlite3BtreeTripAllCursors(p, tripCode); + int rc2 = sqlite3BtreeTripAllCursors(p, tripCode, writeOnly); + assert( rc==SQLITE_OK || (writeOnly==0 && rc2==SQLITE_OK) ); + if( rc2!=SQLITE_OK ) rc = rc2; } btreeIntegrity(p); @@ -3489,7 +3695,7 @@ int sqlite3BtreeRollback(Btree *p, int tripCode){ } /* -** Start a statement subtransaction. The subtransaction can can be rolled +** Start a statement subtransaction. The subtransaction can be rolled ** back independently of the main transaction. You must start a transaction ** before starting a subtransaction. The subtransaction is ended automatically ** if the main transaction commits or rolls back. @@ -3621,6 +3827,10 @@ static int btreeCursor( if( NEVER(wrFlag && (pBt->btsFlags & BTS_READ_ONLY)!=0) ){ return SQLITE_READONLY; } + if( wrFlag ){ + allocateTempSpace(pBt); + if( pBt->pTmpSpace==0 ) return SQLITE_NOMEM; + } if( iTable==1 && btreePagecount(pBt)==0 ){ assert( wrFlag==0 ); iTable = 0; @@ -3704,7 +3914,7 @@ int sqlite3BtreeCloseCursor(BtCursor *pCur){ releasePage(pCur->apPage[i]); } unlockBtreeIfUnused(pBt); - sqlite3DbFree(pBtree->db, pCur->aOverflow); + sqlite3_free(pCur->aOverflow); /* sqlite3_free(pCur); */ sqlite3BtreeLeave(pBtree); } @@ -3723,7 +3933,7 @@ int sqlite3BtreeCloseCursor(BtCursor *pCur){ ** compiler to crash when getCellInfo() is implemented as a macro. ** But there is a measureable speed advantage to using the macro on gcc ** (when less compiler optimizations like -Os or -O0 are used and the -** compiler is not doing agressive inlining.) So we use a real function +** compiler is not doing aggressive inlining.) So we use a real function ** for MSVC and a macro for everything else. Ticket #2457. */ #ifndef NDEBUG @@ -3785,13 +3995,9 @@ int sqlite3BtreeCursorIsValid(BtCursor *pCur){ */ int sqlite3BtreeKeySize(BtCursor *pCur, i64 *pSize){ assert( cursorHoldsMutex(pCur) ); - assert( pCur->eState==CURSOR_INVALID || pCur->eState==CURSOR_VALID ); - if( pCur->eState!=CURSOR_VALID ){ - *pSize = 0; - }else{ - getCellInfo(pCur); - *pSize = pCur->info.nKey; - } + assert( pCur->eState==CURSOR_VALID ); + getCellInfo(pCur); + *pSize = pCur->info.nKey; return SQLITE_OK; } @@ -3810,8 +4016,9 @@ int sqlite3BtreeKeySize(BtCursor *pCur, i64 *pSize){ int sqlite3BtreeDataSize(BtCursor *pCur, u32 *pSize){ assert( cursorHoldsMutex(pCur) ); assert( pCur->eState==CURSOR_VALID ); + assert( pCur->apPage[pCur->iPage]->intKeyLeaf==1 ); getCellInfo(pCur); - *pSize = pCur->info.nData; + *pSize = pCur->info.nPayload; return SQLITE_OK; } @@ -3940,7 +4147,7 @@ static int copyPayload( ** ** If the current cursor entry uses one or more overflow pages and the ** eOp argument is not 2, this function may allocate space for and lazily -** popluates the overflow page-list cache array (BtCursor.aOverflow). +** populates the overflow page-list cache array (BtCursor.aOverflow). ** Subsequent calls use this cache to make seeking to the supplied offset ** more efficient. ** @@ -3962,30 +4169,28 @@ static int accessPayload( ){ unsigned char *aPayload; int rc = SQLITE_OK; - u32 nKey; int iIdx = 0; MemPage *pPage = pCur->apPage[pCur->iPage]; /* Btree page of current entry */ BtShared *pBt = pCur->pBt; /* Btree this cursor belongs to */ #ifdef SQLITE_DIRECT_OVERFLOW_READ - int bEnd; /* True if reading to end of data */ + unsigned char * const pBufStart = pBuf; + int bEnd; /* True if reading to end of data */ #endif assert( pPage ); assert( pCur->eState==CURSOR_VALID ); assert( pCur->aiIdx[pCur->iPage]nCell ); assert( cursorHoldsMutex(pCur) ); - assert( eOp!=2 || offset==0 ); /* Always start from beginning for eOp==2 */ + assert( eOp!=2 || offset==0 ); /* Always start from beginning for eOp==2 */ getCellInfo(pCur); - aPayload = pCur->info.pCell + pCur->info.nHeader; - nKey = (pPage->intKey ? 0 : (int)pCur->info.nKey); + aPayload = pCur->info.pPayload; #ifdef SQLITE_DIRECT_OVERFLOW_READ - bEnd = (offset+amt==nKey+pCur->info.nData); + bEnd = offset+amt==pCur->info.nPayload; #endif + assert( offset+amt <= pCur->info.nPayload ); - if( NEVER(offset+amt > nKey+pCur->info.nData) - || &aPayload[pCur->info.nLocal] > &pPage->aData[pBt->usableSize] - ){ + if( &aPayload[pCur->info.nLocal] > &pPage->aData[pBt->usableSize] ){ /* Trying to read or write past the end of the data is an error */ return SQLITE_CORRUPT_BKPT; } @@ -4004,6 +4209,7 @@ static int accessPayload( offset -= pCur->info.nLocal; } + if( rc==SQLITE_OK && amt>0 ){ const u32 ovflSize = pBt->usableSize - 4; /* Bytes content per ovfl page */ Pgno nextPage; @@ -4021,8 +4227,8 @@ static int accessPayload( if( eOp!=2 && (pCur->curFlags & BTCF_ValidOvfl)==0 ){ int nOvfl = (pCur->info.nPayload-pCur->info.nLocal+ovflSize-1)/ovflSize; if( nOvfl>pCur->nOvflAlloc ){ - Pgno *aNew = (Pgno*)sqlite3DbRealloc( - pCur->pBtree->db, pCur->aOverflow, nOvfl*2*sizeof(Pgno) + Pgno *aNew = (Pgno*)sqlite3Realloc( + pCur->aOverflow, nOvfl*2*sizeof(Pgno) ); if( aNew==0 ){ rc = SQLITE_NOMEM; @@ -4041,7 +4247,9 @@ static int accessPayload( ** entry for the first required overflow page is valid, skip ** directly to it. */ - if( (pCur->curFlags & BTCF_ValidOvfl)!=0 && pCur->aOverflow[offset/ovflSize] ){ + if( (pCur->curFlags & BTCF_ValidOvfl)!=0 + && pCur->aOverflow[offset/ovflSize] + ){ iIdx = (offset/ovflSize); nextPage = pCur->aOverflow[iIdx]; offset = (offset%ovflSize); @@ -4067,6 +4275,7 @@ static int accessPayload( */ assert( eOp!=2 ); assert( pCur->curFlags & BTCF_ValidOvfl ); + assert( pCur->pBtree->db==pBt->db ); if( pCur->aOverflow[iIdx+1] ){ nextPage = pCur->aOverflow[iIdx+1]; }else{ @@ -4094,6 +4303,7 @@ static int accessPayload( ** 4) there is no open write-transaction, and ** 5) the database is not a WAL database, ** 6) all data from the page is being read. + ** 7) at least 4 bytes have already been read into the output buffer ** ** then data can be read directly from the database file into the ** output buffer, bypassing the page-cache altogether. This speeds @@ -4105,9 +4315,11 @@ static int accessPayload( && pBt->inTransaction==TRANS_READ /* (4) */ && (fd = sqlite3PagerFile(pBt->pPager))->pMethods /* (3) */ && pBt->pPage1->aData[19]==0x01 /* (5) */ + && &pBuf[-4]>=pBufStart /* (7) */ ){ u8 aSave[4]; u8 *aWrite = &pBuf[-4]; + assert( aWrite>=pBufStart ); /* hence (7) */ memcpy(aSave, aWrite, 4); rc = sqlite3OsRead(fd, aWrite, a+4, (i64)pBt->pageSize*(nextPage-1)); nextPage = get4byte(aWrite); @@ -4142,7 +4354,7 @@ static int accessPayload( /* ** Read part of the key associated with cursor pCur. Exactly -** "amt" bytes will be transfered into pBuf[]. The transfer +** "amt" bytes will be transferred into pBuf[]. The transfer ** begins at "offset". ** ** The caller must ensure that pCur is pointing to a valid row @@ -4219,7 +4431,7 @@ static const void *fetchPayload( assert( pCur->aiIdx[pCur->iPage]apPage[pCur->iPage]->nCell ); assert( pCur->info.nSize>0 ); *pAmt = pCur->info.nLocal; - return (void*)(pCur->info.pCell + pCur->info.nHeader); + return (void*)pCur->info.pPayload; } @@ -4462,17 +4674,16 @@ static int moveToRightmost(BtCursor *pCur){ assert( cursorHoldsMutex(pCur) ); assert( pCur->eState==CURSOR_VALID ); - while( rc==SQLITE_OK && !(pPage = pCur->apPage[pCur->iPage])->leaf ){ + while( !(pPage = pCur->apPage[pCur->iPage])->leaf ){ pgno = get4byte(&pPage->aData[pPage->hdrOffset+8]); pCur->aiIdx[pCur->iPage] = pPage->nCell; rc = moveToChild(pCur, pgno); + if( rc ) return rc; } - if( rc==SQLITE_OK ){ - pCur->aiIdx[pCur->iPage] = pPage->nCell-1; - pCur->info.nSize = 0; - pCur->curFlags &= ~BTCF_ValidNKey; - } - return rc; + pCur->aiIdx[pCur->iPage] = pPage->nCell-1; + assert( pCur->info.nSize==0 ); + assert( (pCur->curFlags & BTCF_ValidNKey)==0 ); + return SQLITE_OK; } /* Move the cursor to the first entry in the table. Return SQLITE_OK @@ -4603,7 +4814,7 @@ int sqlite3BtreeMovetoUnpacked( if( pIdxKey ){ xRecordCompare = sqlite3VdbeFindCompare(pIdxKey); - pIdxKey->isCorrupt = 0; + pIdxKey->errCode = 0; assert( pIdxKey->default_rc==1 || pIdxKey->default_rc==0 || pIdxKey->default_rc==-1 @@ -4648,7 +4859,7 @@ int sqlite3BtreeMovetoUnpacked( for(;;){ i64 nCellKey; pCell = findCell(pPage, idx) + pPage->childPtrSize; - if( pPage->hasData ){ + if( pPage->intKeyLeaf ){ while( 0x80 <= *(pCell++) ){ if( pCell>=pPage->aDataEnd ) return SQLITE_CORRUPT_BKPT; } @@ -4696,14 +4907,14 @@ int sqlite3BtreeMovetoUnpacked( ** single byte varint and the record fits entirely on the main ** b-tree page. */ testcase( pCell+nCell+1==pPage->aDataEnd ); - c = xRecordCompare(nCell, (void*)&pCell[1], pIdxKey, 0); + c = xRecordCompare(nCell, (void*)&pCell[1], pIdxKey); }else if( !(pCell[1] & 0x80) && (nCell = ((nCell&0x7f)<<7) + pCell[1])<=pPage->maxLocal ){ /* The record-size field is a 2 byte varint and the record ** fits entirely on the main b-tree page. */ testcase( pCell+nCell+2==pPage->aDataEnd ); - c = xRecordCompare(nCell, (void*)&pCell[2], pIdxKey, 0); + c = xRecordCompare(nCell, (void*)&pCell[2], pIdxKey); }else{ /* The record flows over onto one or more overflow pages. In ** this case the whole cell needs to be parsed, a buffer allocated @@ -4724,10 +4935,13 @@ int sqlite3BtreeMovetoUnpacked( sqlite3_free(pCellKey); goto moveto_finish; } - c = xRecordCompare(nCell, pCellKey, pIdxKey, 0); + c = xRecordCompare(nCell, pCellKey, pIdxKey); sqlite3_free(pCellKey); } - assert( pIdxKey->isCorrupt==0 || c==0 ); + assert( + (pIdxKey->errCode!=SQLITE_CORRUPT || c==0) + && (pIdxKey->errCode!=SQLITE_NOMEM || pCur->pBtree->db->mallocFailed) + ); if( c<0 ){ lwr = idx+1; }else if( c>0 ){ @@ -4737,7 +4951,7 @@ int sqlite3BtreeMovetoUnpacked( *pRes = 0; rc = SQLITE_OK; pCur->aiIdx[pCur->iPage] = (u16)idx; - if( pIdxKey->isCorrupt ) rc = SQLITE_CORRUPT; + if( pIdxKey->errCode ) rc = SQLITE_CORRUPT; goto moveto_finish; } if( lwr>upr ) break; @@ -4792,6 +5006,12 @@ int sqlite3BtreeEof(BtCursor *pCur){ ** was already pointing to the last entry in the database before ** this routine was called, then set *pRes=1. ** +** The main entry point is sqlite3BtreeNext(). That routine is optimized +** for the common case of merely incrementing the cell counter BtCursor.aiIdx +** to the next cell on the current page. The (slower) btreeNext() helper +** routine is called when it is necessary to move to a different page or +** to restore the cursor. +** ** The calling function will set *pRes to 0 or 1. The initial *pRes value ** will be 1 if the cursor being stepped corresponds to an SQL index and ** if this routine could have been skipped if that SQL index had been @@ -4801,20 +5021,18 @@ int sqlite3BtreeEof(BtCursor *pCur){ ** SQLite btree implementation does not. (Note that the comdb2 btree ** implementation does use this hint, however.) */ -int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ +static SQLITE_NOINLINE int btreeNext(BtCursor *pCur, int *pRes){ int rc; int idx; MemPage *pPage; assert( cursorHoldsMutex(pCur) ); - assert( pRes!=0 ); - assert( *pRes==0 || *pRes==1 ); assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); + assert( *pRes==0 ); if( pCur->eState!=CURSOR_VALID ){ - invalidateOverflowCache(pCur); + assert( (pCur->curFlags & BTCF_ValidOvfl)==0 ); rc = restoreCursorPosition(pCur); if( rc!=SQLITE_OK ){ - *pRes = 0; return rc; } if( CURSOR_INVALID==pCur->eState ){ @@ -4826,7 +5044,6 @@ int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ pCur->eState = CURSOR_VALID; if( pCur->skipNext>0 ){ pCur->skipNext = 0; - *pRes = 0; return SQLITE_OK; } pCur->skipNext = 0; @@ -4844,18 +5061,11 @@ int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ ** page into more than one b-tree structure. */ testcase( idx>pPage->nCell ); - pCur->info.nSize = 0; - pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); if( idx>=pPage->nCell ){ if( !pPage->leaf ){ rc = moveToChild(pCur, get4byte(&pPage->aData[pPage->hdrOffset+8])); - if( rc ){ - *pRes = 0; - return rc; - } - rc = moveToLeftmost(pCur); - *pRes = 0; - return rc; + if( rc ) return rc; + return moveToLeftmost(pCur); } do{ if( pCur->iPage==0 ){ @@ -4866,22 +5076,39 @@ int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ moveToParent(pCur); pPage = pCur->apPage[pCur->iPage]; }while( pCur->aiIdx[pCur->iPage]>=pPage->nCell ); - *pRes = 0; if( pPage->intKey ){ - rc = sqlite3BtreeNext(pCur, pRes); + return sqlite3BtreeNext(pCur, pRes); }else{ - rc = SQLITE_OK; + return SQLITE_OK; } - return rc; } - *pRes = 0; if( pPage->leaf ){ return SQLITE_OK; + }else{ + return moveToLeftmost(pCur); + } +} +int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ + MemPage *pPage; + assert( cursorHoldsMutex(pCur) ); + assert( pRes!=0 ); + assert( *pRes==0 || *pRes==1 ); + assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); + pCur->info.nSize = 0; + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); + *pRes = 0; + if( pCur->eState!=CURSOR_VALID ) return btreeNext(pCur, pRes); + pPage = pCur->apPage[pCur->iPage]; + if( (++pCur->aiIdx[pCur->iPage])>=pPage->nCell ){ + pCur->aiIdx[pCur->iPage]--; + return btreeNext(pCur, pRes); + } + if( pPage->leaf ){ + return SQLITE_OK; + }else{ + return moveToLeftmost(pCur); } - rc = moveToLeftmost(pCur); - return rc; } - /* ** Step the cursor to the back to the previous entry in the database. If @@ -4889,6 +5116,12 @@ int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ ** was already pointing to the first entry in the database before ** this routine was called, then set *pRes=1. ** +** The main entry point is sqlite3BtreePrevious(). That routine is optimized +** for the common case of merely decrementing the cell counter BtCursor.aiIdx +** to the previous cell on the current page. The (slower) btreePrevious() +** helper routine is called when it is necessary to move to a different page +** or to restore the cursor. +** ** The calling function will set *pRes to 0 or 1. The initial *pRes value ** will be 1 if the cursor being stepped corresponds to an SQL index and ** if this routine could have been skipped if that SQL index had been @@ -4898,22 +5131,20 @@ int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ ** SQLite btree implementation does not. (Note that the comdb2 btree ** implementation does use this hint, however.) */ -int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ +static SQLITE_NOINLINE int btreePrevious(BtCursor *pCur, int *pRes){ int rc; MemPage *pPage; assert( cursorHoldsMutex(pCur) ); assert( pRes!=0 ); - assert( *pRes==0 || *pRes==1 ); + assert( *pRes==0 ); assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); - pCur->curFlags &= ~(BTCF_AtLast|BTCF_ValidOvfl); + assert( (pCur->curFlags & (BTCF_AtLast|BTCF_ValidOvfl|BTCF_ValidNKey))==0 ); + assert( pCur->info.nSize==0 ); if( pCur->eState!=CURSOR_VALID ){ - if( ALWAYS(pCur->eState>=CURSOR_REQUIRESEEK) ){ - rc = btreeRestoreCursorPosition(pCur); - if( rc!=SQLITE_OK ){ - *pRes = 0; - return rc; - } + rc = restoreCursorPosition(pCur); + if( rc!=SQLITE_OK ){ + return rc; } if( CURSOR_INVALID==pCur->eState ){ *pRes = 1; @@ -4924,7 +5155,6 @@ int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ pCur->eState = CURSOR_VALID; if( pCur->skipNext<0 ){ pCur->skipNext = 0; - *pRes = 0; return SQLITE_OK; } pCur->skipNext = 0; @@ -4936,10 +5166,7 @@ int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ if( !pPage->leaf ){ int idx = pCur->aiIdx[pCur->iPage]; rc = moveToChild(pCur, get4byte(findCell(pPage, idx))); - if( rc ){ - *pRes = 0; - return rc; - } + if( rc ) return rc; rc = moveToRightmost(pCur); }else{ while( pCur->aiIdx[pCur->iPage]==0 ){ @@ -4950,8 +5177,8 @@ int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ } moveToParent(pCur); } - pCur->info.nSize = 0; - pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); + assert( pCur->info.nSize==0 ); + assert( (pCur->curFlags & (BTCF_ValidNKey|BTCF_ValidOvfl))==0 ); pCur->aiIdx[pCur->iPage]--; pPage = pCur->apPage[pCur->iPage]; @@ -4961,9 +5188,25 @@ int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ rc = SQLITE_OK; } } - *pRes = 0; return rc; } +int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ + assert( cursorHoldsMutex(pCur) ); + assert( pRes!=0 ); + assert( *pRes==0 || *pRes==1 ); + assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); + *pRes = 0; + pCur->curFlags &= ~(BTCF_AtLast|BTCF_ValidOvfl|BTCF_ValidNKey); + pCur->info.nSize = 0; + if( pCur->eState!=CURSOR_VALID + || pCur->aiIdx[pCur->iPage]==0 + || pCur->apPage[pCur->iPage]->leaf==0 + ){ + return btreePrevious(pCur, pRes); + } + pCur->aiIdx[pCur->iPage]--; + return SQLITE_OK; +} /* ** Allocate a new page from the database file. @@ -5007,6 +5250,8 @@ static int allocateBtreePage( assert( eMode==BTALLOC_ANY || (nearby>0 && IfNotOmitAV(pBt->autoVacuum)) ); pPage1 = pBt->pPage1; mxPage = btreePagecount(pBt); + /* EVIDENCE-OF: R-05119-02637 The 4-byte big-endian integer at offset 36 + ** stores stores the total number of pages on the freelist. */ n = get4byte(&pPage1->aData[36]); testcase( n==mxPage-1 ); if( n>=mxPage ){ @@ -5053,8 +5298,14 @@ static int allocateBtreePage( do { pPrevTrunk = pTrunk; if( pPrevTrunk ){ + /* EVIDENCE-OF: R-01506-11053 The first integer on a freelist trunk page + ** is the page number of the next freelist trunk page in the list or + ** zero if this is the last freelist trunk page. */ iTrunk = get4byte(&pPrevTrunk->aData[0]); }else{ + /* EVIDENCE-OF: R-59841-13798 The 4-byte big-endian integer at offset 32 + ** stores the page number of the first page of the freelist, or zero if + ** the freelist is empty. */ iTrunk = get4byte(&pPage1->aData[32]); } testcase( iTrunk==mxPage ); @@ -5069,8 +5320,9 @@ static int allocateBtreePage( } assert( pTrunk!=0 ); assert( pTrunk->aData!=0 ); - - k = get4byte(&pTrunk->aData[4]); /* # of leaves on this trunk page */ + /* EVIDENCE-OF: R-13523-04394 The second integer on a freelist trunk page + ** is the number of leaf page pointers to follow. */ + k = get4byte(&pTrunk->aData[4]); if( k==0 && !searchList ){ /* The trunk has no leaves and the list is not being searched. ** So extract the trunk page itself and use it as the newly @@ -5204,7 +5456,7 @@ static int allocateBtreePage( memcpy(&aData[8+closest*4], &aData[4+k*4], 4); } put4byte(&aData[4], k-1); - noContent = !btreeGetHasContent(pBt, *pPgno) ? PAGER_GET_NOCONTENT : 0; + noContent = !btreeGetHasContent(pBt, *pPgno)? PAGER_GET_NOCONTENT : 0; rc = btreeGetPage(pBt, *pPgno, ppPage, noContent); if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite((*ppPage)->pDbPage); @@ -5237,7 +5489,7 @@ static int allocateBtreePage( ** here are confined to those pages that lie between the end of the ** database image and the end of the database file. */ - int bNoContent = (0==IfNotOmitAV(pBt->bDoTruncate)) ? PAGER_GET_NOCONTENT : 0; + int bNoContent = (0==IfNotOmitAV(pBt->bDoTruncate))? PAGER_GET_NOCONTENT:0; rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); if( rc ) return rc; @@ -5388,6 +5640,11 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ ** for now. At some point in the future (once everyone has upgraded ** to 3.6.0 or later) we should consider fixing the conditional above ** to read "usableSize/4-2" instead of "usableSize/4-8". + ** + ** EVIDENCE-OF: R-19920-11576 However, newer versions of SQLite still + ** avoid using the last six entries in the freelist trunk page array in + ** order that database files created by newer versions of SQLite can be + ** read by older versions of SQLite. */ rc = sqlite3PagerWrite(pTrunk->pDbPage); if( rc==SQLITE_OK ){ @@ -5436,9 +5693,15 @@ static void freePage(MemPage *pPage, int *pRC){ } /* -** Free any overflow pages associated with the given Cell. +** Free any overflow pages associated with the given Cell. Write the +** local Cell size (the number of bytes on the original page, omitting +** overflow) into *pnSize. */ -static int clearCell(MemPage *pPage, unsigned char *pCell){ +static int clearCell( + MemPage *pPage, /* The page that contains the Cell */ + unsigned char *pCell, /* First byte of the Cell */ + u16 *pnSize /* Write the size of the Cell here */ +){ BtShared *pBt = pPage->pBt; CellInfo info; Pgno ovflPgno; @@ -5448,6 +5711,7 @@ static int clearCell(MemPage *pPage, unsigned char *pCell){ assert( sqlite3_mutex_held(pPage->pBt->mutex) ); btreeParseCellPtr(pPage, pCell, &info); + *pnSize = info.nSize; if( info.iOverflow==0 ){ return SQLITE_OK; /* No overflow pages. Return without doing anything */ } @@ -5531,7 +5795,6 @@ static int fillInCell( BtShared *pBt = pPage->pBt; Pgno pgnoOvfl = 0; int nHeader; - CellInfo info; assert( sqlite3_mutex_held(pPage->pBt->mutex) ); @@ -5541,23 +5804,17 @@ static int fillInCell( || sqlite3PagerIswriteable(pPage->pDbPage) ); /* Fill in the header. */ - nHeader = 0; - if( !pPage->leaf ){ - nHeader += 4; - } - if( pPage->hasData ){ - nHeader += putVarint32(&pCell[nHeader], nData+nZero); + nHeader = pPage->childPtrSize; + nPayload = nData + nZero; + if( pPage->intKeyLeaf ){ + nHeader += putVarint32(&pCell[nHeader], nPayload); }else{ - nData = nZero = 0; + assert( nData==0 ); + assert( nZero==0 ); } nHeader += putVarint(&pCell[nHeader], *(u64*)&nKey); - btreeParseCellPtr(pPage, pCell, &info); - assert( info.nHeader==nHeader ); - assert( info.nKey==nKey ); - assert( info.nData==(u32)(nData+nZero) ); - /* Fill in the payload */ - nPayload = nData + nZero; + /* Fill in the payload size */ if( pPage->intKey ){ pSrc = pData; nSrc = nData; @@ -5566,15 +5823,55 @@ static int fillInCell( if( NEVER(nKey>0x7fffffff || pKey==0) ){ return SQLITE_CORRUPT_BKPT; } - nPayload += (int)nKey; + nPayload = (int)nKey; pSrc = pKey; nSrc = (int)nKey; } - *pnSize = info.nSize; - spaceLeft = info.nLocal; + if( nPayload<=pPage->maxLocal ){ + n = nHeader + nPayload; + testcase( n==3 ); + testcase( n==4 ); + if( n<4 ) n = 4; + *pnSize = n; + spaceLeft = nPayload; + pPrior = pCell; + }else{ + int mn = pPage->minLocal; + n = mn + (nPayload - mn) % (pPage->pBt->usableSize - 4); + testcase( n==pPage->maxLocal ); + testcase( n==pPage->maxLocal+1 ); + if( n > pPage->maxLocal ) n = mn; + spaceLeft = n; + *pnSize = n + nHeader + 4; + pPrior = &pCell[nHeader+n]; + } pPayload = &pCell[nHeader]; - pPrior = &pCell[info.iOverflow]; + /* At this point variables should be set as follows: + ** + ** nPayload Total payload size in bytes + ** pPayload Begin writing payload here + ** spaceLeft Space available at pPayload. If nPayload>spaceLeft, + ** that means content must spill into overflow pages. + ** *pnSize Size of the local cell (not counting overflow pages) + ** pPrior Where to write the pgno of the first overflow page + ** + ** Use a call to btreeParseCellPtr() to verify that the values above + ** were computed correctly. + */ +#if SQLITE_DEBUG + { + CellInfo info; + btreeParseCellPtr(pPage, pCell, &info); + assert( nHeader=(int)(info.pPayload - pCell) ); + assert( info.nKey==nKey ); + assert( *pnSize == info.nSize ); + assert( spaceLeft == info.nLocal ); + assert( pPrior == &pCell[info.iOverflow] ); + } +#endif + + /* Write the payload into the local Cell and any extra into overflow pages */ while( nPayload>0 ){ if( spaceLeft==0 ){ #ifndef SQLITE_OMIT_AUTOVACUUM @@ -5699,9 +5996,17 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ return; } pPage->nCell--; - memmove(ptr, ptr+2, 2*(pPage->nCell - idx)); - put2byte(&data[hdr+3], pPage->nCell); - pPage->nFree += 2; + if( pPage->nCell==0 ){ + memset(&data[hdr+1], 0, 4); + data[hdr+7] = 0; + put2byte(&data[hdr+5], pPage->pBt->usableSize); + pPage->nFree = pPage->pBt->usableSize - pPage->hdrOffset + - pPage->childPtrSize - 8; + }else{ + memmove(ptr, ptr+2, 2*(pPage->nCell - idx)); + put2byte(&data[hdr+3], pPage->nCell); + pPage->nFree += 2; + } } /* @@ -5715,11 +6020,6 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ ** in pTemp or the original pCell) and also record its index. ** Allocating a new entry in pPage->aCell[] implies that ** pPage->nOverflow is incremented. -** -** If nSkip is non-zero, then do not copy the first nSkip bytes of the -** cell. The caller will overwrite them after this function returns. If -** nSkip is non-zero, then pCell may not point to an invalid memory location -** (but pCell+nSkip is always valid). */ static void insertCell( MemPage *pPage, /* Page into which we are copying */ @@ -5736,12 +6036,12 @@ static void insertCell( int ins; /* Index in data[] where new cell pointer is inserted */ int cellOffset; /* Address of first cell pointer in data[] */ u8 *data; /* The content of the whole page */ - int nSkip = (iChild ? 4 : 0); if( *pRC ) return; assert( i>=0 && i<=pPage->nCell+pPage->nOverflow ); - assert( pPage->nCell<=MX_CELL(pPage->pBt) && MX_CELL(pPage->pBt)<=10921 ); + assert( MX_CELL(pPage->pBt)<=10921 ); + assert( pPage->nCell<=MX_CELL(pPage->pBt) || CORRUPT_DB ); assert( pPage->nOverflow<=ArraySize(pPage->apOvfl) ); assert( ArraySize(pPage->apOvfl)==ArraySize(pPage->aiOvfl) ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); @@ -5753,7 +6053,7 @@ static void insertCell( assert( sz==cellSizePtr(pPage, pCell) || (sz==8 && iChild>0) ); if( pPage->nOverflow || sz+2>pPage->nFree ){ if( pTemp ){ - memcpy(pTemp+nSkip, pCell+nSkip, sz-nSkip); + memcpy(pTemp, pCell, sz); pCell = pTemp; } if( iChild ){ @@ -5782,7 +6082,7 @@ static void insertCell( assert( idx+sz <= (int)pPage->pBt->usableSize ); pPage->nCell++; pPage->nFree -= (u16)(2 + sz); - memcpy(&data[idx+nSkip], pCell+nSkip, sz-nSkip); + memcpy(&data[idx], pCell, sz); if( iChild ){ put4byte(&data[idx], iChild); } @@ -5801,45 +6101,271 @@ static void insertCell( } /* -** Add a list of cells to a page. The page should be initially empty. -** The cells are guaranteed to fit on the page. +** Array apCell[] contains pointers to nCell b-tree page cells. The +** szCell[] array contains the size in bytes of each cell. This function +** replaces the current contents of page pPg with the contents of the cell +** array. +** +** Some of the cells in apCell[] may currently be stored in pPg. This +** function works around problems caused by this by making a copy of any +** such cells before overwriting the page data. +** +** The MemPage.nFree field is invalidated by this function. It is the +** responsibility of the caller to set it correctly. */ -static void assemblePage( - MemPage *pPage, /* The page to be assemblied */ - int nCell, /* The number of cells to add to this page */ - u8 **apCell, /* Pointers to cell bodies */ - u16 *aSize /* Sizes of the cells */ +static void rebuildPage( + MemPage *pPg, /* Edit this page */ + int nCell, /* Final number of cells on page */ + u8 **apCell, /* Array of cells */ + u16 *szCell /* Array of cell sizes */ ){ - int i; /* Loop counter */ - u8 *pCellptr; /* Address of next cell pointer */ - int cellbody; /* Address of next cell body */ - u8 * const data = pPage->aData; /* Pointer to data for pPage */ - const int hdr = pPage->hdrOffset; /* Offset of header on pPage */ - const int nUsable = pPage->pBt->usableSize; /* Usable size of page */ + const int hdr = pPg->hdrOffset; /* Offset of header on pPg */ + u8 * const aData = pPg->aData; /* Pointer to data for pPg */ + const int usableSize = pPg->pBt->usableSize; + u8 * const pEnd = &aData[usableSize]; + int i; + u8 *pCellptr = pPg->aCellIdx; + u8 *pTmp = sqlite3PagerTempSpace(pPg->pBt->pPager); + u8 *pData; - assert( pPage->nOverflow==0 ); - assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - assert( nCell>=0 && nCell<=(int)MX_CELL(pPage->pBt) - && (int)MX_CELL(pPage->pBt)<=10921); - assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + i = get2byte(&aData[hdr+5]); + memcpy(&pTmp[i], &aData[i], usableSize - i); - /* Check that the page has just been zeroed by zeroPage() */ - assert( pPage->nCell==0 ); - assert( get2byteNotZero(&data[hdr+5])==nUsable ); - - pCellptr = &pPage->aCellIdx[nCell*2]; - cellbody = nUsable; - for(i=nCell-1; i>=0; i--){ - u16 sz = aSize[i]; - pCellptr -= 2; - cellbody -= sz; - put2byte(pCellptr, cellbody); - memcpy(&data[cellbody], apCell[i], sz); + pData = pEnd; + for(i=0; iaData && pCellnFree -= (nCell*2 + nUsable - cellbody); - pPage->nCell = (u16)nCell; + + /* The pPg->nFree field is now set incorrectly. The caller will fix it. */ + pPg->nCell = nCell; + pPg->nOverflow = 0; + + put2byte(&aData[hdr+1], 0); + put2byte(&aData[hdr+3], pPg->nCell); + put2byte(&aData[hdr+5], pData - aData); + aData[hdr+7] = 0x00; +} + +/* +** Array apCell[] contains nCell pointers to b-tree cells. Array szCell +** contains the size in bytes of each such cell. This function attempts to +** add the cells stored in the array to page pPg. If it cannot (because +** the page needs to be defragmented before the cells will fit), non-zero +** is returned. Otherwise, if the cells are added successfully, zero is +** returned. +** +** Argument pCellptr points to the first entry in the cell-pointer array +** (part of page pPg) to populate. After cell apCell[0] is written to the +** page body, a 16-bit offset is written to pCellptr. And so on, for each +** cell in the array. It is the responsibility of the caller to ensure +** that it is safe to overwrite this part of the cell-pointer array. +** +** When this function is called, *ppData points to the start of the +** content area on page pPg. If the size of the content area is extended, +** *ppData is updated to point to the new start of the content area +** before returning. +** +** Finally, argument pBegin points to the byte immediately following the +** end of the space required by this page for the cell-pointer area (for +** all cells - not just those inserted by the current call). If the content +** area must be extended to before this point in order to accomodate all +** cells in apCell[], then the cells do not fit and non-zero is returned. +*/ +static int pageInsertArray( + MemPage *pPg, /* Page to add cells to */ + u8 *pBegin, /* End of cell-pointer array */ + u8 **ppData, /* IN/OUT: Page content -area pointer */ + u8 *pCellptr, /* Pointer to cell-pointer area */ + int nCell, /* Number of cells to add to pPg */ + u8 **apCell, /* Array of cells */ + u16 *szCell /* Array of cell sizes */ +){ + int i; + u8 *aData = pPg->aData; + u8 *pData = *ppData; + const int bFreelist = aData[1] || aData[2]; + assert( CORRUPT_DB || pPg->hdrOffset==0 ); /* Never called on page 1 */ + for(i=0; iaData; + u8 * const pEnd = &aData[pPg->pBt->usableSize]; + u8 * const pStart = &aData[pPg->hdrOffset + 8 + pPg->childPtrSize]; + int nRet = 0; + int i; + u8 *pFree = 0; + int szFree = 0; + + for(i=0; i=pStart && pCellaData && (pFree - aData)<65536 ); + freeSpace(pPg, (u16)(pFree - aData), szFree); + } + pFree = pCell; + szFree = sz; + if( pFree+sz>pEnd ) return 0; + }else{ + pFree = pCell; + szFree += sz; + } + nRet++; + } + } + if( pFree ){ + assert( pFree>aData && (pFree - aData)<65536 ); + freeSpace(pPg, (u16)(pFree - aData), szFree); + } + return nRet; +} + +/* +** apCell[] and szCell[] contains pointers to and sizes of all cells in the +** pages being balanced. The current page, pPg, has pPg->nCell cells starting +** with apCell[iOld]. After balancing, this page should hold nNew cells +** starting at apCell[iNew]. +** +** This routine makes the necessary adjustments to pPg so that it contains +** the correct cells after being balanced. +** +** The pPg->nFree field is invalid when this function returns. It is the +** responsibility of the caller to set it correctly. +*/ +static void editPage( + MemPage *pPg, /* Edit this page */ + int iOld, /* Index of first cell currently on page */ + int iNew, /* Index of new first cell on page */ + int nNew, /* Final number of cells on page */ + u8 **apCell, /* Array of cells */ + u16 *szCell /* Array of cell sizes */ +){ + u8 * const aData = pPg->aData; + const int hdr = pPg->hdrOffset; + u8 *pBegin = &pPg->aCellIdx[nNew * 2]; + int nCell = pPg->nCell; /* Cells stored on pPg */ + u8 *pData; + u8 *pCellptr; + int i; + int iOldEnd = iOld + pPg->nCell + pPg->nOverflow; + int iNewEnd = iNew + nNew; + +#ifdef SQLITE_DEBUG + u8 *pTmp = sqlite3PagerTempSpace(pPg->pBt->pPager); + memcpy(pTmp, aData, pPg->pBt->usableSize); +#endif + + /* Remove cells from the start and end of the page */ + if( iOldaCellIdx, &pPg->aCellIdx[nShift*2], nCell*2); + nCell -= nShift; + } + if( iNewEnd < iOldEnd ){ + nCell -= pageFreeArray( + pPg, iOldEnd-iNewEnd, &apCell[iNewEnd], &szCell[iNewEnd] + ); + } + + pData = &aData[get2byteNotZero(&aData[hdr+5])]; + if( pDataaCellIdx; + memmove(&pCellptr[nAdd*2], pCellptr, nCell*2); + if( pageInsertArray( + pPg, pBegin, &pData, pCellptr, + nAdd, &apCell[iNew], &szCell[iNew] + ) ) goto editpage_fail; + nCell += nAdd; + } + + /* Add any overflow cells */ + for(i=0; inOverflow; i++){ + int iCell = (iOld + pPg->aiOvfl[i]) - iNew; + if( iCell>=0 && iCellaCellIdx[iCell * 2]; + memmove(&pCellptr[2], pCellptr, (nCell - iCell) * 2); + nCell++; + if( pageInsertArray( + pPg, pBegin, &pData, pCellptr, + 1, &apCell[iCell + iNew], &szCell[iCell + iNew] + ) ) goto editpage_fail; + } + } + + /* Append cells to the end of the page */ + pCellptr = &pPg->aCellIdx[nCell*2]; + if( pageInsertArray( + pPg, pBegin, &pData, pCellptr, + nNew-nCell, &apCell[iNew+nCell], &szCell[iNew+nCell] + ) ) goto editpage_fail; + + pPg->nCell = nNew; + pPg->nOverflow = 0; + + put2byte(&aData[hdr+3], pPg->nCell); + put2byte(&aData[hdr+5], pData - aData); + +#ifdef SQLITE_DEBUG + for(i=0; iaCellIdx[i*2]); + if( pCell>=aData && pCell<&aData[pPg->pBt->usableSize] ){ + pCell = &pTmp[pCell - aData]; + } + assert( 0==memcmp(pCell, &aData[iOff], szCell[i+iNew]) ); + } +#endif + + return; + editpage_fail: + /* Unable to edit this page. Rebuild it from scratch instead. */ + rebuildPage(pPg, nNew, &apCell[iNew], &szCell[iNew]); } /* @@ -5893,7 +6419,7 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ assert( pPage->nOverflow==1 ); /* This error condition is now caught prior to reaching this function */ - if( pPage->nCell==0 ) return SQLITE_CORRUPT_BKPT; + if( NEVER(pPage->nCell==0) ) return SQLITE_CORRUPT_BKPT; /* Allocate a new page. This page will become the right-sibling of ** pPage. Make the parent page writable, so that the new divider cell @@ -5911,7 +6437,8 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ assert( sqlite3PagerIswriteable(pNew->pDbPage) ); assert( pPage->aData[0]==(PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF) ); zeroPage(pNew, PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF); - assemblePage(pNew, 1, &pCell, &szCell); + rebuildPage(pNew, 1, &pCell, &szCell); + pNew->nFree = pBt->usableSize - pNew->cellOffset - 2 - szCell; /* If this is an auto-vacuum database, update the pointer map ** with entries for the new page, and any pointer from the @@ -6130,17 +6657,22 @@ static int balance_nonroot( int iOvflSpace = 0; /* First unused byte of aOvflSpace[] */ int szScratch; /* Size of scratch memory requested */ MemPage *apOld[NB]; /* pPage and up to two siblings */ - MemPage *apCopy[NB]; /* Private copies of apOld[] pages */ MemPage *apNew[NB+2]; /* pPage and up to NB siblings after balancing */ u8 *pRight; /* Location in parent of right-sibling pointer */ u8 *apDiv[NB-1]; /* Divider cells in pParent */ int cntNew[NB+2]; /* Index in aCell[] of cell after i-th page */ - int szNew[NB+2]; /* Combined size of cells place on i-th page */ + int cntOld[NB+2]; /* Old index in aCell[] after i-th page */ + int szNew[NB+2]; /* Combined size of cells placed on i-th page */ u8 **apCell = 0; /* All cells begin balanced */ u16 *szCell; /* Local size of all cells in apCell[] */ u8 *aSpace1; /* Space for copies of dividers cells */ Pgno pgno; /* Temp var to store a page number in */ + u8 abDone[NB+2]; /* True after i'th new page is populated */ + Pgno aPgno[NB+2]; /* Page numbers of new pages before shuffling */ + Pgno aPgOrder[NB+2]; /* Copy of aPgno[] used for sorting pages */ + u16 aPgFlags[NB+2]; /* flags field of new pages before shuffling */ + memset(abDone, 0, sizeof(abDone)); pBt = pParent->pBt; assert( sqlite3_mutex_held(pBt->mutex) ); assert( sqlite3PagerIswriteable(pParent->pDbPage) ); @@ -6249,12 +6781,14 @@ static int balance_nonroot( /* ** Allocate space for memory structures */ - k = pBt->pageSize + ROUND8(sizeof(MemPage)); szScratch = nMaxCells*sizeof(u8*) /* apCell */ + nMaxCells*sizeof(u16) /* szCell */ - + pBt->pageSize /* aSpace1 */ - + k*nOld; /* Page copies (apCopy) */ + + pBt->pageSize; /* aSpace1 */ + + /* EVIDENCE-OF: R-28375-38319 SQLite will never request a scratch buffer + ** that is more than 6 times the database page size. */ + assert( szScratch<=6*(int)pBt->pageSize ); apCell = sqlite3ScratchMalloc( szScratch ); if( apCell==0 ){ rc = SQLITE_NOMEM; @@ -6267,8 +6801,8 @@ static int balance_nonroot( /* ** Load pointers to all cells on sibling pages and the divider cells ** into the local apCell[] array. Make copies of the divider cells - ** into space obtained from aSpace1[] and remove the divider cells - ** from pParent. + ** into space obtained from aSpace1[]. The divider cells have already + ** been removed from pParent. ** ** If the siblings are on leaf pages, then the child pointers of the ** divider cells are stripped from the cells before they are copied @@ -6281,18 +6815,10 @@ static int balance_nonroot( ** leafData: 1 if pPage holds key+data and pParent holds only keys. */ leafCorrection = apOld[0]->leaf*4; - leafData = apOld[0]->hasData; + leafData = apOld[0]->intKeyLeaf; for(i=0; ipageSize + k*i]; - memcpy(pOld, apOld[i], sizeof(MemPage)); - pOld->aData = (void*)&pOld[1]; - memcpy(pOld->aData, apOld[i]->aData, pBt->pageSize); + MemPage *pOld = apOld[i]; limit = pOld->nCell+pOld->nOverflow; if( pOld->nOverflow>0 ){ @@ -6313,6 +6839,7 @@ static int balance_nonroot( nCell++; } } + cntOld[i] = nCell; if( i usableSpace ){ - szNew[k] = subtotal - szCell[i]; + szNew[k] = subtotal - szCell[i] - 2; cntNew[k] = i; if( leafData ){ i--; } subtotal = 0; @@ -6378,9 +6909,10 @@ static int balance_nonroot( /* ** The packing computed by the previous block is biased toward the siblings - ** on the left side. The left siblings are always nearly full, while the - ** right-most sibling might be nearly empty. This block of code attempts - ** to adjust the packing of siblings to get a better balance. + ** on the left side (siblings with smaller keys). The left siblings are + ** always nearly full, while the right-most sibling might be nearly empty. + ** The next block of code attempts to adjust the packing of siblings to + ** get a better balance. ** ** This adjustment is more than an optimization. The packing above might ** be so out of balance as to be illegal. For example, the right-most @@ -6409,22 +6941,18 @@ static int balance_nonroot( szNew[i-1] = szLeft; } - /* Either we found one or more cells (cntnew[0])>0) or pPage is - ** a virtual root page. A virtual root page is when the real root - ** page is page 1 and we are the only child of that page. - ** - ** UPDATE: The assert() below is not necessarily true if the database - ** file is corrupt. The corruption will be detected and reported later - ** in this procedure so there is no need to act upon it now. + /* Sanity check: For a non-corrupt database file one of the follwing + ** must be true: + ** (1) We found one or more cells (cntNew[0])>0), or + ** (2) pPage is a virtual root page. A virtual root page is when + ** the real root page is page 1 and we are the only child of + ** that page. */ -#if 0 - assert( cntNew[0]>0 || (pParent->pgno==1 && pParent->nCell==0) ); -#endif - - TRACE(("BALANCE: old: %d %d %d ", - apOld[0]->pgno, - nOld>=2 ? apOld[1]->pgno : 0, - nOld>=3 ? apOld[2]->pgno : 0 + assert( cntNew[0]>0 || (pParent->pgno==1 && pParent->nCell==0) || CORRUPT_DB); + TRACE(("BALANCE: old: %d(nc=%d) %d(nc=%d) %d(nc=%d)\n", + apOld[0]->pgno, apOld[0]->nCell, + nOld>=2 ? apOld[1]->pgno : 0, nOld>=2 ? apOld[1]->nCell : 0, + nOld>=3 ? apOld[2]->pgno : 0, nOld>=3 ? apOld[2]->nCell : 0 )); /* @@ -6447,8 +6975,10 @@ static int balance_nonroot( assert( i>0 ); rc = allocateBtreePage(pBt, &pNew, &pgno, (bBulk ? 1 : pgno), 0); if( rc ) goto balance_cleanup; + zeroPage(pNew, pageFlags); apNew[i] = pNew; nNew++; + cntOld[i] = nCell; /* Set the pointer-map entry for the new sibling page. */ if( ISAUTOVACUUM ){ @@ -6460,135 +6990,247 @@ static int balance_nonroot( } } - /* Free any old pages that were not reused as new pages. - */ - while( ipgno; - int minI = i; - for(j=i+1; jpgno<(unsigned)minV ){ - minI = j; - minV = apNew[j]->pgno; + for(i=0; ipgno; + aPgFlags[i] = apNew[i]->pDbPage->flags; + for(j=0; ji ){ - MemPage *pT; - pT = apNew[i]; - apNew[i] = apNew[minI]; - apNew[minI] = pT; + } + for(i=0; ii ){ + sqlite3PagerRekey(apNew[iBest]->pDbPage, pBt->nPage+iBest+1, 0); + } + sqlite3PagerRekey(apNew[i]->pDbPage, pgno, aPgFlags[iBest]); + apNew[i]->pgno = pgno; } } - TRACE(("new: %d(%d) %d(%d) %d(%d) %d(%d) %d(%d)\n", - apNew[0]->pgno, szNew[0], + + TRACE(("BALANCE: new: %d(%d nc=%d) %d(%d nc=%d) %d(%d nc=%d) " + "%d(%d nc=%d) %d(%d nc=%d)\n", + apNew[0]->pgno, szNew[0], cntNew[0], nNew>=2 ? apNew[1]->pgno : 0, nNew>=2 ? szNew[1] : 0, + nNew>=2 ? cntNew[1] - cntNew[0] - !leafData : 0, nNew>=3 ? apNew[2]->pgno : 0, nNew>=3 ? szNew[2] : 0, + nNew>=3 ? cntNew[2] - cntNew[1] - !leafData : 0, nNew>=4 ? apNew[3]->pgno : 0, nNew>=4 ? szNew[3] : 0, - nNew>=5 ? apNew[4]->pgno : 0, nNew>=5 ? szNew[4] : 0)); + nNew>=4 ? cntNew[3] - cntNew[2] - !leafData : 0, + nNew>=5 ? apNew[4]->pgno : 0, nNew>=5 ? szNew[4] : 0, + nNew>=5 ? cntNew[4] - cntNew[3] - !leafData : 0 + )); assert( sqlite3PagerIswriteable(pParent->pDbPage) ); put4byte(pRight, apNew[nNew-1]->pgno); - /* - ** Evenly distribute the data in apCell[] across the new pages. - ** Insert divider cells into pParent as necessary. + /* If the sibling pages are not leaves, ensure that the right-child pointer + ** of the right-most new sibling page is set to the value that was + ** originally in the same field of the right-most old sibling page. */ + if( (pageFlags & PTF_LEAF)==0 && nOld!=nNew ){ + MemPage *pOld = (nNew>nOld ? apNew : apOld)[nOld-1]; + memcpy(&apNew[nNew-1]->aData[8], &pOld->aData[8], 4); + } + + /* Make any required updates to pointer map entries associated with + ** cells stored on sibling pages following the balance operation. Pointer + ** map entries associated with divider cells are set by the insertCell() + ** routine. The associated pointer map entries are: + ** + ** a) if the cell contains a reference to an overflow chain, the + ** entry associated with the first page in the overflow chain, and + ** + ** b) if the sibling pages are not leaves, the child page associated + ** with the cell. + ** + ** If the sibling pages are not leaves, then the pointer map entry + ** associated with the right-child of each sibling may also need to be + ** updated. This happens below, after the sibling pages have been + ** populated, not here. */ - j = 0; - for(i=0; inCell>0 || (nNew==1 && cntNew[0]==0) ); - assert( pNew->nOverflow==0 ); + if( ISAUTOVACUUM ){ + MemPage *pNew = apNew[0]; + u8 *aOld = pNew->aData; + int cntOldNext = pNew->nCell + pNew->nOverflow; + int usableSize = pBt->usableSize; + int iNew = 0; + int iOld = 0; - j = cntNew[i]; + for(i=0; inCell + pOld->nOverflow + !leafData; + aOld = pOld->aData; + } + if( i==cntNew[iNew] ){ + pNew = apNew[++iNew]; + if( !leafData ) continue; + } - /* If the sibling page assembled above was not the right-most sibling, - ** insert a divider cell into the parent page. - */ - assert( ileaf ){ - memcpy(&pNew->aData[8], pCell, 4); - }else if( leafData ){ - /* If the tree is a leaf-data tree, and the siblings are leaves, - ** then there is no divider cell in apCell[]. Instead, the divider - ** cell consists of the integer key for the right-most cell of - ** the sibling-page assembled above only. - */ - CellInfo info; - j--; - btreeParseCellPtr(pNew, apCell[j], &info); - pCell = pTemp; - sz = 4 + putVarint(&pCell[4], info.nKey); - pTemp = 0; - }else{ - pCell -= 4; - /* Obscure case for non-leaf-data trees: If the cell at pCell was - ** previously stored on a leaf node, and its reported size was 4 - ** bytes, then it may actually be smaller than this - ** (see btreeParseCellPtr(), 4 bytes is the minimum size of - ** any cell). But it is important to pass the correct size to - ** insertCell(), so reparse the cell now. - ** - ** Note that this can never happen in an SQLite data file, as all - ** cells are at least 4 bytes. It only happens in b-trees used - ** to evaluate "IN (SELECT ...)" and similar clauses. - */ - if( szCell[j]==4 ){ - assert(leafCorrection==4); - sz = cellSizePtr(pParent, pCell); + /* Cell pCell is destined for new sibling page pNew. Originally, it + ** was either part of sibling page iOld (possibly an overflow cell), + ** or else the divider cell to the left of sibling page iOld. So, + ** if sibling page iOld had the same page number as pNew, and if + ** pCell really was a part of sibling page iOld (not a divider or + ** overflow cell), we can skip updating the pointer map entries. */ + if( iOld>=nNew + || pNew->pgno!=aPgno[iOld] + || pCell=&aOld[usableSize] + ){ + if( !leafCorrection ){ + ptrmapPut(pBt, get4byte(pCell), PTRMAP_BTREE, pNew->pgno, &rc); + } + if( szCell[i]>pNew->minLocal ){ + ptrmapPutOvflPtr(pNew, pCell, &rc); } } - iOvflSpace += sz; - assert( sz<=pBt->maxLocal+23 ); - assert( iOvflSpace <= (int)pBt->pageSize ); - insertCell(pParent, nxDiv, pCell, sz, pTemp, pNew->pgno, &rc); - if( rc!=SQLITE_OK ) goto balance_cleanup; - assert( sqlite3PagerIswriteable(pParent->pDbPage) ); - - j++; - nxDiv++; } } - assert( j==nCell ); + + /* Insert new divider cells into pParent. */ + for(i=0; ileaf ){ + memcpy(&pNew->aData[8], pCell, 4); + }else if( leafData ){ + /* If the tree is a leaf-data tree, and the siblings are leaves, + ** then there is no divider cell in apCell[]. Instead, the divider + ** cell consists of the integer key for the right-most cell of + ** the sibling-page assembled above only. + */ + CellInfo info; + j--; + btreeParseCellPtr(pNew, apCell[j], &info); + pCell = pTemp; + sz = 4 + putVarint(&pCell[4], info.nKey); + pTemp = 0; + }else{ + pCell -= 4; + /* Obscure case for non-leaf-data trees: If the cell at pCell was + ** previously stored on a leaf node, and its reported size was 4 + ** bytes, then it may actually be smaller than this + ** (see btreeParseCellPtr(), 4 bytes is the minimum size of + ** any cell). But it is important to pass the correct size to + ** insertCell(), so reparse the cell now. + ** + ** Note that this can never happen in an SQLite data file, as all + ** cells are at least 4 bytes. It only happens in b-trees used + ** to evaluate "IN (SELECT ...)" and similar clauses. + */ + if( szCell[j]==4 ){ + assert(leafCorrection==4); + sz = cellSizePtr(pParent, pCell); + } + } + iOvflSpace += sz; + assert( sz<=pBt->maxLocal+23 ); + assert( iOvflSpace <= (int)pBt->pageSize ); + insertCell(pParent, nxDiv+i, pCell, sz, pTemp, pNew->pgno, &rc); + if( rc!=SQLITE_OK ) goto balance_cleanup; + assert( sqlite3PagerIswriteable(pParent->pDbPage) ); + } + + /* Now update the actual sibling pages. The order in which they are updated + ** is important, as this code needs to avoid disrupting any page from which + ** cells may still to be read. In practice, this means: + ** + ** (1) If cells are moving left (from apNew[iPg] to apNew[iPg-1]) + ** then it is not safe to update page apNew[iPg] until after + ** the left-hand sibling apNew[iPg-1] has been updated. + ** + ** (2) If cells are moving right (from apNew[iPg] to apNew[iPg+1]) + ** then it is not safe to update page apNew[iPg] until after + ** the right-hand sibling apNew[iPg+1] has been updated. + ** + ** If neither of the above apply, the page is safe to update. + ** + ** The iPg value in the following loop starts at nNew-1 goes down + ** to 0, then back up to nNew-1 again, thus making two passes over + ** the pages. On the initial downward pass, only condition (1) above + ** needs to be tested because (2) will always be true from the previous + ** step. On the upward pass, both conditions are always true, so the + ** upwards pass simply processes pages that were missed on the downward + ** pass. + */ + for(i=1-nNew; i=0 && iPg=0 /* On the upwards pass, or... */ + || cntOld[iPg-1]>=cntNew[iPg-1] /* Condition (1) is true */ + ){ + int iNew; + int iOld; + int nNewCell; + + /* Verify condition (1): If cells are moving left, update iPg + ** only after iPg-1 has already been updated. */ + assert( iPg==0 || cntOld[iPg-1]>=cntNew[iPg-1] || abDone[iPg-1] ); + + /* Verify condition (2): If cells are moving right, update iPg + ** only after iPg+1 has already been updated. */ + assert( cntNew[iPg]>=cntOld[iPg] || abDone[iPg+1] ); + + if( iPg==0 ){ + iNew = iOld = 0; + nNewCell = cntNew[0]; + }else{ + iOld = iPgnFree = usableSpace-szNew[iPg]; + assert( apNew[iPg]->nOverflow==0 ); + assert( apNew[iPg]->nCell==nNewCell ); + } + } + + /* All pages have been processed exactly once */ + assert( memcmp(abDone, "\01\01\01\01\01", nNew)==0 ); + assert( nOld>0 ); assert( nNew>0 ); - if( (pageFlags & PTF_LEAF)==0 ){ - u8 *zChild = &apCopy[nOld-1]->aData[8]; - memcpy(&apNew[nNew-1]->aData[8], zChild, 4); - } if( isRoot && pParent->nCell==0 && pParent->hdrOffset<=apNew[0]->nFree ){ /* The root page of the b-tree now contains no cells. The only sibling @@ -6601,126 +7243,50 @@ static int balance_nonroot( ** sets all pointer-map entries corresponding to database image pages ** for which the pointer is stored within the content being copied. ** - ** The second assert below verifies that the child page is defragmented - ** (it must be, as it was just reconstructed using assemblePage()). This - ** is important if the parent page happens to be page 1 of the database - ** image. */ + ** It is critical that the child page be defragmented before being + ** copied into the parent, because if the parent is page 1 then it will + ** by smaller than the child due to the database header, and so all the + ** free space needs to be up front. + */ assert( nNew==1 ); + rc = defragmentPage(apNew[0]); + testcase( rc!=SQLITE_OK ); assert( apNew[0]->nFree == - (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2) + (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2) + || rc!=SQLITE_OK ); copyNodeContent(apNew[0], pParent, &rc); freePage(apNew[0], &rc); - }else if( ISAUTOVACUUM ){ - /* Fix the pointer-map entries for all the cells that were shifted around. - ** There are several different types of pointer-map entries that need to - ** be dealt with by this routine. Some of these have been set already, but - ** many have not. The following is a summary: - ** - ** 1) The entries associated with new sibling pages that were not - ** siblings when this function was called. These have already - ** been set. We don't need to worry about old siblings that were - ** moved to the free-list - the freePage() code has taken care - ** of those. - ** - ** 2) The pointer-map entries associated with the first overflow - ** page in any overflow chains used by new divider cells. These - ** have also already been taken care of by the insertCell() code. - ** - ** 3) If the sibling pages are not leaves, then the child pages of - ** cells stored on the sibling pages may need to be updated. - ** - ** 4) If the sibling pages are not internal intkey nodes, then any - ** overflow pages used by these cells may need to be updated - ** (internal intkey nodes never contain pointers to overflow pages). - ** - ** 5) If the sibling pages are not leaves, then the pointer-map - ** entries for the right-child pages of each sibling may need - ** to be updated. - ** - ** Cases 1 and 2 are dealt with above by other code. The next - ** block deals with cases 3 and 4 and the one after that, case 5. Since - ** setting a pointer map entry is a relatively expensive operation, this - ** code only sets pointer map entries for child or overflow pages that have - ** actually moved between pages. */ - MemPage *pNew = apNew[0]; - MemPage *pOld = apCopy[0]; - int nOverflow = pOld->nOverflow; - int iNextOld = pOld->nCell + nOverflow; - int iOverflow = (nOverflow ? pOld->aiOvfl[0] : -1); - j = 0; /* Current 'old' sibling page */ - k = 0; /* Current 'new' sibling page */ - for(i=0; inCell + pOld->nOverflow; - if( pOld->nOverflow ){ - nOverflow = pOld->nOverflow; - iOverflow = i + !leafData + pOld->aiOvfl[0]; - } - isDivider = !leafData; - } - - assert(nOverflow>0 || iOverflowaiOvfl[0]==pOld->aiOvfl[1]-1); - assert(nOverflow<3 || pOld->aiOvfl[1]==pOld->aiOvfl[2]-1); - if( i==iOverflow ){ - isDivider = 1; - if( (--nOverflow)>0 ){ - iOverflow++; - } - } - - if( i==cntNew[k] ){ - /* Cell i is the cell immediately following the last cell on new - ** sibling page k. If the siblings are not leaf pages of an - ** intkey b-tree, then cell i is a divider cell. */ - pNew = apNew[++k]; - if( !leafData ) continue; - } - assert( jpgno!=pNew->pgno ){ - if( !leafCorrection ){ - ptrmapPut(pBt, get4byte(apCell[i]), PTRMAP_BTREE, pNew->pgno, &rc); - } - if( szCell[i]>pNew->minLocal ){ - ptrmapPutOvflPtr(pNew, apCell[i], &rc); - } - } + }else if( ISAUTOVACUUM && !leafCorrection ){ + /* Fix the pointer map entries associated with the right-child of each + ** sibling page. All other pointer map entries have already been taken + ** care of. */ + for(i=0; iaData[8]); + ptrmapPut(pBt, key, PTRMAP_BTREE, apNew[i]->pgno, &rc); } + } - if( !leafCorrection ){ - for(i=0; iaData[8]); - ptrmapPut(pBt, key, PTRMAP_BTREE, apNew[i]->pgno, &rc); - } - } + assert( pParent->isInit ); + TRACE(("BALANCE: finished: old=%d new=%d cells=%d\n", + nOld, nNew, nCell)); + + /* Free any old pages that were not reused as new pages. + */ + for(i=nNew; iisInit ){ /* The ptrmapCheckPages() contains assert() statements that verify that ** all pointer map pages are set correctly. This is helpful while ** debugging. This is usually disabled because a corrupt database may ** cause an assert() statement to fail. */ ptrmapCheckPages(apNew, nNew); ptrmapCheckPages(&pParent, 1); -#endif } - - assert( pParent->isInit ); - TRACE(("BALANCE: finished: old=%d new=%d cells=%d\n", - nOld, nNew, nCell)); +#endif /* ** Cleanup before returning. @@ -6857,7 +7423,7 @@ static int balance(BtCursor *pCur){ rc = sqlite3PagerWrite(pParent->pDbPage); if( rc==SQLITE_OK ){ #ifndef SQLITE_OMIT_QUICKBALANCE - if( pPage->hasData + if( pPage->intKeyLeaf && pPage->nOverflow==1 && pPage->aiOvfl[0]==pPage->nCell && pParent->pgno!=1 @@ -6866,7 +7432,7 @@ static int balance(BtCursor *pCur){ /* Call balance_quick() to create a new sibling of pPage on which ** to store the overflow cell. balance_quick() inserts a new cell ** into pParent, which may cause pParent overflow. If this - ** happens, the next interation of the do-loop will balance pParent + ** happens, the next iteration of the do-loop will balance pParent ** use either balance_nonroot() or balance_deeper(). Until this ** happens, the overflow cell is stored in the aBalanceQuickSpace[] ** buffer. @@ -6943,7 +7509,7 @@ static int balance(BtCursor *pCur){ ** MovetoUnpacked() to seek cursor pCur to (pKey, nKey) has already ** been performed. seekResult is the search result returned (a negative ** number if pCur points at an entry that is smaller than (pKey, nKey), or -** a positive value if pCur points at an etry that is larger than +** a positive value if pCur points at an entry that is larger than ** (pKey, nKey)). ** ** If the seekResult parameter is non-zero, then the caller guarantees that @@ -6976,7 +7542,8 @@ int sqlite3BtreeInsert( } assert( cursorHoldsMutex(pCur) ); - assert( (pCur->curFlags & BTCF_WriteFlag)!=0 && pBt->inTransaction==TRANS_WRITE + assert( (pCur->curFlags & BTCF_WriteFlag)!=0 + && pBt->inTransaction==TRANS_WRITE && (pBt->btsFlags & BTS_READ_ONLY)==0 ); assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) ); @@ -7009,7 +7576,8 @@ int sqlite3BtreeInsert( /* If the cursor is currently on the last row and we are appending a ** new row onto the end, set the "loc" to avoid an unnecessary btreeMoveto() ** call */ - if( (pCur->curFlags&BTCF_ValidNKey)!=0 && nKey>0 && pCur->info.nKey==nKey-1 ){ + if( (pCur->curFlags&BTCF_ValidNKey)!=0 && nKey>0 + && pCur->info.nKey==nKey-1 ){ loc = -1; } } @@ -7028,9 +7596,8 @@ int sqlite3BtreeInsert( pCur->pgnoRoot, nKey, nData, pPage->pgno, loc==0 ? "overwrite" : "new entry")); assert( pPage->isInit ); - allocateTempSpace(pBt); newCell = pBt->pTmpSpace; - if( newCell==0 ) return SQLITE_NOMEM; + assert( newCell!=0 ); rc = fillInCell(pPage, newCell, pKey, nKey, pData, nData, nZero, &szNew); if( rc ) goto end_insert; assert( szNew==cellSizePtr(pPage, newCell) ); @@ -7047,8 +7614,7 @@ int sqlite3BtreeInsert( if( !pPage->leaf ){ memcpy(newCell, oldCell, 4); } - szOld = cellSizePtr(pPage, oldCell); - rc = clearCell(pPage, oldCell); + rc = clearCell(pPage, oldCell, &szOld); dropCell(pPage, idx, szOld, &rc); if( rc ) goto end_insert; }else if( loc<0 && pPage->nCell>0 ){ @@ -7100,7 +7666,7 @@ end_insert: /* ** Delete the entry that the cursor is pointing to. The cursor -** is left pointing at a arbitrary location. +** is left pointing at an arbitrary location. */ int sqlite3BtreeDelete(BtCursor *pCur){ Btree *p = pCur->pBtree; @@ -7110,6 +7676,7 @@ int sqlite3BtreeDelete(BtCursor *pCur){ unsigned char *pCell; /* Pointer to cell to delete */ int iCellIdx; /* Index of cell to delete */ int iCellDepth; /* Depth of node containing pCell */ + u16 szCell; /* Size of the cell being deleted */ assert( cursorHoldsMutex(pCur) ); assert( pBt->inTransaction==TRANS_WRITE ); @@ -7158,8 +7725,8 @@ int sqlite3BtreeDelete(BtCursor *pCur){ rc = sqlite3PagerWrite(pPage->pDbPage); if( rc ) return rc; - rc = clearCell(pPage, pCell); - dropCell(pPage, iCellIdx, cellSizePtr(pPage, pCell), &rc); + rc = clearCell(pPage, pCell, &szCell); + dropCell(pPage, iCellIdx, szCell, &rc); if( rc ) return rc; /* If the cell deleted was not located on a leaf page, then the cursor @@ -7176,10 +7743,8 @@ int sqlite3BtreeDelete(BtCursor *pCur){ pCell = findCell(pLeaf, pLeaf->nCell-1); nCell = cellSizePtr(pLeaf, pCell); assert( MX_CELL_SIZE(pBt) >= nCell ); - - allocateTempSpace(pBt); pTmp = pBt->pTmpSpace; - + assert( pTmp!=0 ); rc = sqlite3PagerWrite(pLeaf->pDbPage); insertCell(pPage, iCellIdx, pCell-4, nCell+4, pTmp, n, &rc); dropCell(pLeaf, pLeaf->nCell-1, nCell, &rc); @@ -7391,6 +7956,7 @@ static int clearDatabasePage( unsigned char *pCell; int i; int hdr; + u16 szCell; assert( sqlite3_mutex_held(pBt->mutex) ); if( pgno>btreePagecount(pBt) ){ @@ -7406,7 +7972,7 @@ static int clearDatabasePage( rc = clearDatabasePage(pBt, get4byte(pCell), 1, pnChange); if( rc ) goto cleardatabasepage_out; } - rc = clearCell(pPage, pCell); + rc = clearCell(pPage, pCell, &szCell); if( rc ) goto cleardatabasepage_out; } if( !pPage->leaf ){ @@ -7612,6 +8178,13 @@ int sqlite3BtreeDropTable(Btree *p, int iTable, int *piMoved){ ** The schema layer numbers meta values differently. At the schema ** layer (and the SetCookie and ReadCookie opcodes) the number of ** free pages is not visible. So Cookie[0] is the same as Meta[1]. +** +** This routine treats Meta[BTREE_DATA_VERSION] as a special case. Instead +** of reading the value out of the header, it instead loads the "DataVersion" +** from the pager. The BTREE_DATA_VERSION value is not actually stored in the +** database file. It is a number computed by the pager. But its access +** pattern is the same as header meta values, and so it is convenient to +** read it from this routine. */ void sqlite3BtreeGetMeta(Btree *p, int idx, u32 *pMeta){ BtShared *pBt = p->pBt; @@ -7622,7 +8195,11 @@ void sqlite3BtreeGetMeta(Btree *p, int idx, u32 *pMeta){ assert( pBt->pPage1 ); assert( idx>=0 && idx<=15 ); - *pMeta = get4byte(&pBt->pPage1->aData[36 + idx*4]); + if( idx==BTREE_DATA_VERSION ){ + *pMeta = sqlite3PagerDataVersion(pBt->pPager) + p->iDataVersion; + }else{ + *pMeta = get4byte(&pBt->pPage1->aData[36 + idx*4]); + } /* If auto-vacuum is disabled in this build and this is an auto-vacuum ** database, mark the database as read-only. */ @@ -7713,7 +8290,7 @@ int sqlite3BtreeCount(BtCursor *pCur, i64 *pnEntry){ if( pCur->iPage==0 ){ /* All pages of the b-tree have been visited. Return successfully. */ *pnEntry = nEntry; - return SQLITE_OK; + return moveToRoot(pCur); } moveToParent(pCur); }while ( pCur->aiIdx[pCur->iPage]>=pCur->apPage[pCur->iPage]->nCell ); @@ -7752,11 +8329,11 @@ Pager *sqlite3BtreePager(Btree *p){ */ static void checkAppendMsg( IntegrityCk *pCheck, - char *zMsg1, const char *zFormat, ... ){ va_list ap; + char zBuf[200]; if( !pCheck->mxErr ) return; pCheck->mxErr--; pCheck->nErr++; @@ -7764,8 +8341,9 @@ static void checkAppendMsg( if( pCheck->errMsg.nChar ){ sqlite3StrAccumAppend(&pCheck->errMsg, "\n", 1); } - if( zMsg1 ){ - sqlite3StrAccumAppendAll(&pCheck->errMsg, zMsg1); + if( pCheck->zPfx ){ + sqlite3_snprintf(sizeof(zBuf), zBuf, pCheck->zPfx, pCheck->v1, pCheck->v2); + sqlite3StrAccumAppendAll(&pCheck->errMsg, zBuf); } sqlite3VXPrintf(&pCheck->errMsg, 1, zFormat, ap); va_end(ap); @@ -7798,19 +8376,19 @@ static void setPageReferenced(IntegrityCk *pCheck, Pgno iPg){ /* ** Add 1 to the reference count for page iPage. If this is the second ** reference to the page, add an error message to pCheck->zErrMsg. -** Return 1 if there are 2 ore more references to the page and 0 if +** Return 1 if there are 2 or more references to the page and 0 if ** if this is the first reference to the page. ** ** Also check that the page number is in bounds. */ -static int checkRef(IntegrityCk *pCheck, Pgno iPage, char *zContext){ +static int checkRef(IntegrityCk *pCheck, Pgno iPage){ if( iPage==0 ) return 1; if( iPage>pCheck->nPage ){ - checkAppendMsg(pCheck, zContext, "invalid page number %d", iPage); + checkAppendMsg(pCheck, "invalid page number %d", iPage); return 1; } if( getPageReferenced(pCheck, iPage) ){ - checkAppendMsg(pCheck, zContext, "2nd reference to page %d", iPage); + checkAppendMsg(pCheck, "2nd reference to page %d", iPage); return 1; } setPageReferenced(pCheck, iPage); @@ -7827,8 +8405,7 @@ static void checkPtrmap( IntegrityCk *pCheck, /* Integrity check context */ Pgno iChild, /* Child page number */ u8 eType, /* Expected pointer map type */ - Pgno iParent, /* Expected pointer map parent page number */ - char *zContext /* Context description (used for error msg) */ + Pgno iParent /* Expected pointer map parent page number */ ){ int rc; u8 ePtrmapType; @@ -7837,12 +8414,12 @@ static void checkPtrmap( rc = ptrmapGet(pCheck->pBt, iChild, &ePtrmapType, &iPtrmapParent); if( rc!=SQLITE_OK ){ if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ) pCheck->mallocFailed = 1; - checkAppendMsg(pCheck, zContext, "Failed to read ptrmap key=%d", iChild); + checkAppendMsg(pCheck, "Failed to read ptrmap key=%d", iChild); return; } if( ePtrmapType!=eType || iPtrmapParent!=iParent ){ - checkAppendMsg(pCheck, zContext, + checkAppendMsg(pCheck, "Bad ptr map entry key=%d expected=(%d,%d) got=(%d,%d)", iChild, eType, iParent, ePtrmapType, iPtrmapParent); } @@ -7857,8 +8434,7 @@ static void checkList( IntegrityCk *pCheck, /* Integrity checking context */ int isFreeList, /* True for a freelist. False for overflow page list */ int iPage, /* Page number for first page in the list */ - int N, /* Expected number of pages in the list */ - char *zContext /* Context for error messages */ + int N /* Expected number of pages in the list */ ){ int i; int expected = N; @@ -7867,14 +8443,14 @@ static void checkList( DbPage *pOvflPage; unsigned char *pOvflData; if( iPage<1 ){ - checkAppendMsg(pCheck, zContext, + checkAppendMsg(pCheck, "%d of %d pages missing from overflow list starting at %d", N+1, expected, iFirst); break; } - if( checkRef(pCheck, iPage, zContext) ) break; + if( checkRef(pCheck, iPage) ) break; if( sqlite3PagerGet(pCheck->pPager, (Pgno)iPage, &pOvflPage) ){ - checkAppendMsg(pCheck, zContext, "failed to get page %d", iPage); + checkAppendMsg(pCheck, "failed to get page %d", iPage); break; } pOvflData = (unsigned char *)sqlite3PagerGetData(pOvflPage); @@ -7882,11 +8458,11 @@ static void checkList( int n = get4byte(&pOvflData[4]); #ifndef SQLITE_OMIT_AUTOVACUUM if( pCheck->pBt->autoVacuum ){ - checkPtrmap(pCheck, iPage, PTRMAP_FREEPAGE, 0, zContext); + checkPtrmap(pCheck, iPage, PTRMAP_FREEPAGE, 0); } #endif if( n>(int)pCheck->pBt->usableSize/4-2 ){ - checkAppendMsg(pCheck, zContext, + checkAppendMsg(pCheck, "freelist leaf count too big on page %d", iPage); N--; }else{ @@ -7894,10 +8470,10 @@ static void checkList( Pgno iFreePage = get4byte(&pOvflData[8+i*4]); #ifndef SQLITE_OMIT_AUTOVACUUM if( pCheck->pBt->autoVacuum ){ - checkPtrmap(pCheck, iFreePage, PTRMAP_FREEPAGE, 0, zContext); + checkPtrmap(pCheck, iFreePage, PTRMAP_FREEPAGE, 0); } #endif - checkRef(pCheck, iFreePage, zContext); + checkRef(pCheck, iFreePage); } N -= n; } @@ -7910,7 +8486,7 @@ static void checkList( */ if( pCheck->pBt->autoVacuum && N>0 ){ i = get4byte(pOvflData); - checkPtrmap(pCheck, i, PTRMAP_OVERFLOW2, iPage, zContext); + checkPtrmap(pCheck, i, PTRMAP_OVERFLOW2, iPage); } } #endif @@ -7942,7 +8518,6 @@ static void checkList( static int checkTreePage( IntegrityCk *pCheck, /* Context for the sanity check */ int iPage, /* Page number of the page to check */ - char *zParentContext, /* Parent context */ i64 *pnParentMinKey, i64 *pnParentMaxKey ){ @@ -7953,23 +8528,26 @@ static int checkTreePage( u8 *data; BtShared *pBt; int usableSize; - char zContext[100]; char *hit = 0; i64 nMinKey = 0; i64 nMaxKey = 0; - - sqlite3_snprintf(sizeof(zContext), zContext, "Page %d: ", iPage); + const char *saved_zPfx = pCheck->zPfx; + int saved_v1 = pCheck->v1; + int saved_v2 = pCheck->v2; /* Check that the page exists */ pBt = pCheck->pBt; usableSize = pBt->usableSize; if( iPage==0 ) return 0; - if( checkRef(pCheck, iPage, zParentContext) ) return 0; + if( checkRef(pCheck, iPage) ) return 0; + pCheck->zPfx = "Page %d: "; + pCheck->v1 = iPage; if( (rc = btreeGetPage(pBt, (Pgno)iPage, &pPage, 0))!=0 ){ - checkAppendMsg(pCheck, zContext, + checkAppendMsg(pCheck, "unable to get the page. error code=%d", rc); - return 0; + depth = -1; + goto end_of_check; } /* Clear MemPage.isInit to make sure the corruption detection code in @@ -7977,10 +8555,11 @@ static int checkTreePage( pPage->isInit = 0; if( (rc = btreeInitPage(pPage))!=0 ){ assert( rc==SQLITE_CORRUPT ); /* The only possible error from InitPage */ - checkAppendMsg(pCheck, zContext, + checkAppendMsg(pCheck, "btreeInitPage() returns error code %d", rc); releasePage(pPage); - return 0; + depth = -1; + goto end_of_check; } /* Check out all the cells. @@ -7993,23 +8572,23 @@ static int checkTreePage( /* Check payload overflow pages */ - sqlite3_snprintf(sizeof(zContext), zContext, - "On tree page %d cell %d: ", iPage, i); + pCheck->zPfx = "On tree page %d cell %d: "; + pCheck->v1 = iPage; + pCheck->v2 = i; pCell = findCell(pPage,i); btreeParseCellPtr(pPage, pCell, &info); - sz = info.nData; - if( !pPage->intKey ) sz += (int)info.nKey; + sz = info.nPayload; /* For intKey pages, check that the keys are in order. */ - else if( i==0 ) nMinKey = nMaxKey = info.nKey; - else{ - if( info.nKey <= nMaxKey ){ - checkAppendMsg(pCheck, zContext, - "Rowid %lld out of order (previous was %lld)", info.nKey, nMaxKey); + if( pPage->intKey ){ + if( i==0 ){ + nMinKey = nMaxKey = info.nKey; + }else if( info.nKey <= nMaxKey ){ + checkAppendMsg(pCheck, + "Rowid %lld out of order (previous was %lld)", info.nKey, nMaxKey); } nMaxKey = info.nKey; } - assert( sz==info.nPayload ); if( (sz>info.nLocal) && (&pCell[info.iOverflow]<=&pPage->aData[pBt->usableSize]) ){ @@ -8017,10 +8596,10 @@ static int checkTreePage( Pgno pgnoOvfl = get4byte(&pCell[info.iOverflow]); #ifndef SQLITE_OMIT_AUTOVACUUM if( pBt->autoVacuum ){ - checkPtrmap(pCheck, pgnoOvfl, PTRMAP_OVERFLOW1, iPage, zContext); + checkPtrmap(pCheck, pgnoOvfl, PTRMAP_OVERFLOW1, iPage); } #endif - checkList(pCheck, 0, pgnoOvfl, nPage, zContext); + checkList(pCheck, 0, pgnoOvfl, nPage); } /* Check sanity of left child page. @@ -8029,12 +8608,12 @@ static int checkTreePage( pgno = get4byte(pCell); #ifndef SQLITE_OMIT_AUTOVACUUM if( pBt->autoVacuum ){ - checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage, zContext); + checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage); } #endif - d2 = checkTreePage(pCheck, pgno, zContext, &nMinKey, i==0 ? NULL : &nMaxKey); + d2 = checkTreePage(pCheck, pgno, &nMinKey, i==0?NULL:&nMaxKey); if( i>0 && d2!=depth ){ - checkAppendMsg(pCheck, zContext, "Child page depth differs"); + checkAppendMsg(pCheck, "Child page depth differs"); } depth = d2; } @@ -8042,37 +8621,39 @@ static int checkTreePage( if( !pPage->leaf ){ pgno = get4byte(&pPage->aData[pPage->hdrOffset+8]); - sqlite3_snprintf(sizeof(zContext), zContext, - "On page %d at right child: ", iPage); + pCheck->zPfx = "On page %d at right child: "; + pCheck->v1 = iPage; #ifndef SQLITE_OMIT_AUTOVACUUM if( pBt->autoVacuum ){ - checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage, zContext); + checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage); } #endif - checkTreePage(pCheck, pgno, zContext, NULL, !pPage->nCell ? NULL : &nMaxKey); + checkTreePage(pCheck, pgno, NULL, !pPage->nCell?NULL:&nMaxKey); } /* For intKey leaf pages, check that the min/max keys are in order ** with any left/parent/right pages. */ + pCheck->zPfx = "Page %d: "; + pCheck->v1 = iPage; if( pPage->leaf && pPage->intKey ){ /* if we are a left child page */ if( pnParentMinKey ){ /* if we are the left most child page */ if( !pnParentMaxKey ){ if( nMaxKey > *pnParentMinKey ){ - checkAppendMsg(pCheck, zContext, + checkAppendMsg(pCheck, "Rowid %lld out of order (max larger than parent min of %lld)", nMaxKey, *pnParentMinKey); } }else{ if( nMinKey <= *pnParentMinKey ){ - checkAppendMsg(pCheck, zContext, + checkAppendMsg(pCheck, "Rowid %lld out of order (min less than parent min of %lld)", nMinKey, *pnParentMinKey); } if( nMaxKey > *pnParentMaxKey ){ - checkAppendMsg(pCheck, zContext, + checkAppendMsg(pCheck, "Rowid %lld out of order (max larger than parent max of %lld)", nMaxKey, *pnParentMaxKey); } @@ -8081,7 +8662,7 @@ static int checkTreePage( /* else if we're a right child page */ } else if( pnParentMaxKey ){ if( nMinKey <= *pnParentMaxKey ){ - checkAppendMsg(pCheck, zContext, + checkAppendMsg(pCheck, "Rowid %lld out of order (min less than parent max of %lld)", nMinKey, *pnParentMaxKey); } @@ -8093,6 +8674,7 @@ static int checkTreePage( data = pPage->aData; hdr = pPage->hdrOffset; hit = sqlite3PageMalloc( pBt->pageSize ); + pCheck->zPfx = 0; if( hit==0 ){ pCheck->mallocFailed = 1; }else{ @@ -8100,8 +8682,14 @@ static int checkTreePage( assert( contentOffset<=usableSize ); /* Enforced by btreeInitPage() */ memset(hit+contentOffset, 0, usableSize-contentOffset); memset(hit, 1, contentOffset); + /* EVIDENCE-OF: R-37002-32774 The two-byte integer at offset 3 gives the + ** number of cells on the page. */ nCell = get2byte(&data[hdr+3]); + /* EVIDENCE-OF: R-23882-45353 The cell pointer array of a b-tree page + ** immediately follows the b-tree page header. */ cellStart = hdr + 12 - 4*pPage->leaf; + /* EVIDENCE-OF: R-02776-14802 The cell pointer array consists of K 2-byte + ** integer offsets to the cell contents. */ for(i=0; i=usableSize ){ - checkAppendMsg(pCheck, 0, + pCheck->zPfx = 0; + checkAppendMsg(pCheck, "Corruption detected in cell %d on page %d",i,iPage); }else{ for(j=pc+size-1; j>=pc; j--) hit[j]++; } } + /* EVIDENCE-OF: R-20690-50594 The second field of the b-tree page header + ** is the offset of the first freeblock, or zero if there are no + ** freeblocks on the page. */ i = get2byte(&data[hdr+1]); while( i>0 ){ int size, j; @@ -8123,7 +8715,13 @@ static int checkTreePage( size = get2byte(&data[i+2]); assert( i+size<=usableSize ); /* Enforced by btreeInitPage() */ for(j=i+size-1; j>=i; j--) hit[j]++; + /* EVIDENCE-OF: R-58208-19414 The first 2 bytes of a freeblock are a + ** big-endian integer which is the offset in the b-tree page of the next + ** freeblock in the chain, or zero if the freeblock is the last on the + ** chain. */ j = get2byte(&data[i]); + /* EVIDENCE-OF: R-06866-39125 Freeblocks are always connected in order of + ** increasing offset. */ assert( j==0 || j>i+size ); /* Enforced by btreeInitPage() */ assert( j<=usableSize-4 ); /* Enforced by btreeInitPage() */ i = j; @@ -8132,19 +8730,29 @@ static int checkTreePage( if( hit[i]==0 ){ cnt++; }else if( hit[i]>1 ){ - checkAppendMsg(pCheck, 0, + checkAppendMsg(pCheck, "Multiple uses for byte %d of page %d", i, iPage); break; } } + /* EVIDENCE-OF: R-43263-13491 The total number of bytes in all fragments + ** is stored in the fifth field of the b-tree page header. + ** EVIDENCE-OF: R-07161-27322 The one-byte integer at offset 7 gives the + ** number of fragmented free bytes within the cell content area. + */ if( cnt!=data[hdr+7] ){ - checkAppendMsg(pCheck, 0, + checkAppendMsg(pCheck, "Fragmentation of %d bytes reported as %d on page %d", cnt, data[hdr+7], iPage); } } sqlite3PageFree(hit); releasePage(pPage); + +end_of_check: + pCheck->zPfx = saved_zPfx; + pCheck->v1 = saved_v1; + pCheck->v2 = saved_v2; return depth+1; } #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ @@ -8185,6 +8793,9 @@ char *sqlite3BtreeIntegrityCheck( sCheck.mxErr = mxErr; sCheck.nErr = 0; sCheck.mallocFailed = 0; + sCheck.zPfx = 0; + sCheck.v1 = 0; + sCheck.v2 = 0; *pnErr = 0; if( sCheck.nPage==0 ){ sqlite3BtreeLeave(p); @@ -8204,8 +8815,10 @@ char *sqlite3BtreeIntegrityCheck( /* Check the integrity of the freelist */ + sCheck.zPfx = "Main freelist: "; checkList(&sCheck, 1, get4byte(&pBt->pPage1->aData[32]), - get4byte(&pBt->pPage1->aData[36]), "Main freelist: "); + get4byte(&pBt->pPage1->aData[36])); + sCheck.zPfx = 0; /* Check all the tables. */ @@ -8213,10 +8826,12 @@ char *sqlite3BtreeIntegrityCheck( if( aRoot[i]==0 ) continue; #ifndef SQLITE_OMIT_AUTOVACUUM if( pBt->autoVacuum && aRoot[i]>1 ){ - checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0, 0); + checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0); } #endif - checkTreePage(&sCheck, aRoot[i], "List of tree roots: ", NULL, NULL); + sCheck.zPfx = "List of tree roots: "; + checkTreePage(&sCheck, aRoot[i], NULL, NULL); + sCheck.zPfx = 0; } /* Make sure every page in the file is referenced @@ -8224,7 +8839,7 @@ char *sqlite3BtreeIntegrityCheck( for(i=1; i<=sCheck.nPage && sCheck.mxErr; i++){ #ifdef SQLITE_OMIT_AUTOVACUUM if( getPageReferenced(&sCheck, i)==0 ){ - checkAppendMsg(&sCheck, 0, "Page %d is never used", i); + checkAppendMsg(&sCheck, "Page %d is never used", i); } #else /* If the database supports auto-vacuum, make sure no tables contain @@ -8232,11 +8847,11 @@ char *sqlite3BtreeIntegrityCheck( */ if( getPageReferenced(&sCheck, i)==0 && (PTRMAP_PAGENO(pBt, i)!=i || !pBt->autoVacuum) ){ - checkAppendMsg(&sCheck, 0, "Page %d is never used", i); + checkAppendMsg(&sCheck, "Page %d is never used", i); } if( getPageReferenced(&sCheck, i)!=0 && (PTRMAP_PAGENO(pBt, i)==i && pBt->autoVacuum) ){ - checkAppendMsg(&sCheck, 0, "Pointer map page %d is referenced", i); + checkAppendMsg(&sCheck, "Pointer map page %d is referenced", i); } #endif } @@ -8246,7 +8861,7 @@ char *sqlite3BtreeIntegrityCheck( ** of the integrity check. */ if( NEVER(nRef != sqlite3PagerRefcount(pBt->pPager)) ){ - checkAppendMsg(&sCheck, 0, + checkAppendMsg(&sCheck, "Outstanding page count goes from %d to %d during this analysis", nRef, sqlite3PagerRefcount(pBt->pPager) ); @@ -8442,7 +9057,7 @@ int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void *z){ ** required in case any of them are holding references to an xFetch ** version of the b-tree page modified by the accessPayload call below. ** - ** Note that pCsr must be open on a BTREE_INTKEY table and saveCursorPosition() + ** Note that pCsr must be open on a INTKEY table and saveCursorPosition() ** and hence saveAllCursors() cannot fail on a BTREE_INTKEY table, hence ** saveAllCursors can only return SQLITE_OK. */ @@ -8527,3 +9142,8 @@ void sqlite3BtreeCursorHints(BtCursor *pCsr, unsigned int mask){ int sqlite3BtreeIsReadonly(Btree *p){ return (p->pBt->btsFlags & BTS_READ_ONLY)!=0; } + +/* +** Return the size of the header added to each page by this module. +*/ +int sqlite3HeaderSizeBtree(void){ return ROUND8(sizeof(MemPage)); } diff --git a/src/btree.h b/src/btree.h index 7118534f93..b57d500c77 100644 --- a/src/btree.h +++ b/src/btree.h @@ -19,7 +19,7 @@ /* TODO: This definition is just included so other modules compile. It ** needs to be revisited. */ -#define SQLITE_N_BTREE_META 10 +#define SQLITE_N_BTREE_META 16 /* ** If defined as non-zero, auto-vacuum is enabled by default. Otherwise @@ -83,7 +83,7 @@ int sqlite3BtreeBeginTrans(Btree*,int); int sqlite3BtreeCommitPhaseOne(Btree*, const char *zMaster); int sqlite3BtreeCommitPhaseTwo(Btree*, int); int sqlite3BtreeCommit(Btree*); -int sqlite3BtreeRollback(Btree*,int); +int sqlite3BtreeRollback(Btree*,int,int); int sqlite3BtreeBeginStmt(Btree*,int); int sqlite3BtreeCreateTable(Btree*, int*, int flags); int sqlite3BtreeIsInTrans(Btree*); @@ -116,7 +116,7 @@ int sqlite3BtreeIncrVacuum(Btree *); int sqlite3BtreeDropTable(Btree*, int, int*); int sqlite3BtreeClearTable(Btree*, int, int*); int sqlite3BtreeClearTableOfCursor(BtCursor*); -void sqlite3BtreeTripAllCursors(Btree*, int); +int sqlite3BtreeTripAllCursors(Btree*, int, int); void sqlite3BtreeGetMeta(Btree *pBtree, int idx, u32 *pValue); int sqlite3BtreeUpdateMeta(Btree*, int idx, u32 value); @@ -134,6 +134,11 @@ int sqlite3BtreeNewDb(Btree *p); ** For example, the free-page-count field is located at byte offset 36 of ** the database file header. The incr-vacuum-flag field is located at ** byte offset 64 (== 36+4*7). +** +** The BTREE_DATA_VERSION value is not really a value stored in the header. +** It is a read-only number computed by the pager. But we merge it with +** the header value access routines since its access pattern is the same. +** Call it a "virtual meta value". */ #define BTREE_FREE_PAGE_COUNT 0 #define BTREE_SCHEMA_VERSION 1 @@ -144,6 +149,7 @@ int sqlite3BtreeNewDb(Btree *p); #define BTREE_USER_VERSION 6 #define BTREE_INCR_VACUUM 7 #define BTREE_APPLICATION_ID 8 +#define BTREE_DATA_VERSION 15 /* A virtual meta-value */ /* ** Values that may be OR'd together to form the second argument of an @@ -169,7 +175,8 @@ int sqlite3BtreeMovetoUnpacked( int bias, int *pRes ); -int sqlite3BtreeCursorHasMoved(BtCursor*, int*); +int sqlite3BtreeCursorHasMoved(BtCursor*); +int sqlite3BtreeCursorRestore(BtCursor*, int*); int sqlite3BtreeDelete(BtCursor*); int sqlite3BtreeInsert(BtCursor*, const void *pKey, i64 nKey, const void *pData, int nData, @@ -195,6 +202,7 @@ void sqlite3BtreeClearCursor(BtCursor *); int sqlite3BtreeSetVersion(Btree *pBt, int iVersion); void sqlite3BtreeCursorHints(BtCursor *, unsigned int mask); int sqlite3BtreeIsReadonly(Btree *pBt); +int sqlite3HeaderSizeBtree(void); #ifndef NDEBUG int sqlite3BtreeCursorIsValid(BtCursor*); diff --git a/src/btreeInt.h b/src/btreeInt.h index d1cdd46983..ed4d75ee9f 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -9,7 +9,7 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** This file implements a external (disk-based) database using BTrees. +** This file implements an external (disk-based) database using BTrees. ** For a detailed discussion of BTrees, refer to ** ** Donald E. Knuth, THE ART OF COMPUTER PROGRAMMING, Volume 3: @@ -135,7 +135,7 @@ ** ** The flags define the format of this btree page. The leaf flag means that ** this page has no children. The zerodata flag means that this page carries -** only keys and no data. The intkey flag means that the key is a integer +** only keys and no data. The intkey flag means that the key is an integer ** which is stored in the key size entry of the cell header rather than in ** the payload area. ** @@ -273,9 +273,10 @@ typedef struct BtLock BtLock; struct MemPage { u8 isInit; /* True if previously initialized. MUST BE FIRST! */ u8 nOverflow; /* Number of overflow cell bodies in aCell[] */ - u8 intKey; /* True if intkey flag is set */ - u8 leaf; /* True if leaf flag is set */ - u8 hasData; /* True if this page stores data */ + u8 intKey; /* True if table b-trees. False for index b-trees */ + u8 intKeyLeaf; /* True if the leaf of an intKey table */ + u8 noPayload; /* True if internal intKey page (thus w/o data) */ + u8 leaf; /* True if a leaf page */ u8 hdrOffset; /* 100 for page 1. 0 otherwise */ u8 childPtrSize; /* 0 if leaf==1. 4 if leaf==0 */ u8 max1bytePayload; /* min(maxLocal,127) */ @@ -350,6 +351,7 @@ struct Btree { u8 locked; /* True if db currently has pBt locked */ int wantToLock; /* Number of nested calls to sqlite3BtreeEnter() */ int nBackup; /* Number of backup operations reading this btree */ + u32 iDataVersion; /* Combines with pBt->pPager->iDataVersion */ Btree *pNext; /* List of other sharable Btrees from the same db */ Btree *pPrev; /* Back pointer of the same list */ #ifndef SQLITE_OMIT_SHARED_CACHE @@ -435,7 +437,7 @@ struct BtShared { BtLock *pLock; /* List of locks held on this shared-btree struct */ Btree *pWriter; /* Btree with currently open write transaction */ #endif - u8 *pTmpSpace; /* BtShared.pageSize bytes of space for tmp use */ + u8 *pTmpSpace; /* Temp space sufficient to hold a single cell */ }; /* @@ -456,12 +458,10 @@ struct BtShared { */ typedef struct CellInfo CellInfo; struct CellInfo { - i64 nKey; /* The key for INTKEY tables, or number of bytes in key */ - u8 *pCell; /* Pointer to the start of cell content */ - u32 nData; /* Number of bytes of data */ - u32 nPayload; /* Total amount of payload */ - u16 nHeader; /* Size of the cell content header in bytes */ - u16 nLocal; /* Amount of payload held locally */ + i64 nKey; /* The key for INTKEY tables, or nPayload otherwise */ + u8 *pPayload; /* Pointer to the start of payload */ + u32 nPayload; /* Bytes of payload */ + u16 nLocal; /* Amount of payload held locally, not on overflow */ u16 iOverflow; /* Offset to overflow page number. Zero if no overflow */ u16 nSize; /* Size of the cell content on the main b-tree page */ }; @@ -490,6 +490,11 @@ struct CellInfo { ** ** Fields in this structure are accessed under the BtShared.mutex ** found at self->pBt->mutex. +** +** skipNext meaning: +** eState==SKIPNEXT && skipNext>0: Next sqlite3BtreeNext() is no-op. +** eState==SKIPNEXT && skipNext<0: Next sqlite3BtreePrevious() is no-op. +** eState==FAULT: Cursor fault with skipNext as error code. */ struct BtCursor { Btree *pBtree; /* The Btree to which this cursor belongs */ @@ -502,7 +507,8 @@ struct BtCursor { void *pKey; /* Saved key that was cursor last known position */ Pgno pgnoRoot; /* The root page of this tree */ int nOvflAlloc; /* Allocated size of aOverflow[] array */ - int skipNext; /* Prev() is noop if negative. Next() is noop if positive */ + int skipNext; /* Prev() is noop if negative. Next() is noop if positive. + ** Error code if eState==CURSOR_FAULT */ u8 curFlags; /* zero or more BTCF_* flags defined below */ u8 eState; /* One of the CURSOR_XXX constants (see below) */ u8 hints; /* As configured by CursorSetHints() */ @@ -544,11 +550,11 @@ struct BtCursor { ** seek the cursor to the saved position. ** ** CURSOR_FAULT: -** A unrecoverable error (an I/O error or a malloc failure) has occurred +** An unrecoverable error (an I/O error or a malloc failure) has occurred ** on a different connection that shares the BtShared cache with this ** cursor. The error has left the cache in an inconsistent state. ** Do nothing else with this cursor. Any attempt to use the cursor -** should return the error code stored in BtCursor.skip +** should return the error code stored in BtCursor.skipNext */ #define CURSOR_INVALID 0 #define CURSOR_VALID 1 @@ -658,6 +664,8 @@ struct IntegrityCk { int mxErr; /* Stop accumulating errors when this reaches zero */ int nErr; /* Number of messages written to zErrMsg so far */ int mallocFailed; /* A memory allocation error has occurred */ + const char *zPfx; /* Error message prefix */ + int v1, v2; /* Values for up to two %d fields in zPfx */ StrAccum errMsg; /* Accumulate the error message text here */ }; diff --git a/src/build.c b/src/build.c index 677ba326c6..f02989bffe 100644 --- a/src/build.c +++ b/src/build.c @@ -113,6 +113,19 @@ static void codeTableLocks(Parse *pParse){ #define codeTableLocks(x) #endif +/* +** Return TRUE if the given yDbMask object is empty - if it contains no +** 1 bits. This routine is used by the DbMaskAllZero() and DbMaskNotZero() +** macros when SQLITE_MAX_ATTACHED is greater than 30. +*/ +#if SQLITE_MAX_ATTACHED>30 +int sqlite3DbMaskAllZero(yDbMask m){ + int i; + for(i=0; inTableLock>0 && db->init.busy==0 ){ + sqlite3UserAuthInit(db); + if( db->auth.authLevelrc = SQLITE_AUTH_USER; + sqlite3ErrorMsg(pParse, "user not authenticated"); + return; + } + } +#endif + /* The cookie mask contains one bit for each database file open. ** (Bit 0 is for main, bit 1 is for temp, and so forth.) Bits are ** set for each database that is used. Generate code to start a ** transaction on each used database and to verify the schema cookie ** on each used database. */ - if( db->mallocFailed==0 && (pParse->cookieMask || pParse->pConstExpr) ){ - yDbMask mask; + if( db->mallocFailed==0 + && (DbMaskNonZero(pParse->cookieMask) || pParse->pConstExpr) + ){ int iDb, i; assert( sqlite3VdbeGetOp(v, 0)->opcode==OP_Init ); sqlite3VdbeJumpHere(v, 0); - for(iDb=0, mask=1; iDbnDb; mask<<=1, iDb++){ - if( (mask & pParse->cookieMask)==0 ) continue; + for(iDb=0; iDbnDb; iDb++){ + if( DbMaskTest(pParse->cookieMask, iDb)==0 ) continue; sqlite3VdbeUsesBtree(v, iDb); sqlite3VdbeAddOp4Int(v, OP_Transaction, /* Opcode */ iDb, /* P1 */ - (mask & pParse->writeMask)!=0, /* P2 */ + DbMaskTest(pParse->writeMask,iDb), /* P2 */ pParse->cookieValue[iDb], /* P3 */ db->aDb[iDb].pSchema->iGeneration /* P4 */ ); @@ -216,7 +241,7 @@ void sqlite3FinishCoding(Parse *pParse){ pParse->nMem = 0; pParse->nSet = 0; pParse->nVar = 0; - pParse->cookieMask = 0; + DbMaskZero(pParse->cookieMask); } /* @@ -257,6 +282,16 @@ void sqlite3NestedParse(Parse *pParse, const char *zFormat, ...){ pParse->nested--; } +#if SQLITE_USER_AUTHENTICATION +/* +** Return TRUE if zTable is the name of the system table that stores the +** list of users and their access credentials. +*/ +int sqlite3UserAuthTable(const char *zTable){ + return sqlite3_stricmp(zTable, "sqlite_user")==0; +} +#endif + /* ** Locate the in-memory structure that describes a particular database ** table given the name of that table and (optionally) the name of the @@ -272,16 +307,25 @@ void sqlite3NestedParse(Parse *pParse, const char *zFormat, ...){ Table *sqlite3FindTable(sqlite3 *db, const char *zName, const char *zDatabase){ Table *p = 0; int i; - int nName; - assert( zName!=0 ); - nName = sqlite3Strlen30(zName); + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 ) return 0; +#endif + /* All mutexes are required for schema access. Make sure we hold them. */ assert( zDatabase!=0 || sqlite3BtreeHoldsAllMutexes(db) ); +#if SQLITE_USER_AUTHENTICATION + /* Only the admin user is allowed to know that the sqlite_user table + ** exists */ + if( db->auth.authLevelnDb; i++){ int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ if( zDatabase!=0 && sqlite3StrICmp(zDatabase, db->aDb[j].zName) ) continue; assert( sqlite3SchemaMutexHeld(db, j, 0) ); - p = sqlite3HashFind(&db->aDb[j].pSchema->tblHash, zName, nName); + p = sqlite3HashFind(&db->aDb[j].pSchema->tblHash, zName); if( p ) break; } return p; @@ -321,6 +365,12 @@ Table *sqlite3LocateTable( } pParse->checkSchema = 1; } +#if SQLITE_USER_AUTHENICATION + else if( pParse->db->auth.authLevelnDb; i++){ @@ -373,7 +422,7 @@ Index *sqlite3FindIndex(sqlite3 *db, const char *zName, const char *zDb){ assert( pSchema ); if( zDb && sqlite3StrICmp(zDb, db->aDb[j].zName) ) continue; assert( sqlite3SchemaMutexHeld(db, j, 0) ); - p = sqlite3HashFind(&pSchema->idxHash, zName, nName); + p = sqlite3HashFind(&pSchema->idxHash, zName); if( p ) break; } return p; @@ -386,10 +435,12 @@ static void freeIndex(sqlite3 *db, Index *p){ #ifndef SQLITE_OMIT_ANALYZE sqlite3DeleteIndexSamples(db, p); #endif - if( db==0 || db->pnBytesFreed==0 ) sqlite3KeyInfoUnref(p->pKeyInfo); sqlite3ExprDelete(db, p->pPartIdxWhere); sqlite3DbFree(db, p->zColAff); if( p->isResized ) sqlite3DbFree(db, p->azColl); +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + sqlite3_free(p->aiRowEst); +#endif sqlite3DbFree(db, p); } @@ -401,13 +452,11 @@ static void freeIndex(sqlite3 *db, Index *p){ */ void sqlite3UnlinkAndDeleteIndex(sqlite3 *db, int iDb, const char *zIdxName){ Index *pIndex; - int len; Hash *pHash; assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); pHash = &db->aDb[iDb].pSchema->idxHash; - len = sqlite3Strlen30(zIdxName); - pIndex = sqlite3HashInsert(pHash, zIdxName, len, 0); + pIndex = sqlite3HashInsert(pHash, zIdxName, 0); if( ALWAYS(pIndex) ){ if( pIndex->pTable->pIndex==pIndex ){ pIndex->pTable->pIndex = pIndex->pNext; @@ -567,7 +616,7 @@ void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ if( !db || db->pnBytesFreed==0 ){ char *zName = pIndex->zName; TESTONLY ( Index *pOld = ) sqlite3HashInsert( - &pIndex->pSchema->idxHash, zName, sqlite3Strlen30(zName), 0 + &pIndex->pSchema->idxHash, zName, 0 ); assert( db==0 || sqlite3SchemaMutexHeld(db, 0, pIndex->pSchema) ); assert( pOld==pIndex || pOld==0 ); @@ -610,8 +659,7 @@ void sqlite3UnlinkAndDeleteTable(sqlite3 *db, int iDb, const char *zTabName){ assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); testcase( zTabName[0]==0 ); /* Zero-length table names are allowed */ pDb = &db->aDb[iDb]; - p = sqlite3HashInsert(&pDb->pSchema->tblHash, zTabName, - sqlite3Strlen30(zTabName),0); + p = sqlite3HashInsert(&pDb->pSchema->tblHash, zTabName, 0); sqlite3DeleteTable(db, p); db->flags |= SQLITE_InternChanges; } @@ -1135,7 +1183,7 @@ char sqlite3AffinityType(const char *zIn, u8 *pszEst){ ** estimate is scaled so that the size of an integer is 1. */ if( pszEst ){ *pszEst = 1; /* default size is approx 4 bytes */ - if( aff<=SQLITE_AFF_NONE ){ + if( affpNewTable; if( p!=0 ){ pCol = &(p->aCol[p->nCol-1]); - if( !sqlite3ExprIsConstantOrFunction(pSpan->pExpr) ){ + if( !sqlite3ExprIsConstantOrFunction(pSpan->pExpr, db->init.busy) ){ sqlite3ErrorMsg(pParse, "default value of column [%s] is not constant", pCol->zName); }else{ @@ -1506,8 +1554,8 @@ static char *createTableStmt(sqlite3 *db, Table *p){ zStmt[k++] = '('; for(pCol=p->aCol, i=0; inCol; i++, pCol++){ static const char * const azType[] = { - /* SQLITE_AFF_TEXT */ " TEXT", /* SQLITE_AFF_NONE */ "", + /* SQLITE_AFF_TEXT */ " TEXT", /* SQLITE_AFF_NUMERIC */ " NUM", /* SQLITE_AFF_INTEGER */ " INT", /* SQLITE_AFF_REAL */ " REAL" @@ -1519,15 +1567,15 @@ static char *createTableStmt(sqlite3 *db, Table *p){ k += sqlite3Strlen30(&zStmt[k]); zSep = zSep2; identPut(zStmt, &k, pCol->zName); - assert( pCol->affinity-SQLITE_AFF_TEXT >= 0 ); - assert( pCol->affinity-SQLITE_AFF_TEXT < ArraySize(azType) ); - testcase( pCol->affinity==SQLITE_AFF_TEXT ); + assert( pCol->affinity-SQLITE_AFF_NONE >= 0 ); + assert( pCol->affinity-SQLITE_AFF_NONE < ArraySize(azType) ); testcase( pCol->affinity==SQLITE_AFF_NONE ); + testcase( pCol->affinity==SQLITE_AFF_TEXT ); testcase( pCol->affinity==SQLITE_AFF_NUMERIC ); testcase( pCol->affinity==SQLITE_AFF_INTEGER ); testcase( pCol->affinity==SQLITE_AFF_REAL ); - zType = azType[pCol->affinity - SQLITE_AFF_TEXT]; + zType = azType[pCol->affinity - SQLITE_AFF_NONE]; len = sqlite3Strlen30(zType); assert( pCol->affinity==SQLITE_AFF_NONE || pCol->affinity==sqlite3AffinityType(zType, 0) ); @@ -1611,7 +1659,7 @@ static int hasColumn(const i16 *aiCol, int nCol, int x){ ** no rowid btree for a WITHOUT ROWID. Instead, the canonical ** data storage is a covering index btree. ** (2) Bypass the creation of the sqlite_master table entry -** for the PRIMARY KEY as the the primary key index is now +** for the PRIMARY KEY as the primary key index is now ** identified by the sqlite_master table entry of the table itself. ** (3) Set the Index.tnum of the PRIMARY KEY Index object in the ** schema to the rootpage from the main table. @@ -1632,7 +1680,7 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ Vdbe *v = pParse->pVdbe; /* Convert the OP_CreateTable opcode that would normally create the - ** root-page for the table into a OP_CreateIndex opcode. The index + ** root-page for the table into an OP_CreateIndex opcode. The index ** created will become the PRIMARY KEY index. */ if( pParse->addrCrTab ){ @@ -1665,6 +1713,19 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ pTab->iPKey = -1; }else{ pPk = sqlite3PrimaryKeyIndex(pTab); + /* + ** Remove all redundant columns from the PRIMARY KEY. For example, change + ** "PRIMARY KEY(a,b,a,b,c,b,c,d)" into just "PRIMARY KEY(a,b,c,d)". Later + ** code assumes the PRIMARY KEY contains no repeated columns. + */ + for(i=j=1; inKeyCol; i++){ + if( hasColumn(pPk->aiColumn, j, pPk->aiColumn[i]) ){ + pPk->nColumn--; + }else{ + pPk->aiColumn[j++] = pPk->aiColumn[i]; + } + } + pPk->nKeyCol = j; } pPk->isCovering = 1; assert( pPk!=0 ); @@ -1933,8 +1994,7 @@ void sqlite3EndTable( Table *pOld; Schema *pSchema = p->pSchema; assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - pOld = sqlite3HashInsert(&pSchema->tblHash, p->zName, - sqlite3Strlen30(p->zName),p); + pOld = sqlite3HashInsert(&pSchema->tblHash, p->zName, p); if( pOld ){ assert( p==pOld ); /* Malloc must have failed inside HashInsert() */ db->mallocFailed = 1; @@ -2045,7 +2105,7 @@ int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ int nErr = 0; /* Number of errors encountered */ int n; /* Temporarily holds the number of cursors assigned */ sqlite3 *db = pParse->db; /* Database connection for malloc errors */ - int (*xAuth)(void*,int,const char*,const char*,const char*,const char*); + sqlite3_xauth xAuth; /* Saved xAuth pointer */ assert( pTable ); @@ -2116,7 +2176,7 @@ int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ pSelTab->aCol = 0; sqlite3DeleteTable(db, pSelTab); assert( sqlite3SchemaMutexHeld(db, 0, pTable->pSchema) ); - pTable->pSchema->flags |= DB_UnresetViews; + pTable->pSchema->schemaFlags |= DB_UnresetViews; }else{ pTable->nCol = 0; nErr++; @@ -2584,7 +2644,7 @@ void sqlite3CreateForeignKey( assert( sqlite3SchemaMutexHeld(db, 0, p->pSchema) ); pNextTo = (FKey *)sqlite3HashInsert(&p->pSchema->fkeyHash, - pFKey->zTo, sqlite3Strlen30(pFKey->zTo), (void *)pFKey + pFKey->zTo, (void *)pFKey ); if( pNextTo==pFKey ){ db->mallocFailed = 1; @@ -2647,7 +2707,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ int iPartIdxLabel; /* Jump to this label to skip a row */ Vdbe *v; /* Generate code into this virtual machine */ KeyInfo *pKey; /* KeyInfo for index */ - int regRecord; /* Register holding assemblied index record */ + int regRecord; /* Register holding assembled index record */ sqlite3 *db = pParse->db; /* The database connection */ int iDb = sqlite3SchemaToIndex(db, pIndex->pSchema); @@ -2672,7 +2732,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ /* Open the sorter cursor if we are to use one. */ iSorter = pParse->nTab++; - sqlite3VdbeAddOp4(v, OP_SorterOpen, iSorter, 0, 0, (char*) + sqlite3VdbeAddOp4(v, OP_SorterOpen, iSorter, 0, pIndex->nKeyCol, (char*) sqlite3KeyInfoRef(pKey), P4_KEYINFO); /* Open the table. Loop through all rows of the table, inserting index @@ -2693,17 +2753,17 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ addr1 = sqlite3VdbeAddOp2(v, OP_SorterSort, iSorter, 0); VdbeCoverage(v); assert( pKey!=0 || db->mallocFailed || pParse->nErr ); - if( pIndex->onError!=OE_None && pKey!=0 ){ + if( IsUniqueIndex(pIndex) && pKey!=0 ){ int j2 = sqlite3VdbeCurrentAddr(v) + 3; sqlite3VdbeAddOp2(v, OP_Goto, 0, j2); addr2 = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp4Int(v, OP_SorterCompare, iSorter, j2, regRecord, - pKey->nField - pIndex->nKeyCol); VdbeCoverage(v); + pIndex->nKeyCol); VdbeCoverage(v); sqlite3UniqueConstraint(pParse, OE_Abort, pIndex); }else{ addr2 = sqlite3VdbeCurrentAddr(v); } - sqlite3VdbeAddOp2(v, OP_SorterData, iSorter, regRecord); + sqlite3VdbeAddOp3(v, OP_SorterData, iSorter, regRecord, iIdx); sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, 1); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); sqlite3ReleaseTempReg(pParse, regRecord); @@ -2860,6 +2920,10 @@ Index *sqlite3CreateIndex( assert( pTab!=0 ); assert( pParse->nErr==0 ); if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 + && db->init.busy==0 +#if SQLITE_USER_AUTHENTICATION + && sqlite3UserAuthTable(pTab->zName)==0 +#endif && sqlite3StrNICmp(&pTab->zName[7],"altertab_",9)!=0 ){ sqlite3ErrorMsg(pParse, "table %s may not be indexed", pTab->zName); goto exit_create_index; @@ -3021,7 +3085,7 @@ Index *sqlite3CreateIndex( pParse->checkSchema = 1; goto exit_create_index; } - assert( pTab->nCol<=0x7fff && j<=0x7fff ); + assert( j<=0x7fff ); pIndex->aiColumn[i] = (i16)j; if( pListItem->pExpr ){ int nColl; @@ -3090,9 +3154,9 @@ Index *sqlite3CreateIndex( Index *pIdx; for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ int k; - assert( pIdx->onError!=OE_None ); + assert( IsUniqueIndex(pIdx) ); assert( pIdx->idxType!=SQLITE_IDXTYPE_APPDEF ); - assert( pIndex->onError!=OE_None ); + assert( IsUniqueIndex(pIndex) ); if( pIdx->nKeyCol!=pIndex->nKeyCol ) continue; for(k=0; knKeyCol; k++){ @@ -3132,8 +3196,7 @@ Index *sqlite3CreateIndex( Index *p; assert( sqlite3SchemaMutexHeld(db, 0, pIndex->pSchema) ); p = sqlite3HashInsert(&pIndex->pSchema->idxHash, - pIndex->zName, sqlite3Strlen30(pIndex->zName), - pIndex); + pIndex->zName, pIndex); if( p ){ assert( p==pIndex ); /* Malloc must have failed */ db->mallocFailed = 1; @@ -3248,7 +3311,7 @@ exit_create_index: ** Fill the Index.aiRowEst[] array with default information - information ** to be used when we have not run the ANALYZE command. ** -** aiRowEst[0] is suppose to contain the number of elements in the index. +** aiRowEst[0] is supposed to contain the number of elements in the index. ** Since we do not know, guess 1 million. aiRowEst[1] is an estimate of the ** number of rows in the table that match any particular value of the ** first column of the index. aiRowEst[2] is an estimate of the number @@ -3283,7 +3346,7 @@ void sqlite3DefaultRowEst(Index *pIdx){ } assert( 0==sqlite3LogEst(1) ); - if( pIdx->onError!=OE_None ) a[pIdx->nKeyCol] = 0; + if( IsUniqueIndex(pIdx) ) a[pIdx->nKeyCol] = 0; } /* @@ -3627,7 +3690,7 @@ void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ ** if this is the first term of the FROM clause. pTable and pDatabase ** are the name of the table and database named in the FROM clause term. ** pDatabase is NULL if the database name qualifier is missing - the -** usual case. If the term has a alias, then pAlias points to the +** usual case. If the term has an alias, then pAlias points to the ** alias token. If the term is a subquery, then pSubquery is the ** SELECT statement that the subquery encodes. The pTable and ** pDatabase parameters are NULL for subqueries. The pOn and pUsing @@ -3843,15 +3906,13 @@ int sqlite3OpenTempDatabase(Parse *pParse){ void sqlite3CodeVerifySchema(Parse *pParse, int iDb){ Parse *pToplevel = sqlite3ParseToplevel(pParse); sqlite3 *db = pToplevel->db; - yDbMask mask; assert( iDb>=0 && iDbnDb ); assert( db->aDb[iDb].pBt!=0 || iDb==1 ); assert( iDbcookieMask & mask)==0 ){ - pToplevel->cookieMask |= mask; + if( DbMaskTest(pToplevel->cookieMask, iDb)==0 ){ + DbMaskSet(pToplevel->cookieMask, iDb); pToplevel->cookieValue[iDb] = db->aDb[iDb].pSchema->schema_cookie; if( !OMIT_TEMPDB && iDb==1 ){ sqlite3OpenTempDatabase(pToplevel); @@ -3890,7 +3951,7 @@ void sqlite3CodeVerifyNamedSchema(Parse *pParse, const char *zDb){ void sqlite3BeginWriteOperation(Parse *pParse, int setStatement, int iDb){ Parse *pToplevel = sqlite3ParseToplevel(pParse); sqlite3CodeVerifySchema(pParse, iDb); - pToplevel->writeMask |= ((yDbMask)1)<writeMask, iDb); pToplevel->isMultiWrite |= setStatement; } @@ -4141,40 +4202,31 @@ void sqlite3Reindex(Parse *pParse, Token *pName1, Token *pName2){ ** when it has finished using it. */ KeyInfo *sqlite3KeyInfoOfIndex(Parse *pParse, Index *pIdx){ + int i; + int nCol = pIdx->nColumn; + int nKey = pIdx->nKeyCol; + KeyInfo *pKey; if( pParse->nErr ) return 0; -#ifndef SQLITE_OMIT_SHARED_CACHE - if( pIdx->pKeyInfo && pIdx->pKeyInfo->db!=pParse->db ){ - sqlite3KeyInfoUnref(pIdx->pKeyInfo); - pIdx->pKeyInfo = 0; + if( pIdx->uniqNotNull ){ + pKey = sqlite3KeyInfoAlloc(pParse->db, nKey, nCol-nKey); + }else{ + pKey = sqlite3KeyInfoAlloc(pParse->db, nCol, 0); } -#endif - if( pIdx->pKeyInfo==0 ){ - int i; - int nCol = pIdx->nColumn; - int nKey = pIdx->nKeyCol; - KeyInfo *pKey; - if( pIdx->uniqNotNull ){ - pKey = sqlite3KeyInfoAlloc(pParse->db, nKey, nCol-nKey); - }else{ - pKey = sqlite3KeyInfoAlloc(pParse->db, nCol, 0); + if( pKey ){ + assert( sqlite3KeyInfoIsWriteable(pKey) ); + for(i=0; iazColl[i]; + assert( zColl!=0 ); + pKey->aColl[i] = strcmp(zColl,"BINARY")==0 ? 0 : + sqlite3LocateCollSeq(pParse, zColl); + pKey->aSortOrder[i] = pIdx->aSortOrder[i]; } - if( pKey ){ - assert( sqlite3KeyInfoIsWriteable(pKey) ); - for(i=0; iazColl[i]; - assert( zColl!=0 ); - pKey->aColl[i] = strcmp(zColl,"BINARY")==0 ? 0 : - sqlite3LocateCollSeq(pParse, zColl); - pKey->aSortOrder[i] = pIdx->aSortOrder[i]; - } - if( pParse->nErr ){ - sqlite3KeyInfoUnref(pKey); - }else{ - pIdx->pKeyInfo = pKey; - } + if( pParse->nErr ){ + sqlite3KeyInfoUnref(pKey); + pKey = 0; } } - return sqlite3KeyInfoRef(pIdx->pKeyInfo); + return pKey; } #ifndef SQLITE_OMIT_CTE diff --git a/src/callback.c b/src/callback.c index 260fe806bb..cd213b4b28 100644 --- a/src/callback.c +++ b/src/callback.c @@ -142,7 +142,7 @@ int sqlite3CheckCollSeq(Parse *pParse, CollSeq *pColl){ ** ** Each pointer stored in the sqlite3.aCollSeq hash table contains an ** array of three CollSeq structures. The first is the collation sequence -** prefferred for UTF-8, the second UTF-16le, and the third UTF-16be. +** preferred for UTF-8, the second UTF-16le, and the third UTF-16be. ** ** Stored immediately after the three collation sequences is a copy of ** the collation sequence name. A pointer to this string is stored in @@ -154,11 +154,11 @@ static CollSeq *findCollSeqEntry( int create /* Create a new entry if true */ ){ CollSeq *pColl; - int nName = sqlite3Strlen30(zName); - pColl = sqlite3HashFind(&db->aCollSeq, zName, nName); + pColl = sqlite3HashFind(&db->aCollSeq, zName); if( 0==pColl && create ){ - pColl = sqlite3DbMallocZero(db, 3*sizeof(*pColl) + nName + 1 ); + int nName = sqlite3Strlen30(zName); + pColl = sqlite3DbMallocZero(db, 3*sizeof(*pColl) + nName + 1); if( pColl ){ CollSeq *pDel = 0; pColl[0].zName = (char*)&pColl[3]; @@ -169,7 +169,7 @@ static CollSeq *findCollSeqEntry( pColl[2].enc = SQLITE_UTF16BE; memcpy(pColl[0].zName, zName, nName); pColl[0].zName[nName] = 0; - pDel = sqlite3HashInsert(&db->aCollSeq, pColl[0].zName, nName, pColl); + pDel = sqlite3HashInsert(&db->aCollSeq, pColl[0].zName, pColl); /* If a malloc() failure occurred in sqlite3HashInsert(), it will ** return the pColl pointer to be deleted (because it wasn't added @@ -447,9 +447,9 @@ void sqlite3SchemaClear(void *p){ sqlite3HashClear(&temp1); sqlite3HashClear(&pSchema->fkeyHash); pSchema->pSeqTab = 0; - if( pSchema->flags & DB_SchemaLoaded ){ + if( pSchema->schemaFlags & DB_SchemaLoaded ){ pSchema->iGeneration++; - pSchema->flags &= ~DB_SchemaLoaded; + pSchema->schemaFlags &= ~DB_SchemaLoaded; } } diff --git a/src/complete.c b/src/complete.c index 9e9140085c..f7a35cc6f3 100644 --- a/src/complete.c +++ b/src/complete.c @@ -70,7 +70,7 @@ extern const char sqlite3IsEbcdicIdChar[]; ** a statement. ** ** (4) CREATE The keyword CREATE has been seen at the beginning of a -** statement, possibly preceeded by EXPLAIN and/or followed by +** statement, possibly preceded by EXPLAIN and/or followed by ** TEMP or TEMPORARY ** ** (5) TRIGGER We are in the middle of a trigger definition that must be @@ -80,7 +80,7 @@ extern const char sqlite3IsEbcdicIdChar[]; ** the end of a trigger definition. ** ** (7) END We've seen the ";END" of the ";END;" that occurs at the end -** of a trigger difinition. +** of a trigger definition. ** ** Transitions between states above are determined by tokens extracted ** from the input. The following tokens are significant: @@ -123,7 +123,7 @@ int sqlite3_complete(const char *zSql){ }; #else /* If triggers are not supported by this compile then the statement machine - ** used to detect the end of a statement is much simplier + ** used to detect the end of a statement is much simpler */ static const u8 trans[3][3] = { /* Token: */ @@ -134,6 +134,13 @@ int sqlite3_complete(const char *zSql){ }; #endif /* SQLITE_OMIT_TRIGGER */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( zSql==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + while( *zSql ){ switch( *zSql ){ case ';': { /* A semicolon */ diff --git a/src/ctime.c b/src/ctime.c index 286f66e061..59dc972d8d 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -63,6 +63,9 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_DISABLE_LFS "DISABLE_LFS", #endif +#ifdef SQLITE_ENABLE_API_ARMOR + "ENABLE_API_ARMOR", +#endif #ifdef SQLITE_ENABLE_ATOMIC_WRITE "ENABLE_ATOMIC_WRITE", #endif @@ -368,6 +371,9 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_USE_ALLOCA "USE_ALLOCA", #endif +#ifdef SQLITE_USER_AUTHENTICATION + "USER_AUTHENTICATION", +#endif #ifdef SQLITE_WIN32_MALLOC "WIN32_MALLOC", #endif @@ -385,6 +391,13 @@ static const char * const azCompileOpt[] = { */ int sqlite3_compileoption_used(const char *zOptName){ int i, n; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( zOptName==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif if( sqlite3StrNICmp(zOptName, "SQLITE_", 7)==0 ) zOptName += 7; n = sqlite3Strlen30(zOptName); @@ -392,7 +405,7 @@ int sqlite3_compileoption_used(const char *zOptName){ ** linear search is adequate. No need for a binary search. */ for(i=0; inMem; sqlite3VdbeAddOp2(v, OP_Null, 0, iRowSet); }else{ - /* For a WITHOUT ROWID table, create an ephermeral table used to + /* For a WITHOUT ROWID table, create an ephemeral table used to ** hold all primary keys for rows to be deleted. */ pPk = sqlite3PrimaryKeyIndex(pTab); assert( pPk!=0 ); @@ -466,10 +466,11 @@ void sqlite3DeleteFrom( ** triggers. */ if( !isView ){ + testcase( IsVirtual(pTab) ); sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, iTabCur, aToOpen, &iDataCur, &iIdxCur); - assert( pPk || iDataCur==iTabCur ); - assert( pPk || iIdxCur==iDataCur+1 ); + assert( pPk || IsVirtual(pTab) || iDataCur==iTabCur ); + assert( pPk || IsVirtual(pTab) || iIdxCur==iDataCur+1 ); } /* Set up a loop over the rowids/primary-keys that were found in the @@ -477,9 +478,10 @@ void sqlite3DeleteFrom( */ if( okOnePass ){ /* Just one row. Hence the top-of-loop is a no-op */ - assert( nKey==nPk ); /* OP_Found will use an unpacked key */ + assert( nKey==nPk ); /* OP_Found will use an unpacked key */ + assert( !IsVirtual(pTab) ); if( aToOpen[iDataCur-iTabCur] ){ - assert( pPk!=0 ); + assert( pPk!=0 || pTab->pSelect!=0 ); sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, addrBypass, iKey, nKey); VdbeCoverage(v); } @@ -555,7 +557,7 @@ delete_from_cleanup: return; } /* Make sure "isView" and other macros defined above are undefined. Otherwise -** thely may interfere with compilation of other functions in this file +** they may interfere with compilation of other functions in this file ** (or in another file, if this file becomes part of the amalgamation). */ #ifdef isView #undef isView diff --git a/src/expr.c b/src/expr.c index 08c121e386..817975ab3a 100644 --- a/src/expr.c +++ b/src/expr.c @@ -22,7 +22,7 @@ ** affinity of that column is returned. Otherwise, 0x00 is returned, ** indicating no affinity for the expression. ** -** i.e. the WHERE clause expresssions in the following statements all +** i.e. the WHERE clause expressions in the following statements all ** have an affinity: ** ** CREATE TABLE t1(a); @@ -33,7 +33,7 @@ char sqlite3ExprAffinity(Expr *pExpr){ int op; pExpr = sqlite3ExprSkipCollate(pExpr); - if( pExpr->flags & EP_Generic ) return SQLITE_AFF_NONE; + if( pExpr->flags & EP_Generic ) return 0; op = pExpr->op; if( op==TK_SELECT ){ assert( pExpr->flags&EP_xIsSelect ); @@ -501,7 +501,7 @@ void sqlite3ExprAttachSubtrees( } /* -** Allocate a Expr node which joins as many as two subtrees. +** Allocate an Expr node which joins as many as two subtrees. ** ** One or both of the subtrees can be NULL. Return a pointer to the new ** Expr node. Or, if an OOM error occurs, set pParse->db->mallocFailed, @@ -611,7 +611,7 @@ Expr *sqlite3ExprFunction(Parse *pParse, ExprList *pList, Token *pToken){ ** ** Wildcards of the form ":aaa", "@aaa", or "$aaa" are assigned the same number ** as the previous instance of the same wildcard. Or if this is the first -** instance of the wildcard, the next sequenial variable number is +** instance of the wildcard, the next sequential variable number is ** assigned. */ void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr){ @@ -746,7 +746,7 @@ static int exprStructSize(Expr *p){ ** During expression analysis, extra information is computed and moved into ** later parts of teh Expr object and that extra information might get chopped ** off if the expression is reduced. Note also that it does not work to -** make a EXPRDUP_REDUCE copy of a reduced expression. It is only legal +** make an EXPRDUP_REDUCE copy of a reduced expression. It is only legal ** to reduce a pristine expression tree from the parser. The implementation ** of dupedExprStructSize() contain multiple assert() statements that attempt ** to enforce this constraint. @@ -815,7 +815,7 @@ static int dupedExprSize(Expr *p, int flags){ ** is not NULL then *pzBuffer is assumed to point to a buffer large enough ** to store the copy of expression p, the copies of p->u.zToken ** (if applicable), and the copies of the p->pLeft and p->pRight expressions, -** if any. Before returning, *pzBuffer is set to the first byte passed the +** if any. Before returning, *pzBuffer is set to the first byte past the ** portion of the buffer copied into by this function. */ static Expr *exprDup(sqlite3 *db, Expr *p, int flags, u8 **pzBuffer){ @@ -1069,6 +1069,7 @@ Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){ pNew->addrOpenEphm[1] = -1; pNew->nSelectRow = p->nSelectRow; pNew->pWith = withDup(db, p->pWith); + sqlite3SelectSetName(pNew, p->zSelName); return pNew; } #else @@ -1209,37 +1210,51 @@ void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ } /* -** These routines are Walker callbacks. Walker.u.pi is a pointer -** to an integer. These routines are checking an expression to see -** if it is a constant. Set *Walker.u.pi to 0 if the expression is -** not constant. +** These routines are Walker callbacks used to check expressions to +** see if they are "constant" for some definition of constant. The +** Walker.eCode value determines the type of "constant" we are looking +** for. ** ** These callback routines are used to implement the following: ** -** sqlite3ExprIsConstant() -** sqlite3ExprIsConstantNotJoin() -** sqlite3ExprIsConstantOrFunction() +** sqlite3ExprIsConstant() pWalker->eCode==1 +** sqlite3ExprIsConstantNotJoin() pWalker->eCode==2 +** sqlite3ExprRefOneTableOnly() pWalker->eCode==3 +** sqlite3ExprIsConstantOrFunction() pWalker->eCode==4 or 5 ** +** In all cases, the callbacks set Walker.eCode=0 and abort if the expression +** is found to not be a constant. +** +** The sqlite3ExprIsConstantOrFunction() is used for evaluating expressions +** in a CREATE TABLE statement. The Walker.eCode value is 5 when parsing +** an existing schema and 4 when processing a new statement. A bound +** parameter raises an error for new statements, but is silently converted +** to NULL for existing schemas. This allows sqlite_master tables that +** contain a bound parameter because they were generated by older versions +** of SQLite to be parsed by newer versions of SQLite without raising a +** malformed schema error. */ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ - /* If pWalker->u.i is 3 then any term of the expression that comes from - ** the ON or USING clauses of a join disqualifies the expression + /* If pWalker->eCode is 2 then any term of the expression that comes from + ** the ON or USING clauses of a left join disqualifies the expression ** from being considered constant. */ - if( pWalker->u.i==3 && ExprHasProperty(pExpr, EP_FromJoin) ){ - pWalker->u.i = 0; + if( pWalker->eCode==2 && ExprHasProperty(pExpr, EP_FromJoin) ){ + pWalker->eCode = 0; return WRC_Abort; } switch( pExpr->op ){ /* Consider functions to be constant if all their arguments are constant - ** and either pWalker->u.i==2 or the function as the SQLITE_FUNC_CONST - ** flag. */ + ** and either pWalker->eCode==4 or 5 or the function has the + ** SQLITE_FUNC_CONST flag. */ case TK_FUNCTION: - if( pWalker->u.i==2 || ExprHasProperty(pExpr,EP_Constant) ){ + if( pWalker->eCode>=4 || ExprHasProperty(pExpr,EP_Constant) ){ return WRC_Continue; + }else{ + pWalker->eCode = 0; + return WRC_Abort; } - /* Fall through */ case TK_ID: case TK_COLUMN: case TK_AGG_FUNCTION: @@ -1248,8 +1263,25 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ testcase( pExpr->op==TK_COLUMN ); testcase( pExpr->op==TK_AGG_FUNCTION ); testcase( pExpr->op==TK_AGG_COLUMN ); - pWalker->u.i = 0; - return WRC_Abort; + if( pWalker->eCode==3 && pExpr->iTable==pWalker->u.iCur ){ + return WRC_Continue; + }else{ + pWalker->eCode = 0; + return WRC_Abort; + } + case TK_VARIABLE: + if( pWalker->eCode==5 ){ + /* Silently convert bound parameters that appear inside of CREATE + ** statements into a NULL when parsing the CREATE statement text out + ** of the sqlite_master table */ + pExpr->op = TK_NULL; + }else if( pWalker->eCode==4 ){ + /* A bound parameter in a CREATE statement that originates from + ** sqlite3_prepare() causes an error */ + pWalker->eCode = 0; + return WRC_Abort; + } + /* Fall through */ default: testcase( pExpr->op==TK_SELECT ); /* selectNodeIsConstant will disallow */ testcase( pExpr->op==TK_EXISTS ); /* selectNodeIsConstant will disallow */ @@ -1258,21 +1290,22 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ } static int selectNodeIsConstant(Walker *pWalker, Select *NotUsed){ UNUSED_PARAMETER(NotUsed); - pWalker->u.i = 0; + pWalker->eCode = 0; return WRC_Abort; } -static int exprIsConst(Expr *p, int initFlag){ +static int exprIsConst(Expr *p, int initFlag, int iCur){ Walker w; memset(&w, 0, sizeof(w)); - w.u.i = initFlag; + w.eCode = initFlag; w.xExprCallback = exprNodeIsConstant; w.xSelectCallback = selectNodeIsConstant; + w.u.iCur = iCur; sqlite3WalkExpr(&w, p); - return w.u.i; + return w.eCode; } /* -** Walk an expression tree. Return 1 if the expression is constant +** Walk an expression tree. Return non-zero if the expression is constant ** and 0 if it involves variables or function calls. ** ** For the purposes of this function, a double-quoted string (ex: "abc") @@ -1280,21 +1313,31 @@ static int exprIsConst(Expr *p, int initFlag){ ** a constant. */ int sqlite3ExprIsConstant(Expr *p){ - return exprIsConst(p, 1); + return exprIsConst(p, 1, 0); } /* -** Walk an expression tree. Return 1 if the expression is constant +** Walk an expression tree. Return non-zero if the expression is constant ** that does no originate from the ON or USING clauses of a join. ** Return 0 if it involves variables or function calls or terms from ** an ON or USING clause. */ int sqlite3ExprIsConstantNotJoin(Expr *p){ - return exprIsConst(p, 3); + return exprIsConst(p, 2, 0); } /* -** Walk an expression tree. Return 1 if the expression is constant +** Walk an expression tree. Return non-zero if the expression constant +** for any single row of the table with cursor iCur. In other words, the +** expression must not refer to any non-deterministic function nor any +** table other than iCur. +*/ +int sqlite3ExprIsTableConstant(Expr *p, int iCur){ + return exprIsConst(p, 3, iCur); +} + +/* +** Walk an expression tree. Return non-zero if the expression is constant ** or a function call with constant arguments. Return and 0 if there ** are any variables. ** @@ -1302,8 +1345,9 @@ int sqlite3ExprIsConstantNotJoin(Expr *p){ ** is considered a variable but a single-quoted string (ex: 'abc') is ** a constant. */ -int sqlite3ExprIsConstantOrFunction(Expr *p){ - return exprIsConst(p, 2); +int sqlite3ExprIsConstantOrFunction(Expr *p, u8 isInit){ + assert( isInit==0 || isInit==1 ); + return exprIsConst(p, 4+isInit, 0); } /* @@ -1368,6 +1412,10 @@ int sqlite3ExprCanBeNull(const Expr *p){ case TK_FLOAT: case TK_BLOB: return 0; + case TK_COLUMN: + assert( p->pTab!=0 ); + return ExprHasProperty(p, EP_CanBeNull) || + (p->iColumn>=0 && p->pTab->aCol[p->iColumn].notNull==0); default: return 1; } @@ -1475,6 +1523,40 @@ int sqlite3CodeOnce(Parse *pParse){ return sqlite3VdbeAddOp1(v, OP_Once, pParse->nOnce++); } +/* +** Generate code that checks the left-most column of index table iCur to see if +** it contains any NULL entries. Cause the register at regHasNull to be set +** to a non-NULL value if iCur contains no NULLs. Cause register regHasNull +** to be set to NULL if iCur contains one or more NULL values. +*/ +static void sqlite3SetHasNullFlag(Vdbe *v, int iCur, int regHasNull){ + int j1; + sqlite3VdbeAddOp2(v, OP_Integer, 0, regHasNull); + j1 = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_Column, iCur, 0, regHasNull); + sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); + VdbeComment((v, "first_entry_in(%d)", iCur)); + sqlite3VdbeJumpHere(v, j1); +} + + +#ifndef SQLITE_OMIT_SUBQUERY +/* +** The argument is an IN operator with a list (not a subquery) on the +** right-hand side. Return TRUE if that list is constant. +*/ +static int sqlite3InRhsIsConstant(Expr *pIn){ + Expr *pLHS; + int res; + assert( !ExprHasProperty(pIn, EP_xIsSelect) ); + pLHS = pIn->pLeft; + pIn->pLeft = 0; + res = sqlite3ExprIsConstant(pIn); + pIn->pLeft = pLHS; + return res; +} +#endif + /* ** This function is used by the implementation of the IN (...) operator. ** The pX parameter is the expression on the RHS of the IN operator, which @@ -1484,7 +1566,7 @@ int sqlite3CodeOnce(Parse *pParse){ ** be used either to test for membership in the RHS set or to iterate through ** all members of the RHS set, skipping duplicates. ** -** A cursor is opened on the b-tree object that the RHS of the IN operator +** A cursor is opened on the b-tree object that is the RHS of the IN operator ** and pX->iTable is set to the index of that cursor. ** ** The returned value of this function indicates the b-tree type, as follows: @@ -1494,6 +1576,8 @@ int sqlite3CodeOnce(Parse *pParse){ ** IN_INDEX_INDEX_DESC - The cursor was opened on a descending index. ** IN_INDEX_EPH - The cursor was opened on a specially created and ** populated epheremal table. +** IN_INDEX_NOOP - No cursor was allocated. The IN operator must be +** implemented as a sequence of comparisons. ** ** An existing b-tree might be used if the RHS expression pX is a simple ** subquery such as: @@ -1502,52 +1586,57 @@ int sqlite3CodeOnce(Parse *pParse){ ** ** If the RHS of the IN operator is a list or a more complex subquery, then ** an ephemeral table might need to be generated from the RHS and then -** pX->iTable made to point to the ephermeral table instead of an -** existing table. +** pX->iTable made to point to the ephemeral table instead of an +** existing table. ** -** If the prNotFound parameter is 0, then the b-tree will be used to iterate -** through the set members, skipping any duplicates. In this case an -** epheremal table must be used unless the selected is guaranteed +** The inFlags parameter must contain exactly one of the bits +** IN_INDEX_MEMBERSHIP or IN_INDEX_LOOP. If inFlags contains +** IN_INDEX_MEMBERSHIP, then the generated table will be used for a +** fast membership test. When the IN_INDEX_LOOP bit is set, the +** IN index will be used to loop over all values of the RHS of the +** IN operator. +** +** When IN_INDEX_LOOP is used (and the b-tree will be used to iterate +** through the set members) then the b-tree must not contain duplicates. +** An epheremal table must be used unless the selected is guaranteed ** to be unique - either because it is an INTEGER PRIMARY KEY or it ** has a UNIQUE constraint or UNIQUE index. ** -** If the prNotFound parameter is not 0, then the b-tree will be used -** for fast set membership tests. In this case an epheremal table must +** When IN_INDEX_MEMBERSHIP is used (and the b-tree will be used +** for fast set membership tests) then an epheremal table must ** be used unless is an INTEGER PRIMARY KEY or an index can ** be found with as its left-most column. ** +** If the IN_INDEX_NOOP_OK and IN_INDEX_MEMBERSHIP are both set and +** if the RHS of the IN operator is a list (not a subquery) then this +** routine might decide that creating an ephemeral b-tree for membership +** testing is too expensive and return IN_INDEX_NOOP. In that case, the +** calling routine should implement the IN operator using a sequence +** of Eq or Ne comparison operations. +** ** When the b-tree is being used for membership tests, the calling function -** needs to know whether or not the structure contains an SQL NULL -** value in order to correctly evaluate expressions like "X IN (Y, Z)". -** If there is any chance that the (...) might contain a NULL value at +** might need to know whether or not the RHS side of the IN operator +** contains a NULL. If prRhsHasNull is not a NULL pointer and +** if there is any chance that the (...) might contain a NULL value at ** runtime, then a register is allocated and the register number written -** to *prNotFound. If there is no chance that the (...) contains a -** NULL value, then *prNotFound is left unchanged. +** to *prRhsHasNull. If there is no chance that the (...) contains a +** NULL value, then *prRhsHasNull is left unchanged. ** -** If a register is allocated and its location stored in *prNotFound, then -** its initial value is NULL. If the (...) does not remain constant -** for the duration of the query (i.e. the SELECT within the (...) -** is a correlated subquery) then the value of the allocated register is -** reset to NULL each time the subquery is rerun. This allows the -** caller to use vdbe code equivalent to the following: -** -** if( register==NULL ){ -** has_null = -** register = 1 -** } -** -** in order to avoid running the -** test more often than is necessary. +** If a register is allocated and its location stored in *prRhsHasNull, then +** the value in that register will be NULL if the b-tree contains one or more +** NULL values, and it will be some non-NULL value if the b-tree contains no +** NULL values. */ #ifndef SQLITE_OMIT_SUBQUERY -int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ +int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int *prRhsHasNull){ Select *p; /* SELECT to the right of IN operator */ int eType = 0; /* Type of RHS table. IN_INDEX_* */ int iTab = pParse->nTab++; /* Cursor of the RHS table */ - int mustBeUnique = (prNotFound==0); /* True if RHS must be unique */ + int mustBeUnique; /* True if RHS must be unique */ Vdbe *v = sqlite3GetVdbe(pParse); /* Virtual machine being coded */ assert( pX->op==TK_IN ); + mustBeUnique = (inFlags & IN_INDEX_LOOP)!=0; /* Check to see if an existing table or index can be used to ** satisfy the query. This is preferable to generating a new @@ -1604,7 +1693,7 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ for(pIdx=pTab->pIndex; pIdx && eType==0 && affinity_ok; pIdx=pIdx->pNext){ if( (pIdx->aiColumn[0]==iCol) && sqlite3FindCollSeq(db, ENC(db), pIdx->azColl[0], 0)==pReq - && (!mustBeUnique || (pIdx->nKeyCol==1 && pIdx->onError!=OE_None)) + && (!mustBeUnique || (pIdx->nKeyCol==1 && IsUniqueIndex(pIdx))) ){ int iAddr = sqlite3CodeOnce(pParse); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_OpenRead, iTab, pIdx->tnum, iDb); @@ -1613,9 +1702,9 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ assert( IN_INDEX_INDEX_DESC == IN_INDEX_INDEX_ASC+1 ); eType = IN_INDEX_INDEX_ASC + pIdx->aSortOrder[0]; - if( prNotFound && !pTab->aCol[iCol].notNull ){ - *prNotFound = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Null, 0, *prNotFound); + if( prRhsHasNull && !pTab->aCol[iCol].notNull ){ + *prRhsHasNull = ++pParse->nMem; + sqlite3SetHasNullFlag(v, iTab, *prRhsHasNull); } sqlite3VdbeJumpHere(v, iAddr); } @@ -1623,21 +1712,36 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ } } + /* If no preexisting index is available for the IN clause + ** and IN_INDEX_NOOP is an allowed reply + ** and the RHS of the IN operator is a list, not a subquery + ** and the RHS is not contant or has two or fewer terms, + ** then it is not worth creating an ephemeral table to evaluate + ** the IN operator so return IN_INDEX_NOOP. + */ + if( eType==0 + && (inFlags & IN_INDEX_NOOP_OK) + && !ExprHasProperty(pX, EP_xIsSelect) + && (!sqlite3InRhsIsConstant(pX) || pX->x.pList->nExpr<=2) + ){ + eType = IN_INDEX_NOOP; + } + + if( eType==0 ){ - /* Could not found an existing table or index to use as the RHS b-tree. + /* Could not find an existing table or index to use as the RHS b-tree. ** We will have to generate an ephemeral table to do the job. */ u32 savedNQueryLoop = pParse->nQueryLoop; int rMayHaveNull = 0; eType = IN_INDEX_EPH; - if( prNotFound ){ - *prNotFound = rMayHaveNull = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Null, 0, *prNotFound); - }else{ + if( inFlags & IN_INDEX_LOOP ){ pParse->nQueryLoop = 0; if( pX->pLeft->iColumn<0 && !ExprHasProperty(pX, EP_xIsSelect) ){ eType = IN_INDEX_ROWID; } + }else if( prRhsHasNull ){ + *prRhsHasNull = rMayHaveNull = ++pParse->nMem; } sqlite3CodeSubselect(pParse, pX, rMayHaveNull, eType==IN_INDEX_ROWID); pParse->nQueryLoop = savedNQueryLoop; @@ -1668,15 +1772,9 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ ** ** If rMayHaveNull is non-zero, that means that the operation is an IN ** (not a SELECT or EXISTS) and that the RHS might contains NULLs. -** Furthermore, the IN is in a WHERE clause and that we really want -** to iterate over the RHS of the IN operator in order to quickly locate -** all corresponding LHS elements. All this routine does is initialize -** the register given by rMayHaveNull to NULL. Calling routines will take -** care of changing this register value to non-NULL if the RHS is NULL-free. -** -** If rMayHaveNull is zero, that means that the subquery is being used -** for membership testing only. There is no need to initialize any -** registers to indicate the presence or absence of NULLs on the RHS. +** All this routine does is initialize the register given by rMayHaveNull +** to NULL. Calling routines will take care of changing this register +** value to non-NULL if the RHS is NULL-free. ** ** For a SELECT or EXISTS operator, return the register that holds the ** result. For IN operators or if an error occurs, the return value is 0. @@ -1685,10 +1783,10 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ int sqlite3CodeSubselect( Parse *pParse, /* Parsing context */ Expr *pExpr, /* The IN, SELECT, or EXISTS operator */ - int rMayHaveNull, /* Register that records whether NULLs exist in RHS */ + int rHasNullFlag, /* Register that records whether NULLs exist in RHS */ int isRowid /* If true, LHS of IN operator is a rowid */ ){ - int testAddr = -1; /* One-time test address */ + int jmpIfDynamic = -1; /* One-time test address */ int rReg = 0; /* Register storing resulting */ Vdbe *v = sqlite3GetVdbe(pParse); if( NEVER(v==0) ) return 0; @@ -1705,13 +1803,13 @@ int sqlite3CodeSubselect( ** save the results, and reuse the same result on subsequent invocations. */ if( !ExprHasProperty(pExpr, EP_VarSelect) ){ - testAddr = sqlite3CodeOnce(pParse); VdbeCoverage(v); + jmpIfDynamic = sqlite3CodeOnce(pParse); VdbeCoverage(v); } #ifndef SQLITE_OMIT_EXPLAIN if( pParse->explain==2 ){ char *zMsg = sqlite3MPrintf( - pParse->db, "EXECUTE %s%s SUBQUERY %d", testAddr>=0?"":"CORRELATED ", + pParse->db, "EXECUTE %s%s SUBQUERY %d", jmpIfDynamic>=0?"":"CORRELATED ", pExpr->op==TK_IN?"LIST":"SCALAR", pParse->iNextSelectId ); sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg, P4_DYNAMIC); @@ -1725,10 +1823,6 @@ int sqlite3CodeSubselect( Expr *pLeft = pExpr->pLeft; /* the LHS of the IN operator */ KeyInfo *pKeyInfo = 0; /* Key information */ - if( rMayHaveNull ){ - sqlite3VdbeAddOp2(v, OP_Null, 0, rMayHaveNull); - } - affinity = sqlite3ExprAffinity(pLeft); /* Whether this is an 'x IN(SELECT...)' or an 'x IN()' @@ -1754,6 +1848,7 @@ int sqlite3CodeSubselect( ** Generate code to write the results of the select into the temporary ** table allocated and opened above. */ + Select *pSelect = pExpr->x.pSelect; SelectDest dest; ExprList *pEList; @@ -1761,13 +1856,14 @@ int sqlite3CodeSubselect( sqlite3SelectDestInit(&dest, SRT_Set, pExpr->iTable); dest.affSdst = (u8)affinity; assert( (pExpr->iTable&0x0000FFFF)==pExpr->iTable ); - pExpr->x.pSelect->iLimit = 0; + pSelect->iLimit = 0; + testcase( pSelect->selFlags & SF_Distinct ); testcase( pKeyInfo==0 ); /* Caused by OOM in sqlite3KeyInfoAlloc() */ - if( sqlite3Select(pParse, pExpr->x.pSelect, &dest) ){ + if( sqlite3Select(pParse, pSelect, &dest) ){ sqlite3KeyInfoUnref(pKeyInfo); return 0; } - pEList = pExpr->x.pSelect->pEList; + pEList = pSelect->pEList; assert( pKeyInfo!=0 ); /* OOM will cause exit after sqlite3Select() */ assert( pEList!=0 ); assert( pEList->nExpr>0 ); @@ -1798,7 +1894,7 @@ int sqlite3CodeSubselect( /* Loop through each expression in . */ r1 = sqlite3GetTempReg(pParse); r2 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp2(v, OP_Null, 0, r2); + if( isRowid ) sqlite3VdbeAddOp2(v, OP_Null, 0, r2); for(i=pList->nExpr, pItem=pList->a; i>0; i--, pItem++){ Expr *pE2 = pItem->pExpr; int iValToIns; @@ -1808,9 +1904,9 @@ int sqlite3CodeSubselect( ** this code only executes once. Because for a non-constant ** expression we need to rerun this code each time. */ - if( testAddr>=0 && !sqlite3ExprIsConstant(pE2) ){ - sqlite3VdbeChangeToNoop(v, testAddr); - testAddr = -1; + if( jmpIfDynamic>=0 && !sqlite3ExprIsConstant(pE2) ){ + sqlite3VdbeChangeToNoop(v, jmpIfDynamic); + jmpIfDynamic = -1; } /* Evaluate the expression and insert it into the temp table */ @@ -1860,6 +1956,7 @@ int sqlite3CodeSubselect( sqlite3SelectDestInit(&dest, 0, ++pParse->nMem); if( pExpr->op==TK_SELECT ){ dest.eDest = SRT_Mem; + dest.iSdst = dest.iSDParm; sqlite3VdbeAddOp2(v, OP_Null, 0, dest.iSDParm); VdbeComment((v, "Init subquery result")); }else{ @@ -1880,8 +1977,12 @@ int sqlite3CodeSubselect( } } - if( testAddr>=0 ){ - sqlite3VdbeJumpHere(v, testAddr); + if( rHasNullFlag ){ + sqlite3SetHasNullFlag(v, pExpr->iTable, rHasNullFlag); + } + + if( jmpIfDynamic>=0 ){ + sqlite3VdbeJumpHere(v, jmpIfDynamic); } sqlite3ExprCachePop(pParse); @@ -1902,7 +2003,7 @@ int sqlite3CodeSubselect( ** if the LHS is NULL or if the LHS is not contained within the RHS and the ** RHS contains one or more NULL values. ** -** This routine generates code will jump to destIfFalse if the LHS is not +** This routine generates code that jumps to destIfFalse if the LHS is not ** contained within the RHS. If due to NULLs we cannot determine if the LHS ** is contained in the RHS then jump to destIfNull. If the LHS is contained ** within the RHS then fall through. @@ -1925,7 +2026,9 @@ static void sqlite3ExprCodeIN( v = pParse->pVdbe; assert( v!=0 ); /* OOM detected prior to this routine */ VdbeNoopComment((v, "begin IN expr")); - eType = sqlite3FindInIndex(pParse, pExpr, &rRhsHasNull); + eType = sqlite3FindInIndex(pParse, pExpr, + IN_INDEX_MEMBERSHIP | IN_INDEX_NOOP_OK, + destIfFalse==destIfNull ? 0 : &rRhsHasNull); /* Figure out the affinity to use to create a key from the results ** of the expression. affinityStr stores a static string suitable for @@ -1939,82 +2042,114 @@ static void sqlite3ExprCodeIN( r1 = sqlite3GetTempReg(pParse); sqlite3ExprCode(pParse, pExpr->pLeft, r1); - /* If the LHS is NULL, then the result is either false or NULL depending - ** on whether the RHS is empty or not, respectively. + /* If sqlite3FindInIndex() did not find or create an index that is + ** suitable for evaluating the IN operator, then evaluate using a + ** sequence of comparisons. */ - if( destIfNull==destIfFalse ){ - /* Shortcut for the common case where the false and NULL outcomes are - ** the same. */ - sqlite3VdbeAddOp2(v, OP_IsNull, r1, destIfNull); VdbeCoverage(v); + if( eType==IN_INDEX_NOOP ){ + ExprList *pList = pExpr->x.pList; + CollSeq *pColl = sqlite3ExprCollSeq(pParse, pExpr->pLeft); + int labelOk = sqlite3VdbeMakeLabel(v); + int r2, regToFree; + int regCkNull = 0; + int ii; + assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); + if( destIfNull!=destIfFalse ){ + regCkNull = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_BitAnd, r1, r1, regCkNull); + } + for(ii=0; iinExpr; ii++){ + r2 = sqlite3ExprCodeTemp(pParse, pList->a[ii].pExpr, ®ToFree); + if( regCkNull && sqlite3ExprCanBeNull(pList->a[ii].pExpr) ){ + sqlite3VdbeAddOp3(v, OP_BitAnd, regCkNull, r2, regCkNull); + } + if( iinExpr-1 || destIfNull!=destIfFalse ){ + sqlite3VdbeAddOp4(v, OP_Eq, r1, labelOk, r2, + (void*)pColl, P4_COLLSEQ); + VdbeCoverageIf(v, iinExpr-1); + VdbeCoverageIf(v, ii==pList->nExpr-1); + sqlite3VdbeChangeP5(v, affinity); + }else{ + assert( destIfNull==destIfFalse ); + sqlite3VdbeAddOp4(v, OP_Ne, r1, destIfFalse, r2, + (void*)pColl, P4_COLLSEQ); VdbeCoverage(v); + sqlite3VdbeChangeP5(v, affinity | SQLITE_JUMPIFNULL); + } + sqlite3ReleaseTempReg(pParse, regToFree); + } + if( regCkNull ){ + sqlite3VdbeAddOp2(v, OP_IsNull, regCkNull, destIfNull); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfFalse); + } + sqlite3VdbeResolveLabel(v, labelOk); + sqlite3ReleaseTempReg(pParse, regCkNull); }else{ - int addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, r1); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Rewind, pExpr->iTable, destIfFalse); - VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfNull); - sqlite3VdbeJumpHere(v, addr1); - } - - if( eType==IN_INDEX_ROWID ){ - /* In this case, the RHS is the ROWID of table b-tree + + /* If the LHS is NULL, then the result is either false or NULL depending + ** on whether the RHS is empty or not, respectively. */ - sqlite3VdbeAddOp2(v, OP_MustBeInt, r1, destIfFalse); VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_NotExists, pExpr->iTable, destIfFalse, r1); - VdbeCoverage(v); - }else{ - /* In this case, the RHS is an index b-tree. - */ - sqlite3VdbeAddOp4(v, OP_Affinity, r1, 1, 0, &affinity, 1); - - /* If the set membership test fails, then the result of the - ** "x IN (...)" expression must be either 0 or NULL. If the set - ** contains no NULL values, then the result is 0. If the set - ** contains one or more NULL values, then the result of the - ** expression is also NULL. - */ - if( rRhsHasNull==0 || destIfFalse==destIfNull ){ - /* This branch runs if it is known at compile time that the RHS - ** cannot contain NULL values. This happens as the result - ** of a "NOT NULL" constraint in the database schema. - ** - ** Also run this branch if NULL is equivalent to FALSE - ** for this particular IN operator. + if( sqlite3ExprCanBeNull(pExpr->pLeft) ){ + if( destIfNull==destIfFalse ){ + /* Shortcut for the common case where the false and NULL outcomes are + ** the same. */ + sqlite3VdbeAddOp2(v, OP_IsNull, r1, destIfNull); VdbeCoverage(v); + }else{ + int addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, r1); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Rewind, pExpr->iTable, destIfFalse); + VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfNull); + sqlite3VdbeJumpHere(v, addr1); + } + } + + if( eType==IN_INDEX_ROWID ){ + /* In this case, the RHS is the ROWID of table b-tree */ - sqlite3VdbeAddOp4Int(v, OP_NotFound, pExpr->iTable, destIfFalse, r1, 1); + sqlite3VdbeAddOp2(v, OP_MustBeInt, r1, destIfFalse); VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_NotExists, pExpr->iTable, destIfFalse, r1); VdbeCoverage(v); }else{ - /* In this branch, the RHS of the IN might contain a NULL and - ** the presence of a NULL on the RHS makes a difference in the - ** outcome. + /* In this case, the RHS is an index b-tree. */ - int j1, j2; - - /* First check to see if the LHS is contained in the RHS. If so, - ** then the presence of NULLs in the RHS does not matter, so jump - ** over all of the code that follows. + sqlite3VdbeAddOp4(v, OP_Affinity, r1, 1, 0, &affinity, 1); + + /* If the set membership test fails, then the result of the + ** "x IN (...)" expression must be either 0 or NULL. If the set + ** contains no NULL values, then the result is 0. If the set + ** contains one or more NULL values, then the result of the + ** expression is also NULL. */ - j1 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, r1, 1); - VdbeCoverage(v); - - /* Here we begin generating code that runs if the LHS is not - ** contained within the RHS. Generate additional code that - ** tests the RHS for NULLs. If the RHS contains a NULL then - ** jump to destIfNull. If there are no NULLs in the RHS then - ** jump to destIfFalse. - */ - sqlite3VdbeAddOp2(v, OP_If, rRhsHasNull, destIfNull); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_IfNot, rRhsHasNull, destIfFalse); VdbeCoverage(v); - j2 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, rRhsHasNull, 1); - VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Integer, 0, rRhsHasNull); - sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfFalse); - sqlite3VdbeJumpHere(v, j2); - sqlite3VdbeAddOp2(v, OP_Integer, 1, rRhsHasNull); - sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfNull); - - /* The OP_Found at the top of this branch jumps here when true, - ** causing the overall IN expression evaluation to fall through. - */ - sqlite3VdbeJumpHere(v, j1); + assert( destIfFalse!=destIfNull || rRhsHasNull==0 ); + if( rRhsHasNull==0 ){ + /* This branch runs if it is known at compile time that the RHS + ** cannot contain NULL values. This happens as the result + ** of a "NOT NULL" constraint in the database schema. + ** + ** Also run this branch if NULL is equivalent to FALSE + ** for this particular IN operator. + */ + sqlite3VdbeAddOp4Int(v, OP_NotFound, pExpr->iTable, destIfFalse, r1, 1); + VdbeCoverage(v); + }else{ + /* In this branch, the RHS of the IN might contain a NULL and + ** the presence of a NULL on the RHS makes a difference in the + ** outcome. + */ + int j1; + + /* First check to see if the LHS is contained in the RHS. If so, + ** then the answer is TRUE the presence of NULLs in the RHS does + ** not matter. If the LHS is not contained in the RHS, then the + ** answer is NULL if the RHS contains NULLs and the answer is + ** FALSE if the RHS is NULL-free. + */ + j1 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, r1, 1); + VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_IsNull, rRhsHasNull, destIfNull); + VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfFalse); + sqlite3VdbeJumpHere(v, j1); + } } } sqlite3ReleaseTempReg(pParse, r1); @@ -2075,7 +2210,7 @@ static void codeInteger(Parse *pParse, Expr *pExpr, int negFlag, int iMem){ i64 value; const char *z = pExpr->u.zToken; assert( z!=0 ); - c = sqlite3Atoi64(z, &value, sqlite3Strlen30(z), SQLITE_UTF8); + c = sqlite3DecOrHexToI64(z, &value); if( c==0 || (c==2 && negFlag) ){ char *zV; if( negFlag ){ value = c==2 ? SMALLEST_INT64 : -value; } @@ -2085,7 +2220,14 @@ static void codeInteger(Parse *pParse, Expr *pExpr, int negFlag, int iMem){ #ifdef SQLITE_OMIT_FLOATING_POINT sqlite3ErrorMsg(pParse, "oversized integer: %s%s", negFlag ? "-" : "", z); #else - codeReal(v, z, negFlag, iMem); +#ifndef SQLITE_OMIT_HEX_INTEGER + if( sqlite3_strnicmp(z,"0x",2)==0 ){ + sqlite3ErrorMsg(pParse, "hex literal too big: %s", z); + }else +#endif + { + codeReal(v, z, negFlag, iMem); + } #endif } } @@ -2334,16 +2476,9 @@ void sqlite3ExprCacheAffinityChange(Parse *pParse, int iStart, int iCount){ ** over to iTo..iTo+nReg-1. Keep the column cache up-to-date. */ void sqlite3ExprCodeMove(Parse *pParse, int iFrom, int iTo, int nReg){ - int i; - struct yColCache *p; assert( iFrom>=iTo+nReg || iFrom+nReg<=iTo ); sqlite3VdbeAddOp3(pParse->pVdbe, OP_Move, iFrom, iTo, nReg); - for(i=0, p=pParse->aColCache; iiReg; - if( x>=iFrom && xiReg += iTo-iFrom; - } - } + sqlite3ExprCacheRemove(pParse, iFrom, nReg); } #if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) @@ -2498,26 +2633,13 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ #ifndef SQLITE_OMIT_CAST case TK_CAST: { /* Expressions of the form: CAST(pLeft AS token) */ - int aff, to_op; inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); - assert( !ExprHasProperty(pExpr, EP_IntValue) ); - aff = sqlite3AffinityType(pExpr->u.zToken, 0); - to_op = aff - SQLITE_AFF_TEXT + OP_ToText; - assert( to_op==OP_ToText || aff!=SQLITE_AFF_TEXT ); - assert( to_op==OP_ToBlob || aff!=SQLITE_AFF_NONE ); - assert( to_op==OP_ToNumeric || aff!=SQLITE_AFF_NUMERIC ); - assert( to_op==OP_ToInt || aff!=SQLITE_AFF_INTEGER ); - assert( to_op==OP_ToReal || aff!=SQLITE_AFF_REAL ); - testcase( to_op==OP_ToText ); - testcase( to_op==OP_ToBlob ); - testcase( to_op==OP_ToNumeric ); - testcase( to_op==OP_ToInt ); - testcase( to_op==OP_ToReal ); if( inReg!=target ){ sqlite3VdbeAddOp2(v, OP_SCopy, inReg, target); inReg = target; } - sqlite3VdbeAddOp1(v, to_op, inReg); + sqlite3VdbeAddOp2(v, OP_Cast, target, + sqlite3AffinityType(pExpr->u.zToken, 0)); testcase( usedAsColumnCache(pParse, inReg, inReg) ); sqlite3ExprCacheAffinityChange(pParse, inReg, 1); break; @@ -2631,7 +2753,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ addr = sqlite3VdbeAddOp1(v, op, r1); VdbeCoverageIf(v, op==TK_ISNULL); VdbeCoverageIf(v, op==TK_NOTNULL); - sqlite3VdbeAddOp2(v, OP_AddImm, target, -1); + sqlite3VdbeAddOp2(v, OP_Integer, 0, target); sqlite3VdbeJumpHere(v, addr); break; } @@ -2667,13 +2789,13 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ zId = pExpr->u.zToken; nId = sqlite3Strlen30(zId); pDef = sqlite3FindFunction(db, zId, nId, nFarg, enc, 0); - if( pDef==0 ){ + if( pDef==0 || pDef->xFunc==0 ){ sqlite3ErrorMsg(pParse, "unknown function: %.*s()", nId, zId); break; } /* Attempt a direct implementation of the built-in COALESCE() and - ** IFNULL() functions. This avoids unnecessary evalation of + ** IFNULL() functions. This avoids unnecessary evaluation of ** arguments past the first non-NULL argument. */ if( pDef->funcFlags & SQLITE_FUNC_COALESCE ){ @@ -2882,7 +3004,10 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ #ifndef SQLITE_OMIT_FLOATING_POINT /* If the column has REAL affinity, it may currently be stored as an - ** integer. Use OP_RealAffinity to make sure it is really real. */ + ** integer. Use OP_RealAffinity to make sure it is really real. + ** + ** EVIDENCE-OF: R-60985-57662 SQLite will convert the value back to + ** floating point when extracting it from the record. */ if( pExpr->iColumn>=0 && pTab->aCol[pExpr->iColumn].affinity==SQLITE_AFF_REAL ){ @@ -3112,7 +3237,7 @@ void sqlite3ExprCodeFactorable(Parse *pParse, Expr *pExpr, int target){ } /* -** Generate code that evalutes the given expression and puts the result +** Generate code that evaluates the given expression and puts the result ** in register target. ** ** Also make a copy of the expression results into another "cache" register @@ -3135,90 +3260,86 @@ void sqlite3ExprCodeAndCache(Parse *pParse, Expr *pExpr, int target){ exprToRegister(pExpr, iMem); } -#if defined(SQLITE_ENABLE_TREE_EXPLAIN) +#ifdef SQLITE_DEBUG /* ** Generate a human-readable explanation of an expression tree. */ -void sqlite3ExplainExpr(Vdbe *pOut, Expr *pExpr){ - int op; /* The opcode being coded */ +void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ const char *zBinOp = 0; /* Binary operator */ const char *zUniOp = 0; /* Unary operator */ + pView = sqlite3TreeViewPush(pView, moreToFollow); if( pExpr==0 ){ - op = TK_NULL; - }else{ - op = pExpr->op; + sqlite3TreeViewLine(pView, "nil"); + sqlite3TreeViewPop(pView); + return; } - switch( op ){ + switch( pExpr->op ){ case TK_AGG_COLUMN: { - sqlite3ExplainPrintf(pOut, "AGG{%d:%d}", + sqlite3TreeViewLine(pView, "AGG{%d:%d}", pExpr->iTable, pExpr->iColumn); break; } case TK_COLUMN: { if( pExpr->iTable<0 ){ /* This only happens when coding check constraints */ - sqlite3ExplainPrintf(pOut, "COLUMN(%d)", pExpr->iColumn); + sqlite3TreeViewLine(pView, "COLUMN(%d)", pExpr->iColumn); }else{ - sqlite3ExplainPrintf(pOut, "{%d:%d}", + sqlite3TreeViewLine(pView, "{%d:%d}", pExpr->iTable, pExpr->iColumn); } break; } case TK_INTEGER: { if( pExpr->flags & EP_IntValue ){ - sqlite3ExplainPrintf(pOut, "%d", pExpr->u.iValue); + sqlite3TreeViewLine(pView, "%d", pExpr->u.iValue); }else{ - sqlite3ExplainPrintf(pOut, "%s", pExpr->u.zToken); + sqlite3TreeViewLine(pView, "%s", pExpr->u.zToken); } break; } #ifndef SQLITE_OMIT_FLOATING_POINT case TK_FLOAT: { - sqlite3ExplainPrintf(pOut,"%s", pExpr->u.zToken); + sqlite3TreeViewLine(pView,"%s", pExpr->u.zToken); break; } #endif case TK_STRING: { - sqlite3ExplainPrintf(pOut,"%Q", pExpr->u.zToken); + sqlite3TreeViewLine(pView,"%Q", pExpr->u.zToken); break; } case TK_NULL: { - sqlite3ExplainPrintf(pOut,"NULL"); + sqlite3TreeViewLine(pView,"NULL"); break; } #ifndef SQLITE_OMIT_BLOB_LITERAL case TK_BLOB: { - sqlite3ExplainPrintf(pOut,"%s", pExpr->u.zToken); + sqlite3TreeViewLine(pView,"%s", pExpr->u.zToken); break; } #endif case TK_VARIABLE: { - sqlite3ExplainPrintf(pOut,"VARIABLE(%s,%d)", - pExpr->u.zToken, pExpr->iColumn); + sqlite3TreeViewLine(pView,"VARIABLE(%s,%d)", + pExpr->u.zToken, pExpr->iColumn); break; } case TK_REGISTER: { - sqlite3ExplainPrintf(pOut,"REGISTER(%d)", pExpr->iTable); + sqlite3TreeViewLine(pView,"REGISTER(%d)", pExpr->iTable); break; } case TK_AS: { - sqlite3ExplainExpr(pOut, pExpr->pLeft); + sqlite3TreeViewLine(pView,"AS %Q", pExpr->u.zToken); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); + break; + } + case TK_ID: { + sqlite3TreeViewLine(pView,"ID %Q", pExpr->u.zToken); break; } #ifndef SQLITE_OMIT_CAST case TK_CAST: { /* Expressions of the form: CAST(pLeft AS token) */ - const char *zAff = "unk"; - switch( sqlite3AffinityType(pExpr->u.zToken, 0) ){ - case SQLITE_AFF_TEXT: zAff = "TEXT"; break; - case SQLITE_AFF_NONE: zAff = "NONE"; break; - case SQLITE_AFF_NUMERIC: zAff = "NUMERIC"; break; - case SQLITE_AFF_INTEGER: zAff = "INTEGER"; break; - case SQLITE_AFF_REAL: zAff = "REAL"; break; - } - sqlite3ExplainPrintf(pOut, "CAST-%s(", zAff); - sqlite3ExplainExpr(pOut, pExpr->pLeft); - sqlite3ExplainPrintf(pOut, ")"); + sqlite3TreeViewLine(pView,"CAST %Q", pExpr->u.zToken); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); break; } #endif /* SQLITE_OMIT_CAST */ @@ -3242,6 +3363,7 @@ void sqlite3ExplainExpr(Vdbe *pOut, Expr *pExpr){ case TK_LSHIFT: zBinOp = "LSHIFT"; break; case TK_RSHIFT: zBinOp = "RSHIFT"; break; case TK_CONCAT: zBinOp = "CONCAT"; break; + case TK_DOT: zBinOp = "DOT"; break; case TK_UMINUS: zUniOp = "UMINUS"; break; case TK_UPLUS: zUniOp = "UPLUS"; break; @@ -3251,8 +3373,8 @@ void sqlite3ExplainExpr(Vdbe *pOut, Expr *pExpr){ case TK_NOTNULL: zUniOp = "NOTNULL"; break; case TK_COLLATE: { - sqlite3ExplainExpr(pOut, pExpr->pLeft); - sqlite3ExplainPrintf(pOut,".COLLATE(%s)",pExpr->u.zToken); + sqlite3TreeViewLine(pView, "COLLATE %Q", pExpr->u.zToken); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); break; } @@ -3264,41 +3386,36 @@ void sqlite3ExplainExpr(Vdbe *pOut, Expr *pExpr){ }else{ pFarg = pExpr->x.pList; } - if( op==TK_AGG_FUNCTION ){ - sqlite3ExplainPrintf(pOut, "AGG_FUNCTION%d:%s(", + if( pExpr->op==TK_AGG_FUNCTION ){ + sqlite3TreeViewLine(pView, "AGG_FUNCTION%d %Q", pExpr->op2, pExpr->u.zToken); }else{ - sqlite3ExplainPrintf(pOut, "FUNCTION:%s(", pExpr->u.zToken); + sqlite3TreeViewLine(pView, "FUNCTION %Q", pExpr->u.zToken); } if( pFarg ){ - sqlite3ExplainExprList(pOut, pFarg); + sqlite3TreeViewExprList(pView, pFarg, 0, 0); } - sqlite3ExplainPrintf(pOut, ")"); break; } #ifndef SQLITE_OMIT_SUBQUERY case TK_EXISTS: { - sqlite3ExplainPrintf(pOut, "EXISTS("); - sqlite3ExplainSelect(pOut, pExpr->x.pSelect); - sqlite3ExplainPrintf(pOut,")"); + sqlite3TreeViewLine(pView, "EXISTS-expr"); + sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0); break; } case TK_SELECT: { - sqlite3ExplainPrintf(pOut, "("); - sqlite3ExplainSelect(pOut, pExpr->x.pSelect); - sqlite3ExplainPrintf(pOut, ")"); + sqlite3TreeViewLine(pView, "SELECT-expr"); + sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0); break; } case TK_IN: { - sqlite3ExplainPrintf(pOut, "IN("); - sqlite3ExplainExpr(pOut, pExpr->pLeft); - sqlite3ExplainPrintf(pOut, ","); + sqlite3TreeViewLine(pView, "IN"); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 1); if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - sqlite3ExplainSelect(pOut, pExpr->x.pSelect); + sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0); }else{ - sqlite3ExplainExprList(pOut, pExpr->x.pList); + sqlite3TreeViewExprList(pView, pExpr->x.pList, 0, 0); } - sqlite3ExplainPrintf(pOut, ")"); break; } #endif /* SQLITE_OMIT_SUBQUERY */ @@ -3318,13 +3435,10 @@ void sqlite3ExplainExpr(Vdbe *pOut, Expr *pExpr){ Expr *pX = pExpr->pLeft; Expr *pY = pExpr->x.pList->a[0].pExpr; Expr *pZ = pExpr->x.pList->a[1].pExpr; - sqlite3ExplainPrintf(pOut, "BETWEEN("); - sqlite3ExplainExpr(pOut, pX); - sqlite3ExplainPrintf(pOut, ","); - sqlite3ExplainExpr(pOut, pY); - sqlite3ExplainPrintf(pOut, ","); - sqlite3ExplainExpr(pOut, pZ); - sqlite3ExplainPrintf(pOut, ")"); + sqlite3TreeViewLine(pView, "BETWEEN"); + sqlite3TreeViewExpr(pView, pX, 1); + sqlite3TreeViewExpr(pView, pY, 1); + sqlite3TreeViewExpr(pView, pZ, 0); break; } case TK_TRIGGER: { @@ -3335,15 +3449,14 @@ void sqlite3ExplainExpr(Vdbe *pOut, Expr *pExpr){ ** is set to the column of the pseudo-table to read, or to -1 to ** read the rowid field. */ - sqlite3ExplainPrintf(pOut, "%s(%d)", + sqlite3TreeViewLine(pView, "%s(%d)", pExpr->iTable ? "NEW" : "OLD", pExpr->iColumn); break; } case TK_CASE: { - sqlite3ExplainPrintf(pOut, "CASE("); - sqlite3ExplainExpr(pOut, pExpr->pLeft); - sqlite3ExplainPrintf(pOut, ","); - sqlite3ExplainExprList(pOut, pExpr->x.pList); + sqlite3TreeViewLine(pView, "CASE"); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 1); + sqlite3TreeViewExprList(pView, pExpr->x.pList, 0, 0); break; } #ifndef SQLITE_OMIT_TRIGGER @@ -3355,55 +3468,57 @@ void sqlite3ExplainExpr(Vdbe *pOut, Expr *pExpr){ case OE_Fail: zType = "fail"; break; case OE_Ignore: zType = "ignore"; break; } - sqlite3ExplainPrintf(pOut, "RAISE-%s(%s)", zType, pExpr->u.zToken); + sqlite3TreeViewLine(pView, "RAISE %s(%Q)", zType, pExpr->u.zToken); break; } #endif + default: { + sqlite3TreeViewLine(pView, "op=%d", pExpr->op); + break; + } } if( zBinOp ){ - sqlite3ExplainPrintf(pOut,"%s(", zBinOp); - sqlite3ExplainExpr(pOut, pExpr->pLeft); - sqlite3ExplainPrintf(pOut,","); - sqlite3ExplainExpr(pOut, pExpr->pRight); - sqlite3ExplainPrintf(pOut,")"); + sqlite3TreeViewLine(pView, "%s", zBinOp); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 1); + sqlite3TreeViewExpr(pView, pExpr->pRight, 0); }else if( zUniOp ){ - sqlite3ExplainPrintf(pOut,"%s(", zUniOp); - sqlite3ExplainExpr(pOut, pExpr->pLeft); - sqlite3ExplainPrintf(pOut,")"); + sqlite3TreeViewLine(pView, "%s", zUniOp); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); } + sqlite3TreeViewPop(pView); } -#endif /* defined(SQLITE_ENABLE_TREE_EXPLAIN) */ +#endif /* SQLITE_DEBUG */ -#if defined(SQLITE_ENABLE_TREE_EXPLAIN) +#ifdef SQLITE_DEBUG /* ** Generate a human-readable explanation of an expression list. */ -void sqlite3ExplainExprList(Vdbe *pOut, ExprList *pList){ +void sqlite3TreeViewExprList( + TreeView *pView, + const ExprList *pList, + u8 moreToFollow, + const char *zLabel +){ int i; - if( pList==0 || pList->nExpr==0 ){ - sqlite3ExplainPrintf(pOut, "(empty-list)"); - return; - }else if( pList->nExpr==1 ){ - sqlite3ExplainExpr(pOut, pList->a[0].pExpr); + pView = sqlite3TreeViewPush(pView, moreToFollow); + if( zLabel==0 || zLabel[0]==0 ) zLabel = "LIST"; + if( pList==0 ){ + sqlite3TreeViewLine(pView, "%s (empty)", zLabel); }else{ - sqlite3ExplainPush(pOut); + sqlite3TreeViewLine(pView, "%s", zLabel); for(i=0; inExpr; i++){ - sqlite3ExplainPrintf(pOut, "item[%d] = ", i); - sqlite3ExplainPush(pOut); - sqlite3ExplainExpr(pOut, pList->a[i].pExpr); - sqlite3ExplainPop(pOut); - if( pList->a[i].zName ){ + sqlite3TreeViewExpr(pView, pList->a[i].pExpr, inExpr-1); +#if 0 + if( pList->a[i].zName ){ sqlite3ExplainPrintf(pOut, " AS %s", pList->a[i].zName); } if( pList->a[i].bSpanIsTab ){ sqlite3ExplainPrintf(pOut, " (%s)", pList->a[i].zSpan); } - if( inExpr-1 ){ - sqlite3ExplainNL(pOut); - } +#endif } - sqlite3ExplainPop(pOut); } + sqlite3TreeViewPop(pView); } #endif /* SQLITE_DEBUG */ @@ -3467,7 +3582,7 @@ int sqlite3ExprCodeExprList( ** x>=y AND x<=z ** ** Code it as such, taking care to do the common subexpression -** elementation of x. +** elimination of x. */ static void exprCodeBetween( Parse *pParse, /* Parsing and code generating context */ @@ -4204,7 +4319,7 @@ int sqlite3GetTempReg(Parse *pParse){ ** purpose. ** ** If a register is currently being used by the column cache, then -** the dallocation is deferred until the column cache line that uses +** the deallocation is deferred until the column cache line that uses ** the register becomes stale. */ void sqlite3ReleaseTempReg(Parse *pParse, int iReg){ diff --git a/src/fkey.c b/src/fkey.c index c3cac276a9..fa148ba6a3 100644 --- a/src/fkey.c +++ b/src/fkey.c @@ -173,7 +173,7 @@ ** ** 4) No parent key columns were provided explicitly as part of the ** foreign key definition, and the PRIMARY KEY of the parent table -** consists of a a different number of columns to the child key in +** consists of a different number of columns to the child key in ** the child table. ** ** then non-zero is returned, and a "foreign key mismatch" error loaded @@ -225,7 +225,7 @@ int sqlite3FkLocateIndex( } for(pIdx=pParent->pIndex; pIdx; pIdx=pIdx->pNext){ - if( pIdx->nKeyCol==nCol && pIdx->onError!=OE_None ){ + if( pIdx->nKeyCol==nCol && IsUniqueIndex(pIdx) ){ /* pIdx is a UNIQUE index (or a PRIMARY KEY) and has the right number ** of columns. If each indexed column corresponds to a foreign key ** column of pFKey, then this index is a winner. */ @@ -437,7 +437,7 @@ static void fkLookupParent( OE_Abort, 0, P4_STATIC, P5_ConstraintFK); }else{ if( nIncr>0 && pFKey->isDeferred==0 ){ - sqlite3ParseToplevel(pParse)->mayAbort = 1; + sqlite3MayAbort(pParse); } sqlite3VdbeAddOp2(v, OP_FkCounter, pFKey->isDeferred, nIncr); } @@ -509,6 +509,10 @@ static Expr *exprTableColumn( ** code for an SQL UPDATE operation, this function may be called twice - ** once to "delete" the old row and once to "insert" the new row. ** +** Parameter nIncr is passed -1 when inserting a row (as this may decrease +** the number of FK violations in the db) or +1 when deleting one (as this +** may increase the number of FK constraint problems). +** ** The code generated by this function scans through the rows in the child ** table that correspond to the parent table row being deleted or inserted. ** For each child row found, one of the following actions is taken: @@ -625,13 +629,9 @@ static void fkScanChildren( sqlite3ResolveExprNames(&sNameContext, pWhere); /* Create VDBE to loop through the entries in pSrc that match the WHERE - ** clause. If the constraint is not deferred, throw an exception for - ** each row found. Otherwise, for deferred constraints, increment the - ** deferred constraint counter by nIncr for each row selected. */ + ** clause. For each row found, increment either the deferred or immediate + ** foreign key constraint counter. */ pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0, 0, 0, 0); - if( nIncr>0 && pFKey->isDeferred==0 ){ - sqlite3ParseToplevel(pParse)->mayAbort = 1; - } sqlite3VdbeAddOp2(v, OP_FkCounter, pFKey->isDeferred, nIncr); if( pWInfo ){ sqlite3WhereEnd(pWInfo); @@ -659,8 +659,7 @@ static void fkScanChildren( ** table). */ FKey *sqlite3FkReferences(Table *pTab){ - int nName = sqlite3Strlen30(pTab->zName); - return (FKey *)sqlite3HashFind(&pTab->pSchema->fkeyHash, pTab->zName, nName); + return (FKey *)sqlite3HashFind(&pTab->pSchema->fkeyHash, pTab->zName); } /* @@ -811,6 +810,24 @@ static int fkParentIsModified( return 0; } +/* +** Return true if the parser passed as the first argument is being +** used to code a trigger that is really a "SET NULL" action belonging +** to trigger pFKey. +*/ +static int isSetNullAction(Parse *pParse, FKey *pFKey){ + Parse *pTop = sqlite3ParseToplevel(pParse); + if( pTop->pTriggerPrg ){ + Trigger *p = pTop->pTriggerPrg->pTrigger; + if( (p==pFKey->apTrigger[0] && pFKey->aAction[0]==OE_SetNull) + || (p==pFKey->apTrigger[1] && pFKey->aAction[1]==OE_SetNull) + ){ + return 1; + } + } + return 0; +} + /* ** This function is called when inserting, deleting or updating a row of ** table pTab to generate VDBE code to perform foreign key constraint @@ -863,7 +880,7 @@ void sqlite3FkCheck( int *aiCol; int iCol; int i; - int isIgnore = 0; + int bIgnore = 0; if( aChange && sqlite3_stricmp(pTab->zName, pFKey->zTo)!=0 @@ -922,7 +939,7 @@ void sqlite3FkCheck( int rcauth; char *zCol = pTo->aCol[pIdx ? pIdx->aiColumn[i] : pTo->iPKey].zName; rcauth = sqlite3AuthReadCol(pParse, pTo->zName, zCol, iDb); - isIgnore = (rcauth==SQLITE_IGNORE); + bIgnore = (rcauth==SQLITE_IGNORE); } #endif } @@ -937,12 +954,18 @@ void sqlite3FkCheck( /* A row is being removed from the child table. Search for the parent. ** If the parent does not exist, removing the child row resolves an ** outstanding foreign key constraint violation. */ - fkLookupParent(pParse, iDb, pTo, pIdx, pFKey, aiCol, regOld, -1,isIgnore); + fkLookupParent(pParse, iDb, pTo, pIdx, pFKey, aiCol, regOld, -1, bIgnore); } - if( regNew!=0 ){ + if( regNew!=0 && !isSetNullAction(pParse, pFKey) ){ /* A row is being added to the child table. If a parent row cannot - ** be found, adding the child row has violated the FK constraint. */ - fkLookupParent(pParse, iDb, pTo, pIdx, pFKey, aiCol, regNew, +1,isIgnore); + ** be found, adding the child row has violated the FK constraint. + ** + ** If this operation is being performed as part of a trigger program + ** that is actually a "SET NULL" action belonging to this very + ** foreign key, then omit this scan altogether. As all child key + ** values are guaranteed to be NULL, it is not possible for adding + ** this row to cause an FK violation. */ + fkLookupParent(pParse, iDb, pTo, pIdx, pFKey, aiCol, regNew, +1, bIgnore); } sqlite3DbFree(db, aiFree); @@ -963,8 +986,8 @@ void sqlite3FkCheck( && !pParse->pToplevel && !pParse->isMultiWrite ){ assert( regOld==0 && regNew!=0 ); - /* Inserting a single row into a parent table cannot cause an immediate - ** foreign key violation. So do nothing in this case. */ + /* Inserting a single row into a parent table cannot cause (or fix) + ** an immediate foreign key violation. So do nothing in this case. */ continue; } @@ -988,13 +1011,28 @@ void sqlite3FkCheck( fkScanChildren(pParse, pSrc, pTab, pIdx, pFKey, aiCol, regNew, -1); } if( regOld!=0 ){ - /* If there is a RESTRICT action configured for the current operation - ** on the parent table of this FK, then throw an exception - ** immediately if the FK constraint is violated, even if this is a - ** deferred trigger. That's what RESTRICT means. To defer checking - ** the constraint, the FK should specify NO ACTION (represented - ** using OE_None). NO ACTION is the default. */ + int eAction = pFKey->aAction[aChange!=0]; fkScanChildren(pParse, pSrc, pTab, pIdx, pFKey, aiCol, regOld, 1); + /* If this is a deferred FK constraint, or a CASCADE or SET NULL + ** action applies, then any foreign key violations caused by + ** removing the parent key will be rectified by the action trigger. + ** So do not set the "may-abort" flag in this case. + ** + ** Note 1: If the FK is declared "ON UPDATE CASCADE", then the + ** may-abort flag will eventually be set on this statement anyway + ** (when this function is called as part of processing the UPDATE + ** within the action trigger). + ** + ** Note 2: At first glance it may seem like SQLite could simply omit + ** all OP_FkCounter related scans when either CASCADE or SET NULL + ** applies. The trouble starts if the CASCADE or SET NULL action + ** trigger causes other triggers or action rules attached to the + ** child table to fire. In these cases the fk constraint counters + ** might be set incorrectly if any OP_FkCounter related scans are + ** omitted. */ + if( !pFKey->isDeferred && eAction!=OE_Cascade && eAction!=OE_SetNull ){ + sqlite3MayAbort(pParse); + } } pItem->zName = 0; sqlite3SrcListDelete(db, pSrc); @@ -1338,7 +1376,7 @@ void sqlite3FkDelete(sqlite3 *db, Table *pTab){ }else{ void *p = (void *)pFKey->pNextTo; const char *z = (p ? pFKey->pNextTo->zTo : pFKey->zTo); - sqlite3HashInsert(&pTab->pSchema->fkeyHash, z, sqlite3Strlen30(z), p); + sqlite3HashInsert(&pTab->pSchema->fkeyHash, z, p); } if( pFKey->pNextTo ){ pFKey->pNextTo->pPrevTo = pFKey->pPrevTo; diff --git a/src/func.c b/src/func.c index efbb3efbb7..a057993413 100644 --- a/src/func.c +++ b/src/func.c @@ -9,12 +9,9 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** This file contains the C functions that implement various SQL -** functions of SQLite. -** -** There is only one exported symbol in this file - the function -** sqliteRegisterBuildinFunctions() found at the bottom of the file. -** All other code has file scope. +** This file contains the C-language implementations for many of the SQL +** functions of SQLite. (Some function, and in particular the date and +** time functions, are implemented separately.) */ #include "sqliteInt.h" #include @@ -25,7 +22,10 @@ ** Return the collating function associated with a function. */ static CollSeq *sqlite3GetFuncCollSeq(sqlite3_context *context){ - return context->pColl; + VdbeOp *pOp = &context->pVdbe->aOp[context->iOp-1]; + assert( pOp->opcode==OP_CollSeq ); + assert( pOp->p4type==P4_COLLSEQ ); + return pOp->p4.pColl; } /* @@ -157,8 +157,8 @@ static void absFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ default: { /* Because sqlite3_value_double() returns 0.0 if the argument is not ** something that can be converted into a number, we have: - ** IMP: R-57326-31541 Abs(X) return 0.0 if X is a string or blob that - ** cannot be converted to a numeric value. + ** IMP: R-01992-00519 Abs(X) returns 0.0 if X is a string or blob + ** that cannot be converted to a numeric value. */ double rVal = sqlite3_value_double(argv[0]); if( rVal<0 ) rVal = -rVal; @@ -328,13 +328,14 @@ static void substrFunc( for(z2=z; *z2 && p2; p2--){ SQLITE_SKIP_UTF8(z2); } - sqlite3_result_text(context, (char*)z, (int)(z2-z), SQLITE_TRANSIENT); + sqlite3_result_text64(context, (char*)z, z2-z, SQLITE_TRANSIENT, + SQLITE_UTF8); }else{ if( p1+p2>len ){ p2 = len-p1; if( p2<0 ) p2 = 0; } - sqlite3_result_blob(context, (char*)&z[p1], (int)p2, SQLITE_TRANSIENT); + sqlite3_result_blob64(context, (char*)&z[p1], (u64)p2, SQLITE_TRANSIENT); } } @@ -393,7 +394,7 @@ static void *contextMalloc(sqlite3_context *context, i64 nByte){ sqlite3_result_error_toobig(context); z = 0; }else{ - z = sqlite3Malloc((int)nByte); + z = sqlite3Malloc(nByte); if( !z ){ sqlite3_result_error_nomem(context); } @@ -569,10 +570,12 @@ struct compareInfo { ** whereas only characters less than 0x80 do in ASCII. */ #if defined(SQLITE_EBCDIC) -# define sqlite3Utf8Read(A) (*((*A)++)) -# define GlobUpperToLower(A) A = sqlite3UpperToLower[A] +# define sqlite3Utf8Read(A) (*((*A)++)) +# define GlobUpperToLower(A) A = sqlite3UpperToLower[A] +# define GlobUpperToLowerAscii(A) A = sqlite3UpperToLower[A] #else -# define GlobUpperToLower(A) if( !((A)&~0x7f) ){ A = sqlite3UpperToLower[A]; } +# define GlobUpperToLower(A) if( A<=0x7f ){ A = sqlite3UpperToLower[A]; } +# define GlobUpperToLowerAscii(A) A = sqlite3UpperToLower[A] #endif static const struct compareInfo globInfo = { '*', '?', '[', 0 }; @@ -585,7 +588,7 @@ static const struct compareInfo likeInfoAlt = { '%', '_', 0, 0 }; /* ** Compare two UTF-8 strings for equality where the first string can -** potentially be a "glob" expression. Return true (1) if they +** potentially be a "glob" or "like" expression. Return true (1) if they ** are the same and false (0) if they are different. ** ** Globbing rules: @@ -605,11 +608,18 @@ static const struct compareInfo likeInfoAlt = { '%', '_', 0, 0 }; ** "[a-z]" matches any single lower-case letter. To match a '-', make ** it the last character in the list. ** +** Like matching rules: +** +** '%' Matches any sequence of zero or more characters +** +*** '_' Matches any one character +** +** Ec Where E is the "esc" character and c is any other +** character, including '%', '_', and esc, match exactly c. +** +** The comments through this routine usually assume glob matching. +** ** This routine is usually quick, but can be N**2 in the worst case. -** -** Hints: to match '*' or '?', put them in "[]". Like this: -** -** abc[*]xyz Matches "abc*xyz" only */ static int patternCompare( const u8 *zPattern, /* The glob pattern */ @@ -617,17 +627,25 @@ static int patternCompare( const struct compareInfo *pInfo, /* Information about how to do the compare */ u32 esc /* The escape character */ ){ - u32 c, c2; - int invert; - int seen; - u8 matchOne = pInfo->matchOne; - u8 matchAll = pInfo->matchAll; - u8 matchSet = pInfo->matchSet; - u8 noCase = pInfo->noCase; - int prevEscape = 0; /* True if the previous character was 'escape' */ + u32 c, c2; /* Next pattern and input string chars */ + u32 matchOne = pInfo->matchOne; /* "?" or "_" */ + u32 matchAll = pInfo->matchAll; /* "*" or "%" */ + u32 matchOther; /* "[" or the escape character */ + u8 noCase = pInfo->noCase; /* True if uppercase==lowercase */ + const u8 *zEscaped = 0; /* One past the last escaped input char */ + + /* The GLOB operator does not have an ESCAPE clause. And LIKE does not + ** have the matchSet operator. So we either have to look for one or + ** the other, never both. Hence the single variable matchOther is used + ** to store the one we have to look for. + */ + matchOther = esc ? esc : pInfo->matchSet; while( (c = sqlite3Utf8Read(&zPattern))!=0 ){ - if( c==matchAll && !prevEscape ){ + if( c==matchAll ){ /* Match "*" */ + /* Skip over multiple "*" characters in the pattern. If there + ** are also "?" characters, skip those as well, but consume a + ** single character of the input string for each "?" skipped */ while( (c=sqlite3Utf8Read(&zPattern)) == matchAll || c == matchOne ){ if( c==matchOne && sqlite3Utf8Read(&zString)==0 ){ @@ -635,86 +653,98 @@ static int patternCompare( } } if( c==0 ){ - return 1; - }else if( c==esc ){ - c = sqlite3Utf8Read(&zPattern); - if( c==0 ){ - return 0; - } - }else if( c==matchSet ){ - assert( esc==0 ); /* This is GLOB, not LIKE */ - assert( matchSet<0x80 ); /* '[' is a single-byte character */ - while( *zString && patternCompare(&zPattern[-1],zString,pInfo,esc)==0 ){ - SQLITE_SKIP_UTF8(zString); - } - return *zString!=0; - } - while( (c2 = sqlite3Utf8Read(&zString))!=0 ){ - if( noCase ){ - GlobUpperToLower(c2); - GlobUpperToLower(c); - while( c2 != 0 && c2 != c ){ - c2 = sqlite3Utf8Read(&zString); - GlobUpperToLower(c2); - } + return 1; /* "*" at the end of the pattern matches */ + }else if( c==matchOther ){ + if( esc ){ + c = sqlite3Utf8Read(&zPattern); + if( c==0 ) return 0; }else{ - while( c2 != 0 && c2 != c ){ - c2 = sqlite3Utf8Read(&zString); + /* "[...]" immediately follows the "*". We have to do a slow + ** recursive search in this case, but it is an unusual case. */ + assert( matchOther<0x80 ); /* '[' is a single-byte character */ + while( *zString + && patternCompare(&zPattern[-1],zString,pInfo,esc)==0 ){ + SQLITE_SKIP_UTF8(zString); } + return *zString!=0; + } + } + + /* At this point variable c contains the first character of the + ** pattern string past the "*". Search in the input string for the + ** first matching character and recursively contine the match from + ** that point. + ** + ** For a case-insensitive search, set variable cx to be the same as + ** c but in the other case and search the input string for either + ** c or cx. + */ + if( c<=0x80 ){ + u32 cx; + if( noCase ){ + cx = sqlite3Toupper(c); + c = sqlite3Tolower(c); + }else{ + cx = c; + } + while( (c2 = *(zString++))!=0 ){ + if( c2!=c && c2!=cx ) continue; + if( patternCompare(zPattern,zString,pInfo,esc) ) return 1; + } + }else{ + while( (c2 = sqlite3Utf8Read(&zString))!=0 ){ + if( c2!=c ) continue; + if( patternCompare(zPattern,zString,pInfo,esc) ) return 1; } - if( c2==0 ) return 0; - if( patternCompare(zPattern,zString,pInfo,esc) ) return 1; } return 0; - }else if( c==matchOne && !prevEscape ){ - if( sqlite3Utf8Read(&zString)==0 ){ - return 0; - } - }else if( c==matchSet ){ - u32 prior_c = 0; - assert( esc==0 ); /* This only occurs for GLOB, not LIKE */ - seen = 0; - invert = 0; - c = sqlite3Utf8Read(&zString); - if( c==0 ) return 0; - c2 = sqlite3Utf8Read(&zPattern); - if( c2=='^' ){ - invert = 1; - c2 = sqlite3Utf8Read(&zPattern); - } - if( c2==']' ){ - if( c==']' ) seen = 1; - c2 = sqlite3Utf8Read(&zPattern); - } - while( c2 && c2!=']' ){ - if( c2=='-' && zPattern[0]!=']' && zPattern[0]!=0 && prior_c>0 ){ - c2 = sqlite3Utf8Read(&zPattern); - if( c>=prior_c && c<=c2 ) seen = 1; - prior_c = 0; - }else{ - if( c==c2 ){ - seen = 1; - } - prior_c = c2; - } - c2 = sqlite3Utf8Read(&zPattern); - } - if( c2==0 || (seen ^ invert)==0 ){ - return 0; - } - }else if( esc==c && !prevEscape ){ - prevEscape = 1; - }else{ - c2 = sqlite3Utf8Read(&zString); - if( noCase ){ - GlobUpperToLower(c); - GlobUpperToLower(c2); - } - if( c!=c2 ){ - return 0; - } - prevEscape = 0; } + if( c==matchOther ){ + if( esc ){ + c = sqlite3Utf8Read(&zPattern); + if( c==0 ) return 0; + zEscaped = zPattern; + }else{ + u32 prior_c = 0; + int seen = 0; + int invert = 0; + c = sqlite3Utf8Read(&zString); + if( c==0 ) return 0; + c2 = sqlite3Utf8Read(&zPattern); + if( c2=='^' ){ + invert = 1; + c2 = sqlite3Utf8Read(&zPattern); + } + if( c2==']' ){ + if( c==']' ) seen = 1; + c2 = sqlite3Utf8Read(&zPattern); + } + while( c2 && c2!=']' ){ + if( c2=='-' && zPattern[0]!=']' && zPattern[0]!=0 && prior_c>0 ){ + c2 = sqlite3Utf8Read(&zPattern); + if( c>=prior_c && c<=c2 ) seen = 1; + prior_c = 0; + }else{ + if( c==c2 ){ + seen = 1; + } + prior_c = c2; + } + c2 = sqlite3Utf8Read(&zPattern); + } + if( c2==0 || (seen ^ invert)==0 ){ + return 0; + } + continue; + } + } + c2 = sqlite3Utf8Read(&zString); + if( c==c2 ) continue; + if( noCase && c<0x80 && c2<0x80 && sqlite3Tolower(c)==sqlite3Tolower(c2) ){ + continue; + } + if( c==matchOne && zPattern!=zEscaped && c2!=0 ) continue; + return 0; } return *zString==0; } @@ -1044,7 +1074,7 @@ static void charFunc( *zOut++ = 0x80 + (u8)(c & 0x3F); } \ } - sqlite3_result_text(context, (char*)z, (int)(zOut-z), sqlite3_free); + sqlite3_result_text64(context, (char*)z, zOut-z, sqlite3_free, SQLITE_UTF8); } /* @@ -1494,6 +1524,7 @@ static void minmaxStep( sqlite3SkipAccumulatorLoad(context); } }else{ + pBest->db = sqlite3_context_db_handle(context); sqlite3VdbeMemCopy(pBest, pArg); } } @@ -1641,7 +1672,7 @@ int sqlite3IsLikeFunction(sqlite3 *db, Expr *pExpr, int *pIsNocase, char *aWc){ } /* -** All all of the FuncDef structures in the aBuiltinFunc[] array above +** All of the FuncDef structures in the aBuiltinFunc[] array above ** to the global function hash table. This occurs at start-time (as ** a consequence of calling sqlite3_initialize()). ** @@ -1665,10 +1696,12 @@ void sqlite3RegisterGlobalFunctions(void){ FUNCTION(trim, 2, 3, 0, trimFunc ), FUNCTION(min, -1, 0, 1, minmaxFunc ), FUNCTION(min, 0, 0, 1, 0 ), - AGGREGATE(min, 1, 0, 1, minmaxStep, minMaxFinalize ), + AGGREGATE2(min, 1, 0, 1, minmaxStep, minMaxFinalize, + SQLITE_FUNC_MINMAX ), FUNCTION(max, -1, 1, 1, minmaxFunc ), FUNCTION(max, 0, 1, 1, 0 ), - AGGREGATE(max, 1, 1, 1, minmaxStep, minMaxFinalize ), + AGGREGATE2(max, 1, 1, 1, minmaxStep, minMaxFinalize, + SQLITE_FUNC_MINMAX ), FUNCTION2(typeof, 1, 0, 0, typeofFunc, SQLITE_FUNC_TYPEOF), FUNCTION2(length, 1, 0, 0, lengthFunc, SQLITE_FUNC_LENGTH), FUNCTION(instr, 2, 0, 0, instrFunc ), @@ -1698,6 +1731,9 @@ void sqlite3RegisterGlobalFunctions(void){ FUNCTION(sqlite_version, 0, 0, 0, versionFunc ), FUNCTION(sqlite_source_id, 0, 0, 0, sourceidFunc ), FUNCTION(sqlite_log, 2, 0, 0, errlogFunc ), +#if SQLITE_USER_AUTHENTICATION + FUNCTION(sqlite_crypt, 2, 0, 0, sqlite3CryptFunc ), +#endif #ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS FUNCTION(sqlite_compileoption_used,1, 0, 0, compileoptionusedFunc ), FUNCTION(sqlite_compileoption_get, 1, 0, 0, compileoptiongetFunc ), @@ -1718,8 +1754,8 @@ void sqlite3RegisterGlobalFunctions(void){ AGGREGATE(sum, 1, 0, 0, sumStep, sumFinalize ), AGGREGATE(total, 1, 0, 0, sumStep, totalFinalize ), AGGREGATE(avg, 1, 0, 0, sumStep, avgFinalize ), - /* AGGREGATE(count, 0, 0, 0, countStep, countFinalize ), */ - {0,SQLITE_UTF8|SQLITE_FUNC_COUNT,0,0,0,countStep,countFinalize,"count",0,0}, + AGGREGATE2(count, 0, 0, 0, countStep, countFinalize, + SQLITE_FUNC_COUNT ), AGGREGATE(count, 1, 0, 0, countStep, countFinalize ), AGGREGATE(group_concat, 1, 0, 0, groupConcatStep, groupConcatFinalize), AGGREGATE(group_concat, 2, 0, 0, groupConcatStep, groupConcatFinalize), diff --git a/src/global.c b/src/global.c index 2c14b58abd..4bc8edb3bc 100644 --- a/src/global.c +++ b/src/global.c @@ -10,7 +10,7 @@ ** ************************************************************************* ** -** This file contains definitions of global variables and contants. +** This file contains definitions of global variables and constants. */ #include "sqliteInt.h" @@ -129,10 +129,25 @@ const unsigned char sqlite3CtypeMap[256] = { }; #endif +/* EVIDENCE-OF: R-02982-34736 In order to maintain full backwards +** compatibility for legacy applications, the URI filename capability is +** disabled by default. +** +** EVIDENCE-OF: R-38799-08373 URI filenames can be enabled or disabled +** using the SQLITE_USE_URI=1 or SQLITE_USE_URI=0 compile-time options. +** +** EVIDENCE-OF: R-43642-56306 By default, URI handling is globally +** disabled. The default value may be changed by compiling with the +** SQLITE_USE_URI symbol defined. +*/ #ifndef SQLITE_USE_URI # define SQLITE_USE_URI 0 #endif +/* EVIDENCE-OF: R-38720-18127 The default setting is determined by the +** SQLITE_ALLOW_COVERING_INDEX_SCAN compile-time option, or is "on" if +** that compile-time option is omitted. +*/ #ifndef SQLITE_ALLOW_COVERING_INDEX_SCAN # define SQLITE_ALLOW_COVERING_INDEX_SCAN 1 #endif @@ -222,8 +237,8 @@ const Token sqlite3IntTokens[] = { ** ** IMPORTANT: Changing the pending byte to any value other than ** 0x40000000 results in an incompatible database file format! -** Changing the pending byte during operating results in undefined -** and dileterious behavior. +** Changing the pending byte during operation will result in undefined +** and incorrect behavior. */ #ifndef SQLITE_OMIT_WSD int sqlite3PendingByte = 0x40000000; diff --git a/src/hash.c b/src/hash.c index f9901fee8f..b5886e0641 100644 --- a/src/hash.c +++ b/src/hash.c @@ -52,12 +52,11 @@ void sqlite3HashClear(Hash *pH){ /* ** The hashing function. */ -static unsigned int strHash(const char *z, int nKey){ +static unsigned int strHash(const char *z){ unsigned int h = 0; - assert( nKey>=0 ); - while( nKey > 0 ){ - h = (h<<3) ^ h ^ sqlite3UpperToLower[(unsigned char)*z++]; - nKey--; + unsigned char c; + while( (c = (unsigned char)*z++)!=0 ){ + h = (h<<3) ^ h ^ sqlite3UpperToLower[c]; } return h; } @@ -129,7 +128,7 @@ static int rehash(Hash *pH, unsigned int new_size){ pH->htsize = new_size = sqlite3MallocSize(new_ht)/sizeof(struct _ht); memset(new_ht, 0, new_size*sizeof(struct _ht)); for(elem=pH->first, pH->first=0; elem; elem = next_elem){ - unsigned int h = strHash(elem->pKey, elem->nKey) % new_size; + unsigned int h = strHash(elem->pKey) % new_size; next_elem = elem->next; insertElement(pH, &new_ht[h], elem); } @@ -137,28 +136,33 @@ static int rehash(Hash *pH, unsigned int new_size){ } /* This function (for internal use only) locates an element in an -** hash table that matches the given key. The hash for this key has -** already been computed and is passed as the 4th parameter. +** hash table that matches the given key. The hash for this key is +** also computed and returned in the *pH parameter. */ -static HashElem *findElementGivenHash( +static HashElem *findElementWithHash( const Hash *pH, /* The pH to be searched */ const char *pKey, /* The key we are searching for */ - int nKey, /* Bytes in key (not counting zero terminator) */ - unsigned int h /* The hash for this key. */ + unsigned int *pHash /* Write the hash value here */ ){ HashElem *elem; /* Used to loop thru the element list */ int count; /* Number of elements left to test */ + unsigned int h; /* The computed hash */ if( pH->ht ){ - struct _ht *pEntry = &pH->ht[h]; + struct _ht *pEntry; + h = strHash(pKey) % pH->htsize; + pEntry = &pH->ht[h]; elem = pEntry->chain; count = pEntry->count; }else{ + h = 0; elem = pH->first; count = pH->count; } - while( count-- && ALWAYS(elem) ){ - if( elem->nKey==nKey && sqlite3StrNICmp(elem->pKey,pKey,nKey)==0 ){ + *pHash = h; + while( count-- ){ + assert( elem!=0 ); + if( sqlite3StrICmp(elem->pKey,pKey)==0 ){ return elem; } elem = elem->next; @@ -201,26 +205,20 @@ static void removeElementGivenHash( } /* Attempt to locate an element of the hash table pH with a key -** that matches pKey,nKey. Return the data for this element if it is +** that matches pKey. Return the data for this element if it is ** found, or NULL if there is no match. */ -void *sqlite3HashFind(const Hash *pH, const char *pKey, int nKey){ +void *sqlite3HashFind(const Hash *pH, const char *pKey){ HashElem *elem; /* The element that matches key */ unsigned int h; /* A hash on key */ assert( pH!=0 ); assert( pKey!=0 ); - assert( nKey>=0 ); - if( pH->ht ){ - h = strHash(pKey, nKey) % pH->htsize; - }else{ - h = 0; - } - elem = findElementGivenHash(pH, pKey, nKey, h); + elem = findElementWithHash(pH, pKey, &h); return elem ? elem->data : 0; } -/* Insert an element into the hash table pH. The key is pKey,nKey +/* Insert an element into the hash table pH. The key is pKey ** and the data is "data". ** ** If no element exists with a matching key, then a new @@ -234,20 +232,14 @@ void *sqlite3HashFind(const Hash *pH, const char *pKey, int nKey){ ** If the "data" parameter to this function is NULL, then the ** element corresponding to "key" is removed from the hash table. */ -void *sqlite3HashInsert(Hash *pH, const char *pKey, int nKey, void *data){ +void *sqlite3HashInsert(Hash *pH, const char *pKey, void *data){ unsigned int h; /* the hash of the key modulo hash table size */ HashElem *elem; /* Used to loop thru the element list */ HashElem *new_elem; /* New element added to the pH */ assert( pH!=0 ); assert( pKey!=0 ); - assert( nKey>=0 ); - if( pH->htsize ){ - h = strHash(pKey, nKey) % pH->htsize; - }else{ - h = 0; - } - elem = findElementGivenHash(pH,pKey,nKey,h); + elem = findElementWithHash(pH,pKey,&h); if( elem ){ void *old_data = elem->data; if( data==0 ){ @@ -255,7 +247,6 @@ void *sqlite3HashInsert(Hash *pH, const char *pKey, int nKey, void *data){ }else{ elem->data = data; elem->pKey = pKey; - assert(nKey==elem->nKey); } return old_data; } @@ -263,19 +254,14 @@ void *sqlite3HashInsert(Hash *pH, const char *pKey, int nKey, void *data){ new_elem = (HashElem*)sqlite3Malloc( sizeof(HashElem) ); if( new_elem==0 ) return data; new_elem->pKey = pKey; - new_elem->nKey = nKey; new_elem->data = data; pH->count++; if( pH->count>=10 && pH->count > 2*pH->htsize ){ if( rehash(pH, pH->count*2) ){ assert( pH->htsize>0 ); - h = strHash(pKey, nKey) % pH->htsize; + h = strHash(pKey) % pH->htsize; } } - if( pH->ht ){ - insertElement(pH, &pH->ht[h], new_elem); - }else{ - insertElement(pH, 0, new_elem); - } + insertElement(pH, pH->ht ? &pH->ht[h] : 0, new_elem); return 0; } diff --git a/src/hash.h b/src/hash.h index 82b7c58c71..6dfa4e035c 100644 --- a/src/hash.h +++ b/src/hash.h @@ -59,15 +59,15 @@ struct Hash { struct HashElem { HashElem *next, *prev; /* Next and previous elements in the table */ void *data; /* Data associated with this element */ - const char *pKey; int nKey; /* Key associated with this element */ + const char *pKey; /* Key associated with this element */ }; /* ** Access routines. To delete, insert a NULL pointer. */ void sqlite3HashInit(Hash*); -void *sqlite3HashInsert(Hash*, const char *pKey, int nKey, void *pData); -void *sqlite3HashFind(const Hash*, const char *pKey, int nKey); +void *sqlite3HashInsert(Hash*, const char *pKey, void *pData); +void *sqlite3HashFind(const Hash*, const char *pKey); void sqlite3HashClear(Hash*); /* diff --git a/src/insert.c b/src/insert.c index 7b427d1eb6..a5c3f3e92d 100644 --- a/src/insert.c +++ b/src/insert.c @@ -56,13 +56,13 @@ void sqlite3OpenTable( ** ** Character Column affinity ** ------------------------------ -** 'a' TEXT -** 'b' NONE -** 'c' NUMERIC -** 'd' INTEGER -** 'e' REAL +** 'A' NONE +** 'B' TEXT +** 'C' NUMERIC +** 'D' INTEGER +** 'F' REAL ** -** An extra 'd' is appended to the end of the string to cover the +** An extra 'D' is appended to the end of the string to cover the ** rowid that appears as the last column in every index. ** ** Memory for the buffer containing the column index affinity string @@ -111,11 +111,11 @@ const char *sqlite3IndexAffinityStr(Vdbe *v, Index *pIdx){ ** ** Character Column affinity ** ------------------------------ -** 'a' TEXT -** 'b' NONE -** 'c' NUMERIC -** 'd' INTEGER -** 'e' REAL +** 'A' NONE +** 'B' TEXT +** 'C' NUMERIC +** 'D' INTEGER +** 'E' REAL */ void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ int i; @@ -410,7 +410,7 @@ static int xferOptimization( ** The 4th template is used if the insert statement takes its ** values from a SELECT but the data is being inserted into a table ** that is also read as part of the SELECT. In the third form, -** we have to use a intermediate table to store the results of +** we have to use an intermediate table to store the results of ** the select. The template is like this: ** ** X <- A @@ -575,7 +575,7 @@ void sqlite3Insert( regAutoinc = autoIncBegin(pParse, iDb, pTab); /* Allocate registers for holding the rowid of the new row, - ** the content of the new row, and the assemblied row record. + ** the content of the new row, and the assembled row record. */ regRowid = regIns = pParse->nMem+1; pParse->nMem += pTab->nCol + 1; @@ -1027,7 +1027,7 @@ insert_cleanup: } /* Make sure "isView" and other macros defined above are undefined. Otherwise -** thely may interfere with compilation of other functions in this file +** they may interfere with compilation of other functions in this file ** (or in another file, if this file becomes part of the amalgamation). */ #ifdef isView #undef isView @@ -1143,7 +1143,7 @@ void sqlite3GenerateConstraintChecks( int ix; /* Index loop counter */ int nCol; /* Number of columns */ int onError; /* Conflict resolution strategy */ - int j1; /* Addresss of jump instruction */ + int j1; /* Address of jump instruction */ int seenReplace = 0; /* True if REPLACE is used to resolve INT PK conflict */ int nPkField; /* Number of fields in PRIMARY KEY. 1 for ROWID tables */ int ipkTop = 0; /* Top of the rowid change constraint check */ @@ -1547,7 +1547,7 @@ void sqlite3CompleteInsertion( Index *pIdx; /* An index being inserted or updated */ u8 pik_flags; /* flag values passed to the btree insert */ int regData; /* Content registers (after the rowid) */ - int regRec; /* Register holding assemblied record for the table */ + int regRec; /* Register holding assembled record for the table */ int i; /* Loop counter */ u8 bAffinityDone = 0; /* True if OP_Affinity has been run already */ @@ -1612,6 +1612,9 @@ void sqlite3CompleteInsertion( ** For a WITHOUT ROWID table, *piDataCur will be somewhere in the range ** of *piIdxCurs, depending on where the PRIMARY KEY index appears on the ** pTab->pIndex list. +** +** If pTab is a virtual table, then this routine is a no-op and the +** *piDataCur and *piIdxCur values are left uninitialized. */ int sqlite3OpenTableAndIndices( Parse *pParse, /* Parsing context */ @@ -1630,9 +1633,9 @@ int sqlite3OpenTableAndIndices( assert( op==OP_OpenRead || op==OP_OpenWrite ); if( IsVirtual(pTab) ){ - assert( aToOpen==0 ); - *piDataCur = 0; - *piIdxCur = 1; + /* This routine is a no-op for virtual tables. Leave the output + ** variables *piDataCur and *piIdxCur uninitialized so that valgrind + ** can detect if they are used by mistake in the caller. */ return 0; } iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); @@ -1669,7 +1672,7 @@ int sqlite3OpenTableAndIndices( ** The following global variable is incremented whenever the ** transfer optimization is used. This is used for testing ** purposes only - to make sure the transfer optimization really -** is happening when it is suppose to. +** is happening when it is supposed to. */ int sqlite3_xferopt_count; #endif /* SQLITE_TEST */ @@ -1736,7 +1739,7 @@ static int xferCompatibleIndex(Index *pDest, Index *pSrc){ ** INSERT INTO tab1 SELECT * FROM tab2; ** ** The xfer optimization transfers raw records from tab2 over to tab1. -** Columns are not decoded and reassemblied, which greatly improves +** Columns are not decoded and reassembled, which greatly improves ** performance. Raw index records are transferred in the same way. ** ** The xfer optimization is only attempted if tab1 and tab2 are compatible. @@ -1886,7 +1889,7 @@ static int xferOptimization( } } for(pDestIdx=pDest->pIndex; pDestIdx; pDestIdx=pDestIdx->pNext){ - if( pDestIdx->onError!=OE_None ){ + if( IsUniqueIndex(pDestIdx) ){ destHasUniqueIdx = 1; } for(pSrcIdx=pSrc->pIndex; pSrcIdx; pSrcIdx=pSrcIdx->pNext){ diff --git a/src/legacy.c b/src/legacy.c index 94649ae705..a10006e558 100644 --- a/src/legacy.c +++ b/src/legacy.c @@ -44,7 +44,7 @@ int sqlite3_exec( if( zSql==0 ) zSql = ""; sqlite3_mutex_enter(db->mutex); - sqlite3Error(db, SQLITE_OK, 0); + sqlite3Error(db, SQLITE_OK); while( rc==SQLITE_OK && zSql[0] ){ int nCol; char **azVals = 0; @@ -96,10 +96,13 @@ int sqlite3_exec( } } if( xCallback(pArg, nCol, azVals, azCols) ){ + /* EVIDENCE-OF: R-38229-40159 If the callback function to + ** sqlite3_exec() returns non-zero, then sqlite3_exec() will + ** return SQLITE_ABORT. */ rc = SQLITE_ABORT; sqlite3VdbeFinalize((Vdbe *)pStmt); pStmt = 0; - sqlite3Error(db, SQLITE_ABORT, 0); + sqlite3Error(db, SQLITE_ABORT); goto exec_out; } } @@ -122,14 +125,14 @@ exec_out: sqlite3DbFree(db, azCols); rc = sqlite3ApiExit(db, rc); - if( rc!=SQLITE_OK && ALWAYS(rc==sqlite3_errcode(db)) && pzErrMsg ){ + if( rc!=SQLITE_OK && pzErrMsg ){ int nErrMsg = 1 + sqlite3Strlen30(sqlite3_errmsg(db)); *pzErrMsg = sqlite3Malloc(nErrMsg); if( *pzErrMsg ){ memcpy(*pzErrMsg, sqlite3_errmsg(db), nErrMsg); }else{ rc = SQLITE_NOMEM; - sqlite3Error(db, SQLITE_NOMEM, 0); + sqlite3Error(db, SQLITE_NOMEM); } }else if( pzErrMsg ){ *pzErrMsg = 0; diff --git a/src/lempar.c b/src/lempar.c index 2afaa6cea6..ba0837c0ab 100644 --- a/src/lempar.c +++ b/src/lempar.c @@ -271,9 +271,9 @@ static void yyGrowStack(yyParser *p){ ** A pointer to a parser. This pointer is used in subsequent calls ** to Parse and ParseFree. */ -void *ParseAlloc(void *(*mallocProc)(size_t)){ +void *ParseAlloc(void *(*mallocProc)(u64)){ yyParser *pParser; - pParser = (yyParser*)(*mallocProc)( (size_t)sizeof(yyParser) ); + pParser = (yyParser*)(*mallocProc)( (u64)sizeof(yyParser) ); if( pParser ){ pParser->yyidx = -1; #ifdef YYTRACKMAXSTACKDEPTH diff --git a/src/loadext.c b/src/loadext.c index 828e865b61..7b39ff1671 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -34,7 +34,6 @@ # define sqlite3_column_table_name16 0 # define sqlite3_column_origin_name 0 # define sqlite3_column_origin_name16 0 -# define sqlite3_table_column_metadata 0 #endif #ifdef SQLITE_OMIT_AUTHORIZATION @@ -390,7 +389,20 @@ static const sqlite3_api_routines sqlite3Apis = { sqlite3_uri_int64, sqlite3_uri_parameter, sqlite3_vsnprintf, - sqlite3_wal_checkpoint_v2 + sqlite3_wal_checkpoint_v2, + /* Version 3.8.7 and later */ + sqlite3_auto_extension, + sqlite3_bind_blob64, + sqlite3_bind_text64, + sqlite3_cancel_auto_extension, + sqlite3_load_extension, + sqlite3_malloc64, + sqlite3_msize, + sqlite3_realloc64, + sqlite3_reset_auto_extension, + sqlite3_result_blob64, + sqlite3_result_text64, + sqlite3_strglob }; /* @@ -749,7 +761,7 @@ void sqlite3AutoLoadExtensions(sqlite3 *db){ sqlite3_mutex_leave(mutex); zErrmsg = 0; if( xInit && (rc = xInit(db, &zErrmsg, &sqlite3Apis))!=0 ){ - sqlite3Error(db, rc, + sqlite3ErrorWithMsg(db, rc, "automatic extension loading failed: %s", zErrmsg); go = 0; } diff --git a/src/main.c b/src/main.c index d0ecbb56b8..1b551b7074 100644 --- a/src/main.c +++ b/src/main.c @@ -65,7 +65,7 @@ int sqlite3_threadsafe(void){ return SQLITE_THREADSAFE; } ** I/O active are written using this function. These messages ** are intended for debugging activity only. */ -void (*sqlite3IoTrace)(const char*, ...) = 0; +/* not-private */ void (*sqlite3IoTrace)(const char*, ...) = 0; #endif /* @@ -274,6 +274,13 @@ int sqlite3_initialize(void){ ** when this routine is invoked, then this routine is a harmless no-op. */ int sqlite3_shutdown(void){ +#ifdef SQLITE_OMIT_WSD + int rc = sqlite3_wsd_init(4096, 24); + if( rc!=SQLITE_OK ){ + return rc; + } +#endif + if( sqlite3GlobalConfig.isInit ){ #ifdef SQLITE_EXTRA_SHUTDOWN void SQLITE_EXTRA_SHUTDOWN(void); @@ -332,15 +339,17 @@ int sqlite3_config(int op, ...){ switch( op ){ /* Mutex configuration options are only available in a threadsafe - ** compile. + ** compile. */ -#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-54466-46756 */ case SQLITE_CONFIG_SINGLETHREAD: { /* Disable all mutexing */ sqlite3GlobalConfig.bCoreMutex = 0; sqlite3GlobalConfig.bFullMutex = 0; break; } +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-20520-54086 */ case SQLITE_CONFIG_MULTITHREAD: { /* Disable mutexing of database connections */ /* Enable mutexing of core data structures */ @@ -348,17 +357,23 @@ int sqlite3_config(int op, ...){ sqlite3GlobalConfig.bFullMutex = 0; break; } +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-59593-21810 */ case SQLITE_CONFIG_SERIALIZED: { /* Enable all mutexing */ sqlite3GlobalConfig.bCoreMutex = 1; sqlite3GlobalConfig.bFullMutex = 1; break; } +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-63666-48755 */ case SQLITE_CONFIG_MUTEX: { /* Specify an alternative mutex implementation */ sqlite3GlobalConfig.mutex = *va_arg(ap, sqlite3_mutex_methods*); break; } +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-14450-37597 */ case SQLITE_CONFIG_GETMUTEX: { /* Retrieve the current mutex implementation */ *va_arg(ap, sqlite3_mutex_methods*) = sqlite3GlobalConfig.mutex; @@ -366,37 +381,61 @@ int sqlite3_config(int op, ...){ } #endif - case SQLITE_CONFIG_MALLOC: { - /* Specify an alternative malloc implementation */ + /* EVIDENCE-OF: R-55594-21030 The SQLITE_CONFIG_MALLOC option takes a + ** single argument which is a pointer to an instance of the + ** sqlite3_mem_methods structure. The argument specifies alternative + ** low-level memory allocation routines to be used in place of the memory + ** allocation routines built into SQLite. */ sqlite3GlobalConfig.m = *va_arg(ap, sqlite3_mem_methods*); break; } case SQLITE_CONFIG_GETMALLOC: { - /* Retrieve the current malloc() implementation */ + /* EVIDENCE-OF: R-51213-46414 The SQLITE_CONFIG_GETMALLOC option takes a + ** single argument which is a pointer to an instance of the + ** sqlite3_mem_methods structure. The sqlite3_mem_methods structure is + ** filled with the currently defined memory allocation routines. */ if( sqlite3GlobalConfig.m.xMalloc==0 ) sqlite3MemSetDefault(); *va_arg(ap, sqlite3_mem_methods*) = sqlite3GlobalConfig.m; break; } case SQLITE_CONFIG_MEMSTATUS: { - /* Enable or disable the malloc status collection */ + /* EVIDENCE-OF: R-61275-35157 The SQLITE_CONFIG_MEMSTATUS option takes + ** single argument of type int, interpreted as a boolean, which enables + ** or disables the collection of memory allocation statistics. */ sqlite3GlobalConfig.bMemstat = va_arg(ap, int); break; } case SQLITE_CONFIG_SCRATCH: { - /* Designate a buffer for scratch memory space */ + /* EVIDENCE-OF: R-08404-60887 There are three arguments to + ** SQLITE_CONFIG_SCRATCH: A pointer an 8-byte aligned memory buffer from + ** which the scratch allocations will be drawn, the size of each scratch + ** allocation (sz), and the maximum number of scratch allocations (N). */ sqlite3GlobalConfig.pScratch = va_arg(ap, void*); sqlite3GlobalConfig.szScratch = va_arg(ap, int); sqlite3GlobalConfig.nScratch = va_arg(ap, int); break; } case SQLITE_CONFIG_PAGECACHE: { - /* Designate a buffer for page cache memory space */ + /* EVIDENCE-OF: R-31408-40510 There are three arguments to + ** SQLITE_CONFIG_PAGECACHE: A pointer to 8-byte aligned memory, the size + ** of each page buffer (sz), and the number of pages (N). */ sqlite3GlobalConfig.pPage = va_arg(ap, void*); sqlite3GlobalConfig.szPage = va_arg(ap, int); sqlite3GlobalConfig.nPage = va_arg(ap, int); break; } + case SQLITE_CONFIG_PCACHE_HDRSZ: { + /* EVIDENCE-OF: R-39100-27317 The SQLITE_CONFIG_PCACHE_HDRSZ option takes + ** a single parameter which is a pointer to an integer and writes into + ** that integer the number of extra bytes per page required for each page + ** in SQLITE_CONFIG_PAGECACHE. */ + *va_arg(ap, int*) = + sqlite3HeaderSizeBtree() + + sqlite3HeaderSizePcache() + + sqlite3HeaderSizePcache1(); + break; + } case SQLITE_CONFIG_PCACHE: { /* no-op */ @@ -409,11 +448,18 @@ int sqlite3_config(int op, ...){ } case SQLITE_CONFIG_PCACHE2: { - /* Specify an alternative page cache implementation */ + /* EVIDENCE-OF: R-63325-48378 The SQLITE_CONFIG_PCACHE2 option takes a + ** single argument which is a pointer to an sqlite3_pcache_methods2 + ** object. This object specifies the interface to a custom page cache + ** implementation. */ sqlite3GlobalConfig.pcache2 = *va_arg(ap, sqlite3_pcache_methods2*); break; } case SQLITE_CONFIG_GETPCACHE2: { + /* EVIDENCE-OF: R-22035-46182 The SQLITE_CONFIG_GETPCACHE2 option takes a + ** single argument which is a pointer to an sqlite3_pcache_methods2 + ** object. SQLite copies of the current page cache implementation into + ** that object. */ if( sqlite3GlobalConfig.pcache2.xInit==0 ){ sqlite3PCacheSetDefault(); } @@ -421,9 +467,14 @@ int sqlite3_config(int op, ...){ break; } +/* EVIDENCE-OF: R-06626-12911 The SQLITE_CONFIG_HEAP option is only +** available if SQLite is compiled with either SQLITE_ENABLE_MEMSYS3 or +** SQLITE_ENABLE_MEMSYS5 and returns SQLITE_ERROR if invoked otherwise. */ #if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5) case SQLITE_CONFIG_HEAP: { - /* Designate a buffer for heap memory space */ + /* EVIDENCE-OF: R-19854-42126 There are three arguments to + ** SQLITE_CONFIG_HEAP: An 8-byte aligned pointer to the memory, the + ** number of bytes in the memory buffer, and the minimum allocation size. */ sqlite3GlobalConfig.pHeap = va_arg(ap, void*); sqlite3GlobalConfig.nHeap = va_arg(ap, int); sqlite3GlobalConfig.mnReq = va_arg(ap, int); @@ -436,17 +487,19 @@ int sqlite3_config(int op, ...){ } if( sqlite3GlobalConfig.pHeap==0 ){ - /* If the heap pointer is NULL, then restore the malloc implementation - ** back to NULL pointers too. This will cause the malloc to go - ** back to its default implementation when sqlite3_initialize() is - ** run. + /* EVIDENCE-OF: R-49920-60189 If the first pointer (the memory pointer) + ** is NULL, then SQLite reverts to using its default memory allocator + ** (the system malloc() implementation), undoing any prior invocation of + ** SQLITE_CONFIG_MALLOC. + ** + ** Setting sqlite3GlobalConfig.m to all zeros will cause malloc to + ** revert to its default implementation when sqlite3_initialize() is run */ memset(&sqlite3GlobalConfig.m, 0, sizeof(sqlite3GlobalConfig.m)); }else{ - /* The heap pointer is not NULL, then install one of the - ** mem5.c/mem3.c methods. The enclosing #if guarantees at - ** least one of these methods is currently enabled. - */ + /* EVIDENCE-OF: R-61006-08918 If the memory pointer is not NULL then the + ** alternative memory allocator is engaged to handle all of SQLites + ** memory allocation needs. */ #ifdef SQLITE_ENABLE_MEMSYS3 sqlite3GlobalConfig.m = *sqlite3MemGetMemsys3(); #endif @@ -479,12 +532,25 @@ int sqlite3_config(int op, ...){ break; } + /* EVIDENCE-OF: R-55548-33817 The compile-time setting for URI filenames + ** can be changed at start-time using the + ** sqlite3_config(SQLITE_CONFIG_URI,1) or + ** sqlite3_config(SQLITE_CONFIG_URI,0) configuration calls. + */ case SQLITE_CONFIG_URI: { + /* EVIDENCE-OF: R-25451-61125 The SQLITE_CONFIG_URI option takes a single + ** argument of type int. If non-zero, then URI handling is globally + ** enabled. If the parameter is zero, then URI handling is globally + ** disabled. */ sqlite3GlobalConfig.bOpenUri = va_arg(ap, int); break; } case SQLITE_CONFIG_COVERING_INDEX_SCAN: { + /* EVIDENCE-OF: R-36592-02772 The SQLITE_CONFIG_COVERING_INDEX_SCAN + ** option takes a single integer argument which is interpreted as a + ** boolean in order to enable or disable the use of covering indices for + ** full table scans in the query optimizer. */ sqlite3GlobalConfig.bUseCis = va_arg(ap, int); break; } @@ -499,20 +565,33 @@ int sqlite3_config(int op, ...){ #endif case SQLITE_CONFIG_MMAP_SIZE: { + /* EVIDENCE-OF: R-58063-38258 SQLITE_CONFIG_MMAP_SIZE takes two 64-bit + ** integer (sqlite3_int64) values that are the default mmap size limit + ** (the default setting for PRAGMA mmap_size) and the maximum allowed + ** mmap size limit. */ sqlite3_int64 szMmap = va_arg(ap, sqlite3_int64); sqlite3_int64 mxMmap = va_arg(ap, sqlite3_int64); - if( mxMmap<0 || mxMmap>SQLITE_MAX_MMAP_SIZE ){ - mxMmap = SQLITE_MAX_MMAP_SIZE; - } - sqlite3GlobalConfig.mxMmap = mxMmap; + /* EVIDENCE-OF: R-53367-43190 If either argument to this option is + ** negative, then that argument is changed to its compile-time default. + ** + ** EVIDENCE-OF: R-34993-45031 The maximum allowed mmap size will be + ** silently truncated if necessary so that it does not exceed the + ** compile-time maximum mmap size set by the SQLITE_MAX_MMAP_SIZE + ** compile-time option. + */ + if( mxMmap<0 || mxMmap>SQLITE_MAX_MMAP_SIZE ) mxMmap = SQLITE_MAX_MMAP_SIZE; if( szMmap<0 ) szMmap = SQLITE_DEFAULT_MMAP_SIZE; if( szMmap>mxMmap) szMmap = mxMmap; + sqlite3GlobalConfig.mxMmap = mxMmap; sqlite3GlobalConfig.szMmap = szMmap; break; } -#if SQLITE_OS_WIN && defined(SQLITE_WIN32_MALLOC) +#if SQLITE_OS_WIN && defined(SQLITE_WIN32_MALLOC) /* IMP: R-04780-55815 */ case SQLITE_CONFIG_WIN32_HEAPSIZE: { + /* EVIDENCE-OF: R-34926-03360 SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit + ** unsigned integer value that specifies the maximum size of the created + ** heap. */ sqlite3GlobalConfig.nHeap = va_arg(ap, int); break; } @@ -596,6 +675,12 @@ static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ ** Return the mutex associated with a database connection. */ sqlite3_mutex *sqlite3_db_mutex(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif return db->mutex; } @@ -605,6 +690,10 @@ sqlite3_mutex *sqlite3_db_mutex(sqlite3 *db){ */ int sqlite3_db_release_memory(sqlite3 *db){ int i; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); sqlite3BtreeEnterAll(db); for(i=0; inDb; i++){ @@ -694,13 +783,20 @@ static int binCollFunc( ){ int rc, n; n = nKey1lastRowid; } @@ -742,6 +844,12 @@ sqlite_int64 sqlite3_last_insert_rowid(sqlite3 *db){ ** Return the number of changes in the most recent call to sqlite3_exec(). */ int sqlite3_changes(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif return db->nChange; } @@ -749,6 +857,12 @@ int sqlite3_changes(sqlite3 *db){ ** Return the number of changes since the database handle was opened. */ int sqlite3_total_changes(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif return db->nTotalChange; } @@ -830,6 +944,8 @@ static int connectionIsBusy(sqlite3 *db){ */ static int sqlite3Close(sqlite3 *db, int forceZombie){ if( !db ){ + /* EVIDENCE-OF: R-63257-11740 Calling sqlite3_close() or + ** sqlite3_close_v2() with a NULL pointer argument is a harmless no-op. */ return SQLITE_OK; } if( !sqlite3SafetyCheckSickOrOk(db) ){ @@ -853,7 +969,7 @@ static int sqlite3Close(sqlite3 *db, int forceZombie){ ** SQLITE_BUSY if the connection can not be closed immediately. */ if( !forceZombie && connectionIsBusy(db) ){ - sqlite3Error(db, SQLITE_BUSY, "unable to close due to unfinalized " + sqlite3ErrorWithMsg(db, SQLITE_BUSY, "unable to close due to unfinalized " "statements or unfinished backups"); sqlite3_mutex_leave(db->mutex); return SQLITE_BUSY; @@ -983,9 +1099,13 @@ void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ sqlite3HashClear(&db->aModule); #endif - sqlite3Error(db, SQLITE_OK, 0); /* Deallocates any cached error strings. */ + sqlite3Error(db, SQLITE_OK); /* Deallocates any cached error strings. */ sqlite3ValueFree(db->pErr); sqlite3CloseExtensions(db); +#if SQLITE_USER_AUTHENTICATION + sqlite3_free(db->auth.zAuthUser); + sqlite3_free(db->auth.zAuthPW); +#endif db->magic = SQLITE_MAGIC_ERROR; @@ -1008,13 +1128,15 @@ void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ /* ** Rollback all database files. If tripCode is not SQLITE_OK, then -** any open cursors are invalidated ("tripped" - as in "tripping a circuit +** any write cursors are invalidated ("tripped" - as in "tripping a circuit ** breaker") and made to return tripCode if there are any further -** attempts to use that cursor. +** attempts to use that cursor. Read cursors remain open and valid +** but are "saved" in case the table pages are moved around. */ void sqlite3RollbackAll(sqlite3 *db, int tripCode){ int i; int inTrans = 0; + int schemaChange; assert( sqlite3_mutex_held(db->mutex) ); sqlite3BeginBenignMalloc(); @@ -1025,6 +1147,7 @@ void sqlite3RollbackAll(sqlite3 *db, int tripCode){ ** the database rollback and schema reset, which can cause false ** corruption reports in some cases. */ sqlite3BtreeEnterAll(db); + schemaChange = (db->flags & SQLITE_InternChanges)!=0 && db->init.busy==0; for(i=0; inDb; i++){ Btree *p = db->aDb[i].pBt; @@ -1032,7 +1155,7 @@ void sqlite3RollbackAll(sqlite3 *db, int tripCode){ if( sqlite3BtreeIsInTrans(p) ){ inTrans = 1; } - sqlite3BtreeRollback(p, tripCode); + sqlite3BtreeRollback(p, tripCode, !schemaChange); } } sqlite3VtabRollback(db); @@ -1059,7 +1182,7 @@ void sqlite3RollbackAll(sqlite3 *db, int tripCode){ ** Return a static string containing the name corresponding to the error code ** specified in the argument. */ -#if defined(SQLITE_TEST) +#if (defined(SQLITE_DEBUG) && SQLITE_OS_WIN) || defined(SQLITE_TEST) const char *sqlite3ErrName(int rc){ const char *zName = 0; int i, origRc = rc; @@ -1094,7 +1217,6 @@ const char *sqlite3ErrName(int rc){ case SQLITE_IOERR_UNLOCK: zName = "SQLITE_IOERR_UNLOCK"; break; case SQLITE_IOERR_RDLOCK: zName = "SQLITE_IOERR_RDLOCK"; break; case SQLITE_IOERR_DELETE: zName = "SQLITE_IOERR_DELETE"; break; - case SQLITE_IOERR_BLOCKED: zName = "SQLITE_IOERR_BLOCKED"; break; case SQLITE_IOERR_NOMEM: zName = "SQLITE_IOERR_NOMEM"; break; case SQLITE_IOERR_ACCESS: zName = "SQLITE_IOERR_ACCESS"; break; case SQLITE_IOERR_CHECKRESERVEDLOCK: @@ -1289,6 +1411,9 @@ int sqlite3_busy_handler( int (*xBusy)(void*,int), void *pArg ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE; +#endif sqlite3_mutex_enter(db->mutex); db->busyHandler.xFunc = xBusy; db->busyHandler.pArg = pArg; @@ -1310,6 +1435,12 @@ void sqlite3_progress_handler( int (*xProgress)(void*), void *pArg ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return; + } +#endif sqlite3_mutex_enter(db->mutex); if( nOps>0 ){ db->xProgress = xProgress; @@ -1330,6 +1461,9 @@ void sqlite3_progress_handler( ** specified number of milliseconds before returning 0. */ int sqlite3_busy_timeout(sqlite3 *db, int ms){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif if( ms>0 ){ sqlite3_busy_handler(db, sqliteDefaultBusyCallback, (void*)db); db->busyTimeout = ms; @@ -1343,6 +1477,12 @@ int sqlite3_busy_timeout(sqlite3 *db, int ms){ ** Cause any pending operation to stop at its earliest opportunity. */ void sqlite3_interrupt(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return; + } +#endif db->u1.isInterrupted = 1; } @@ -1417,7 +1557,7 @@ int sqlite3CreateFunc( p = sqlite3FindFunction(db, zFunctionName, nName, nArg, (u8)enc, 0); if( p && (p->funcFlags & SQLITE_FUNC_ENCMASK)==enc && p->nArg==nArg ){ if( db->nVdbeActive ){ - sqlite3Error(db, SQLITE_BUSY, + sqlite3ErrorWithMsg(db, SQLITE_BUSY, "unable to delete/modify user-function due to active statements"); assert( !db->mallocFailed ); return SQLITE_BUSY; @@ -1480,6 +1620,12 @@ int sqlite3_create_function_v2( ){ int rc = SQLITE_ERROR; FuncDestructor *pArg = 0; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } +#endif sqlite3_mutex_enter(db->mutex); if( xDestroy ){ pArg = (FuncDestructor *)sqlite3DbMallocZero(db, sizeof(FuncDestructor)); @@ -1516,6 +1662,10 @@ int sqlite3_create_function16( ){ int rc; char *zFunc8; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zFunctionName==0 ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); assert( !db->mallocFailed ); zFunc8 = sqlite3Utf16to8(db, zFunctionName, -1, SQLITE_UTF16NATIVE); @@ -1547,6 +1697,12 @@ int sqlite3_overload_function( ){ int nName = sqlite3Strlen30(zName); int rc = SQLITE_OK; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 || nArg<-2 ){ + return SQLITE_MISUSE_BKPT; + } +#endif sqlite3_mutex_enter(db->mutex); if( sqlite3FindFunction(db, zName, nName, nArg, SQLITE_UTF8, 0)==0 ){ rc = sqlite3CreateFunc(db, zName, nArg, SQLITE_UTF8, @@ -1568,6 +1724,13 @@ int sqlite3_overload_function( */ void *sqlite3_trace(sqlite3 *db, void (*xTrace)(void*,const char*), void *pArg){ void *pOld; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif sqlite3_mutex_enter(db->mutex); pOld = db->pTraceArg; db->xTrace = xTrace; @@ -1589,6 +1752,13 @@ void *sqlite3_profile( void *pArg ){ void *pOld; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif sqlite3_mutex_enter(db->mutex); pOld = db->pProfileArg; db->xProfile = xProfile; @@ -1609,6 +1779,13 @@ void *sqlite3_commit_hook( void *pArg /* Argument to the function */ ){ void *pOld; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif sqlite3_mutex_enter(db->mutex); pOld = db->pCommitArg; db->xCommitCallback = xCallback; @@ -1627,6 +1804,13 @@ void *sqlite3_update_hook( void *pArg /* Argument to the function */ ){ void *pRet; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif sqlite3_mutex_enter(db->mutex); pRet = db->pUpdateArg; db->xUpdateCallback = xCallback; @@ -1645,6 +1829,13 @@ void *sqlite3_rollback_hook( void *pArg /* Argument to the function */ ){ void *pRet; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif sqlite3_mutex_enter(db->mutex); pRet = db->pRollbackArg; db->xRollbackCallback = xCallback; @@ -1691,6 +1882,9 @@ int sqlite3_wal_autocheckpoint(sqlite3 *db, int nFrame){ UNUSED_PARAMETER(db); UNUSED_PARAMETER(nFrame); #else +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif if( nFrame>0 ){ sqlite3_wal_hook(db, sqlite3WalDefaultHook, SQLITE_INT_TO_PTR(nFrame)); }else{ @@ -1711,6 +1905,12 @@ void *sqlite3_wal_hook( ){ #ifndef SQLITE_OMIT_WAL void *pRet; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif sqlite3_mutex_enter(db->mutex); pRet = db->pWalArg; db->xWalCallback = xCallback; @@ -1738,14 +1938,21 @@ int sqlite3_wal_checkpoint_v2( int rc; /* Return code */ int iDb = SQLITE_MAX_ATTACHED; /* sqlite3.aDb[] index of db to checkpoint */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + /* Initialize the output variables to -1 in case an error occurs. */ if( pnLog ) *pnLog = -1; if( pnCkpt ) *pnCkpt = -1; - assert( SQLITE_CHECKPOINT_FULL>SQLITE_CHECKPOINT_PASSIVE ); - assert( SQLITE_CHECKPOINT_FULLSQLITE_CHECKPOINT_RESTART ){ + assert( SQLITE_CHECKPOINT_PASSIVE==0 ); + assert( SQLITE_CHECKPOINT_FULL==1 ); + assert( SQLITE_CHECKPOINT_RESTART==2 ); + assert( SQLITE_CHECKPOINT_TRUNCATE==3 ); + if( eModeSQLITE_CHECKPOINT_TRUNCATE ){ + /* EVIDENCE-OF: R-03996-12088 The M parameter must be a valid checkpoint + ** mode: */ return SQLITE_MISUSE; } @@ -1755,10 +1962,10 @@ int sqlite3_wal_checkpoint_v2( } if( iDb<0 ){ rc = SQLITE_ERROR; - sqlite3Error(db, SQLITE_ERROR, "unknown database: %s", zDb); + sqlite3ErrorWithMsg(db, SQLITE_ERROR, "unknown database: %s", zDb); }else{ rc = sqlite3Checkpoint(db, iDb, eMode, pnLog, pnCkpt); - sqlite3Error(db, rc, 0); + sqlite3Error(db, rc); } rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); @@ -1773,7 +1980,9 @@ int sqlite3_wal_checkpoint_v2( ** checkpointed. */ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){ - return sqlite3_wal_checkpoint_v2(db, zDb, SQLITE_CHECKPOINT_PASSIVE, 0, 0); + /* EVIDENCE-OF: R-41613-20553 The sqlite3_wal_checkpoint(D,X) is equivalent to + ** sqlite3_wal_checkpoint_v2(D,X,SQLITE_CHECKPOINT_PASSIVE,0,0). */ + return sqlite3_wal_checkpoint_v2(db,zDb,SQLITE_CHECKPOINT_PASSIVE,0,0); } #ifndef SQLITE_OMIT_WAL @@ -1913,7 +2122,7 @@ const void *sqlite3_errmsg16(sqlite3 *db){ }else{ z = sqlite3_value_text16(db->pErr); if( z==0 ){ - sqlite3Error(db, db->errCode, sqlite3ErrStr(db->errCode)); + sqlite3ErrorWithMsg(db, db->errCode, sqlite3ErrStr(db->errCode)); z = sqlite3_value_text16(db->pErr); } /* A malloc() may have failed within the call to sqlite3_value_text16() @@ -1960,32 +2169,6 @@ const char *sqlite3_errstr(int rc){ return sqlite3ErrStr(rc); } -/* -** Invalidate all cached KeyInfo objects for database connection "db" -*/ -static void invalidateCachedKeyInfo(sqlite3 *db){ - Db *pDb; /* A single database */ - int iDb; /* The database index number */ - HashElem *k; /* For looping over tables in pDb */ - Table *pTab; /* A table in the database */ - Index *pIdx; /* Each index */ - - for(iDb=0, pDb=db->aDb; iDbnDb; iDb++, pDb++){ - if( pDb->pBt==0 ) continue; - sqlite3BtreeEnter(pDb->pBt); - for(k=sqliteHashFirst(&pDb->pSchema->tblHash); k; k=sqliteHashNext(k)){ - pTab = (Table*)sqliteHashData(k); - for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - if( pIdx->pKeyInfo && pIdx->pKeyInfo->db==db ){ - sqlite3KeyInfoUnref(pIdx->pKeyInfo); - pIdx->pKeyInfo = 0; - } - } - } - sqlite3BtreeLeave(pDb->pBt); - } -} - /* ** Create a new collating function for database "db". The name is zName ** and the encoding is enc. @@ -2000,7 +2183,6 @@ static int createCollation( ){ CollSeq *pColl; int enc2; - int nName = sqlite3Strlen30(zName); assert( sqlite3_mutex_held(db->mutex) ); @@ -2025,12 +2207,11 @@ static int createCollation( pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 0); if( pColl && pColl->xCmp ){ if( db->nVdbeActive ){ - sqlite3Error(db, SQLITE_BUSY, + sqlite3ErrorWithMsg(db, SQLITE_BUSY, "unable to delete/modify collation sequence due to active statements"); return SQLITE_BUSY; } sqlite3ExpirePreparedStatements(db); - invalidateCachedKeyInfo(db); /* If collation sequence pColl was created directly by a call to ** sqlite3_create_collation, and not generated by synthCollSeq(), @@ -2039,7 +2220,7 @@ static int createCollation( ** to be called. */ if( (pColl->enc & ~SQLITE_UTF16_ALIGNED)==enc2 ){ - CollSeq *aColl = sqlite3HashFind(&db->aCollSeq, zName, nName); + CollSeq *aColl = sqlite3HashFind(&db->aCollSeq, zName); int j; for(j=0; j<3; j++){ CollSeq *p = &aColl[j]; @@ -2059,7 +2240,7 @@ static int createCollation( pColl->pUser = pCtx; pColl->xDel = xDel; pColl->enc = (u8)(enc2 | (enc & SQLITE_UTF16_ALIGNED)); - sqlite3Error(db, SQLITE_OK, 0); + sqlite3Error(db, SQLITE_OK); return SQLITE_OK; } @@ -2079,8 +2260,9 @@ static const int aHardLimit[] = { SQLITE_MAX_FUNCTION_ARG, SQLITE_MAX_ATTACHED, SQLITE_MAX_LIKE_PATTERN_LENGTH, - SQLITE_MAX_VARIABLE_NUMBER, + SQLITE_MAX_VARIABLE_NUMBER, /* IMP: R-38091-32352 */ SQLITE_MAX_TRIGGER_DEPTH, + SQLITE_MAX_WORKER_THREADS, }; /* @@ -2104,8 +2286,8 @@ static const int aHardLimit[] = { #if SQLITE_MAX_FUNCTION_ARG<0 || SQLITE_MAX_FUNCTION_ARG>1000 # error SQLITE_MAX_FUNCTION_ARG must be between 0 and 1000 #endif -#if SQLITE_MAX_ATTACHED<0 || SQLITE_MAX_ATTACHED>62 -# error SQLITE_MAX_ATTACHED must be between 0 and 62 +#if SQLITE_MAX_ATTACHED<0 || SQLITE_MAX_ATTACHED>125 +# error SQLITE_MAX_ATTACHED must be between 0 and 125 #endif #if SQLITE_MAX_LIKE_PATTERN_LENGTH<1 # error SQLITE_MAX_LIKE_PATTERN_LENGTH must be at least 1 @@ -2116,6 +2298,9 @@ static const int aHardLimit[] = { #if SQLITE_MAX_TRIGGER_DEPTH<1 # error SQLITE_MAX_TRIGGER_DEPTH must be at least 1 #endif +#if SQLITE_MAX_WORKER_THREADS<0 || SQLITE_MAX_WORKER_THREADS>50 +# error SQLITE_MAX_WORKER_THREADS must be between 0 and 50 +#endif /* @@ -2131,6 +2316,12 @@ static const int aHardLimit[] = { int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ int oldLimit; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return -1; + } +#endif /* EVIDENCE-OF: R-30189-54097 For each limit category SQLITE_LIMIT_NAME ** there is a hard upper bound set at compile-time by a C preprocessor @@ -2149,7 +2340,8 @@ int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ SQLITE_MAX_LIKE_PATTERN_LENGTH ); assert( aHardLimit[SQLITE_LIMIT_VARIABLE_NUMBER]==SQLITE_MAX_VARIABLE_NUMBER); assert( aHardLimit[SQLITE_LIMIT_TRIGGER_DEPTH]==SQLITE_MAX_TRIGGER_DEPTH ); - assert( SQLITE_LIMIT_TRIGGER_DEPTH==(SQLITE_N_LIMIT-1) ); + assert( aHardLimit[SQLITE_LIMIT_WORKER_THREADS]==SQLITE_MAX_WORKER_THREADS ); + assert( SQLITE_LIMIT_WORKER_THREADS==(SQLITE_N_LIMIT-1) ); if( limitId<0 || limitId>=SQLITE_N_LIMIT ){ @@ -2206,8 +2398,9 @@ int sqlite3ParseUri( assert( *pzErrMsg==0 ); - if( ((flags & SQLITE_OPEN_URI) || sqlite3GlobalConfig.bOpenUri) - && nUri>=5 && memcmp(zUri, "file:", 5)==0 + if( ((flags & SQLITE_OPEN_URI) /* IMP: R-48725-32206 */ + || sqlite3GlobalConfig.bOpenUri) /* IMP: R-51689-46548 */ + && nUri>=5 && memcmp(zUri, "file:", 5)==0 /* IMP: R-57884-37496 */ ){ char *zOpt; int eState; /* Parser state when parsing URI */ @@ -2415,6 +2608,9 @@ static int openDatabase( char *zOpen = 0; /* Filename argument to pass to BtreeOpen() */ char *zErrMsg = 0; /* Error message from sqlite3ParseUri() */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( ppDb==0 ) return SQLITE_MISUSE_BKPT; +#endif *ppDb = 0; #ifndef SQLITE_OMIT_AUTOINIT rc = sqlite3_initialize(); @@ -2437,7 +2633,9 @@ static int openDatabase( testcase( (1<<(flags&7))==0x02 ); /* READONLY */ testcase( (1<<(flags&7))==0x04 ); /* READWRITE */ testcase( (1<<(flags&7))==0x40 ); /* READWRITE | CREATE */ - if( ((1<<(flags&7)) & 0x46)==0 ) return SQLITE_MISUSE_BKPT; + if( ((1<<(flags&7)) & 0x46)==0 ){ + return SQLITE_MISUSE_BKPT; /* IMP: R-65497-44594 */ + } if( sqlite3GlobalConfig.bCoreMutex==0 ){ isThreadsafe = 0; @@ -2496,10 +2694,12 @@ static int openDatabase( assert( sizeof(db->aLimit)==sizeof(aHardLimit) ); memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); + db->aLimit[SQLITE_LIMIT_WORKER_THREADS] = SQLITE_DEFAULT_WORKER_THREADS; db->autoCommit = 1; db->nextAutovac = -1; db->szMmap = sqlite3GlobalConfig.szMmap; db->nextPagesize = 0; + db->nMaxSorterMmap = 0x7FFFFFFF; db->flags |= SQLITE_ShortColNames | SQLITE_EnableTrigger | SQLITE_CacheSpill #if !defined(SQLITE_DEFAULT_AUTOMATIC_INDEX) || SQLITE_DEFAULT_AUTOMATIC_INDEX | SQLITE_AutoIndex @@ -2515,6 +2715,9 @@ static int openDatabase( #endif #if defined(SQLITE_DEFAULT_FOREIGN_KEYS) && SQLITE_DEFAULT_FOREIGN_KEYS | SQLITE_ForeignKeys +#endif +#if defined(SQLITE_REVERSE_UNORDERED_SELECTS) + | SQLITE_ReverseOrder #endif ; sqlite3HashInit(&db->aCollSeq); @@ -2525,26 +2728,30 @@ static int openDatabase( /* Add the default collation sequence BINARY. BINARY works for both UTF-8 ** and UTF-16, so add a version for each to avoid any unnecessary ** conversions. The only error that can occur here is a malloc() failure. + ** + ** EVIDENCE-OF: R-52786-44878 SQLite defines three built-in collating + ** functions: */ createCollation(db, "BINARY", SQLITE_UTF8, 0, binCollFunc, 0); createCollation(db, "BINARY", SQLITE_UTF16BE, 0, binCollFunc, 0); createCollation(db, "BINARY", SQLITE_UTF16LE, 0, binCollFunc, 0); + createCollation(db, "NOCASE", SQLITE_UTF8, 0, nocaseCollatingFunc, 0); createCollation(db, "RTRIM", SQLITE_UTF8, (void*)1, binCollFunc, 0); if( db->mallocFailed ){ goto opendb_out; } + /* EVIDENCE-OF: R-08308-17224 The default collating function for all + ** strings is BINARY. + */ db->pDfltColl = sqlite3FindCollSeq(db, SQLITE_UTF8, "BINARY", 0); assert( db->pDfltColl!=0 ); - /* Also add a UTF-8 case-insensitive collation sequence. */ - createCollation(db, "NOCASE", SQLITE_UTF8, 0, nocaseCollatingFunc, 0); - /* Parse the filename/URI argument. */ db->openFlags = flags; rc = sqlite3ParseUri(zVfs, zFilename, &flags, &db->pVfs, &zOpen, &zErrMsg); if( rc!=SQLITE_OK ){ if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; - sqlite3Error(db, rc, zErrMsg ? "%s" : 0, zErrMsg); + sqlite3ErrorWithMsg(db, rc, zErrMsg ? "%s" : 0, zErrMsg); sqlite3_free(zErrMsg); goto opendb_out; } @@ -2556,13 +2763,15 @@ static int openDatabase( if( rc==SQLITE_IOERR_NOMEM ){ rc = SQLITE_NOMEM; } - sqlite3Error(db, rc, 0); + sqlite3Error(db, rc); goto opendb_out; } + sqlite3BtreeEnter(db->aDb[0].pBt); db->aDb[0].pSchema = sqlite3SchemaGet(db, db->aDb[0].pBt); + if( !db->mallocFailed ) ENC(db) = SCHEMA_ENC(db); + sqlite3BtreeLeave(db->aDb[0].pBt); db->aDb[1].pSchema = sqlite3SchemaGet(db, 0); - /* The default safety_level for the main database is 'full'; for the temp ** database it is 'NONE'. This matches the pager layer defaults. */ @@ -2580,7 +2789,7 @@ static int openDatabase( ** database schema yet. This is delayed until the first time the database ** is accessed. */ - sqlite3Error(db, SQLITE_OK, 0); + sqlite3Error(db, SQLITE_OK); sqlite3RegisterBuiltinFunctions(db); /* Load automatic extensions - extensions that have been registered @@ -2643,7 +2852,7 @@ static int openDatabase( SQLITE_DEFAULT_LOCKING_MODE); #endif - if( rc ) sqlite3Error(db, rc, 0); + if( rc ) sqlite3Error(db, rc); /* Enable the lookaside-malloc subsystem */ setupLookaside(db, 0, sqlite3GlobalConfig.szLookaside, @@ -2707,13 +2916,15 @@ int sqlite3_open16( sqlite3_value *pVal; int rc; - assert( zFilename ); - assert( ppDb ); +#ifdef SQLITE_ENABLE_API_ARMOR + if( ppDb==0 ) return SQLITE_MISUSE_BKPT; +#endif *ppDb = 0; #ifndef SQLITE_OMIT_AUTOINIT rc = sqlite3_initialize(); if( rc ) return rc; #endif + if( zFilename==0 ) zFilename = "\000\000"; pVal = sqlite3ValueNew(0); sqlite3ValueSetStr(pVal, -1, zFilename, SQLITE_UTF16NATIVE, SQLITE_STATIC); zFilename8 = sqlite3ValueText(pVal, SQLITE_UTF8); @@ -2722,7 +2933,7 @@ int sqlite3_open16( SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0); assert( *ppDb || rc==SQLITE_NOMEM ); if( rc==SQLITE_OK && !DbHasProperty(*ppDb, 0, DB_SchemaLoaded) ){ - ENC(*ppDb) = SQLITE_UTF16NATIVE; + SCHEMA_ENC(*ppDb) = ENC(*ppDb) = SQLITE_UTF16NATIVE; } }else{ rc = SQLITE_NOMEM; @@ -2743,13 +2954,7 @@ int sqlite3_create_collation( void* pCtx, int(*xCompare)(void*,int,const void*,int,const void*) ){ - int rc; - sqlite3_mutex_enter(db->mutex); - assert( !db->mallocFailed ); - rc = createCollation(db, zName, (u8)enc, pCtx, xCompare, 0); - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); - return rc; + return sqlite3_create_collation_v2(db, zName, enc, pCtx, xCompare, 0); } /* @@ -2764,6 +2969,10 @@ int sqlite3_create_collation_v2( void(*xDel)(void*) ){ int rc; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); assert( !db->mallocFailed ); rc = createCollation(db, zName, (u8)enc, pCtx, xCompare, xDel); @@ -2785,6 +2994,10 @@ int sqlite3_create_collation16( ){ int rc = SQLITE_OK; char *zName8; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); assert( !db->mallocFailed ); zName8 = sqlite3Utf16to8(db, zName, -1, SQLITE_UTF16NATIVE); @@ -2807,6 +3020,9 @@ int sqlite3_collation_needed( void *pCollNeededArg, void(*xCollNeeded)(void*,sqlite3*,int eTextRep,const char*) ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); db->xCollNeeded = xCollNeeded; db->xCollNeeded16 = 0; @@ -2825,6 +3041,9 @@ int sqlite3_collation_needed16( void *pCollNeededArg, void(*xCollNeeded16)(void*,sqlite3*,int eTextRep,const void*) ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); db->xCollNeeded = 0; db->xCollNeeded16 = xCollNeeded16; @@ -2851,13 +3070,19 @@ int sqlite3_global_recover(void){ ** by the next COMMIT or ROLLBACK. */ int sqlite3_get_autocommit(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif return db->autoCommit; } /* -** The following routines are subtitutes for constants SQLITE_CORRUPT, +** The following routines are substitutes for constants SQLITE_CORRUPT, ** SQLITE_MISUSE, SQLITE_CANTOPEN, SQLITE_IOERR and possibly other error -** constants. They server two purposes: +** constants. They serve two purposes: ** ** 1. Serve as a convenient place to set a breakpoint in a debugger ** to detect when version error conditions occurs. @@ -2904,7 +3129,6 @@ void sqlite3_thread_cleanup(void){ ** Return meta information about a specific column of a database table. ** See comment in sqlite3.h (sqlite.h.in) for details. */ -#ifdef SQLITE_ENABLE_COLUMN_METADATA int sqlite3_table_column_metadata( sqlite3 *db, /* Connection handle */ const char *zDbName, /* Database name or NULL */ @@ -2944,11 +3168,8 @@ int sqlite3_table_column_metadata( } /* Find the column for which info is requested */ - if( sqlite3IsRowid(zColumnName) ){ - iCol = pTab->iPKey; - if( iCol>=0 ){ - pCol = &pTab->aCol[iCol]; - } + if( zColumnName==0 ){ + /* Query for existance of table only */ }else{ for(iCol=0; iColnCol; iCol++){ pCol = &pTab->aCol[iCol]; @@ -2957,8 +3178,13 @@ int sqlite3_table_column_metadata( } } if( iCol==pTab->nCol ){ - pTab = 0; - goto error_out; + if( HasRowid(pTab) && sqlite3IsRowid(zColumnName) ){ + iCol = pTab->iPKey; + pCol = iCol>=0 ? &pTab->aCol[iCol] : 0; + }else{ + pTab = 0; + goto error_out; + } } } @@ -3005,13 +3231,12 @@ error_out: zColumnName); rc = SQLITE_ERROR; } - sqlite3Error(db, rc, (zErrMsg?"%s":0), zErrMsg); + sqlite3ErrorWithMsg(db, rc, (zErrMsg?"%s":0), zErrMsg); sqlite3DbFree(db, zErrMsg); rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); return rc; } -#endif /* ** Sleep for a little while. Return the amount of time slept. @@ -3033,6 +3258,9 @@ int sqlite3_sleep(int ms){ ** Enable or disable the extended result codes. */ int sqlite3_extended_result_codes(sqlite3 *db, int onoff){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); db->errMask = onoff ? 0xffffffff : 0xff; sqlite3_mutex_leave(db->mutex); @@ -3046,6 +3274,9 @@ int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, void *pArg){ int rc = SQLITE_ERROR; Btree *pBtree; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); pBtree = sqlite3DbNameToBtree(db, zDbName); if( pBtree ){ @@ -3171,7 +3402,7 @@ int sqlite3_test_control(int op, ...){ ** IMPORTANT: Changing the PENDING byte from 0x40000000 results in ** an incompatible database file format. Changing the PENDING byte ** while any database connection is open results in undefined and - ** dileterious behavior. + ** deleterious behavior. */ case SQLITE_TESTCTRL_PENDING_BYTE: { rc = PENDING_BYTE; @@ -3326,22 +3557,6 @@ int sqlite3_test_control(int op, ...){ break; } -#if defined(SQLITE_ENABLE_TREE_EXPLAIN) - /* sqlite3_test_control(SQLITE_TESTCTRL_EXPLAIN_STMT, - ** sqlite3_stmt*,const char**); - ** - ** If compiled with SQLITE_ENABLE_TREE_EXPLAIN, each sqlite3_stmt holds - ** a string that describes the optimized parse tree. This test-control - ** returns a pointer to that string. - */ - case SQLITE_TESTCTRL_EXPLAIN_STMT: { - sqlite3_stmt *pStmt = va_arg(ap, sqlite3_stmt*); - const char **pzRet = va_arg(ap, const char**); - *pzRet = sqlite3VdbeExplanation((Vdbe*)pStmt); - break; - } -#endif - /* sqlite3_test_control(SQLITE_TESTCTRL_NEVER_CORRUPT, int); ** ** Set or clear a flag that indicates that the database file is always well- @@ -3370,6 +3585,22 @@ int sqlite3_test_control(int op, ...){ break; } + /* sqlite3_test_control(SQLITE_TESTCTRL_SORTER_MMAP, db, nMax); */ + case SQLITE_TESTCTRL_SORTER_MMAP: { + sqlite3 *db = va_arg(ap, sqlite3*); + db->nMaxSorterMmap = va_arg(ap, int); + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_ISINIT); + ** + ** Return SQLITE_OK if SQLite has been initialized and SQLITE_ERROR if + ** not. + */ + case SQLITE_TESTCTRL_ISINIT: { + if( sqlite3GlobalConfig.isInit==0 ) rc = SQLITE_ERROR; + break; + } } va_end(ap); #endif /* SQLITE_OMIT_BUILTIN_TEST */ @@ -3388,7 +3619,7 @@ int sqlite3_test_control(int op, ...){ ** returns a NULL pointer. */ const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam){ - if( zFilename==0 ) return 0; + if( zFilename==0 || zParam==0 ) return 0; zFilename += sqlite3Strlen30(zFilename) + 1; while( zFilename[0] ){ int x = strcmp(zFilename, zParam); @@ -3418,7 +3649,7 @@ sqlite3_int64 sqlite3_uri_int64( ){ const char *z = sqlite3_uri_parameter(zFilename, zParam); sqlite3_int64 v; - if( z && sqlite3Atoi64(z, &v, sqlite3Strlen30(z), SQLITE_UTF8)==SQLITE_OK ){ + if( z && sqlite3DecOrHexToI64(z, &v)==SQLITE_OK ){ bDflt = v; } return bDflt; @@ -3444,7 +3675,14 @@ Btree *sqlite3DbNameToBtree(sqlite3 *db, const char *zDbName){ ** connection. */ const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName){ - Btree *pBt = sqlite3DbNameToBtree(db, zDbName); + Btree *pBt; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + pBt = sqlite3DbNameToBtree(db, zDbName); return pBt ? sqlite3BtreeGetFilename(pBt) : 0; } @@ -3453,6 +3691,13 @@ const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName){ ** no such database exists. */ int sqlite3_db_readonly(sqlite3 *db, const char *zDbName){ - Btree *pBt = sqlite3DbNameToBtree(db, zDbName); + Btree *pBt; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return -1; + } +#endif + pBt = sqlite3DbNameToBtree(db, zDbName); return pBt ? sqlite3BtreeIsReadonly(pBt) : -1; } diff --git a/src/malloc.c b/src/malloc.c index 9c11d07767..4960f91e02 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -294,11 +294,9 @@ static int mallocWithAlarm(int n, void **pp){ ** Allocate memory. This routine is like sqlite3_malloc() except that it ** assumes the memory subsystem has already been initialized. */ -void *sqlite3Malloc(int n){ +void *sqlite3Malloc(u64 n){ void *p; - if( n<=0 /* IMP: R-65312-04917 */ - || n>=0x7fffff00 - ){ + if( n==0 || n>=0x7fffff00 ){ /* A memory allocation of a number of bytes which is near the maximum ** signed integer value might cause an integer overflow inside of the ** xMalloc(). Hence we limit the maximum size to 0x7fffff00, giving @@ -307,12 +305,12 @@ void *sqlite3Malloc(int n){ p = 0; }else if( sqlite3GlobalConfig.bMemstat ){ sqlite3_mutex_enter(mem0.mutex); - mallocWithAlarm(n, &p); + mallocWithAlarm((int)n, &p); sqlite3_mutex_leave(mem0.mutex); }else{ - p = sqlite3GlobalConfig.m.xMalloc(n); + p = sqlite3GlobalConfig.m.xMalloc((int)n); } - assert( EIGHT_BYTE_ALIGNMENT(p) ); /* IMP: R-04675-44850 */ + assert( EIGHT_BYTE_ALIGNMENT(p) ); /* IMP: R-11148-40995 */ return p; } @@ -322,6 +320,12 @@ void *sqlite3Malloc(int n){ ** allocation. */ void *sqlite3_malloc(int n){ +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return n<=0 ? 0 : sqlite3Malloc(n); +} +void *sqlite3_malloc64(sqlite3_uint64 n){ #ifndef SQLITE_OMIT_AUTOINIT if( sqlite3_initialize() ) return 0; #endif @@ -352,22 +356,20 @@ void *sqlite3ScratchMalloc(int n){ assert( n>0 ); sqlite3_mutex_enter(mem0.mutex); + sqlite3StatusSet(SQLITE_STATUS_SCRATCH_SIZE, n); if( mem0.nScratchFree && sqlite3GlobalConfig.szScratch>=n ){ p = mem0.pScratchFree; mem0.pScratchFree = mem0.pScratchFree->pNext; mem0.nScratchFree--; sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_USED, 1); - sqlite3StatusSet(SQLITE_STATUS_SCRATCH_SIZE, n); sqlite3_mutex_leave(mem0.mutex); }else{ - if( sqlite3GlobalConfig.bMemstat ){ - sqlite3StatusSet(SQLITE_STATUS_SCRATCH_SIZE, n); - n = mallocWithAlarm(n, &p); - if( p ) sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_OVERFLOW, n); + sqlite3_mutex_leave(mem0.mutex); + p = sqlite3Malloc(n); + if( sqlite3GlobalConfig.bMemstat && p ){ + sqlite3_mutex_enter(mem0.mutex); + sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_OVERFLOW, sqlite3MallocSize(p)); sqlite3_mutex_leave(mem0.mutex); - }else{ - sqlite3_mutex_leave(mem0.mutex); - p = sqlite3GlobalConfig.m.xMalloc(n); } sqlite3MemdebugSetType(p, MEMTYPE_SCRATCH); } @@ -375,11 +377,12 @@ void *sqlite3ScratchMalloc(int n){ #if SQLITE_THREADSAFE==0 && !defined(NDEBUG) - /* Verify that no more than two scratch allocations per thread - ** are outstanding at one time. (This is only checked in the - ** single-threaded case since checking in the multi-threaded case - ** would be much more complicated.) */ - assert( scratchAllocOut<=1 ); + /* EVIDENCE-OF: R-12970-05880 SQLite will not use more than one scratch + ** buffers per thread. + ** + ** This can only be checked in single-threaded mode. + */ + assert( scratchAllocOut==0 ); if( p ) scratchAllocOut++; #endif @@ -445,29 +448,37 @@ static int isLookaside(sqlite3 *db, void *p){ */ int sqlite3MallocSize(void *p){ assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); - assert( sqlite3MemdebugNoType(p, MEMTYPE_DB) ); return sqlite3GlobalConfig.m.xSize(p); } int sqlite3DbMallocSize(sqlite3 *db, void *p){ - assert( db!=0 ); - assert( sqlite3_mutex_held(db->mutex) ); - if( isLookaside(db, p) ){ - return db->lookaside.sz; + if( db==0 ){ + assert( sqlite3MemdebugNoType(p, ~MEMTYPE_HEAP) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); + return sqlite3MallocSize(p); }else{ - assert( sqlite3MemdebugHasType(p, MEMTYPE_DB) ); - assert( sqlite3MemdebugHasType(p, MEMTYPE_LOOKASIDE|MEMTYPE_HEAP) ); - assert( db!=0 || sqlite3MemdebugNoType(p, MEMTYPE_LOOKASIDE) ); - return sqlite3GlobalConfig.m.xSize(p); + assert( sqlite3_mutex_held(db->mutex) ); + if( isLookaside(db, p) ){ + return db->lookaside.sz; + }else{ + assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + assert( sqlite3MemdebugNoType(p, ~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + return sqlite3GlobalConfig.m.xSize(p); + } } } +sqlite3_uint64 sqlite3_msize(void *p){ + assert( sqlite3MemdebugNoType(p, ~MEMTYPE_HEAP) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); + return (sqlite3_uint64)sqlite3GlobalConfig.m.xSize(p); +} /* ** Free memory previously obtained from sqlite3Malloc(). */ void sqlite3_free(void *p){ if( p==0 ) return; /* IMP: R-49053-54554 */ - assert( sqlite3MemdebugNoType(p, MEMTYPE_DB) ); assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); + assert( sqlite3MemdebugNoType(p, ~MEMTYPE_HEAP) ); if( sqlite3GlobalConfig.bMemstat ){ sqlite3_mutex_enter(mem0.mutex); sqlite3StatusAdd(SQLITE_STATUS_MEMORY_USED, -sqlite3MallocSize(p)); @@ -479,6 +490,14 @@ void sqlite3_free(void *p){ } } +/* +** Add the size of memory allocation "p" to the count in +** *db->pnBytesFreed. +*/ +static SQLITE_NOINLINE void measureAllocationSize(sqlite3 *db, void *p){ + *db->pnBytesFreed += sqlite3DbMallocSize(db,p); +} + /* ** Free memory that might be associated with a particular database ** connection. @@ -488,7 +507,7 @@ void sqlite3DbFree(sqlite3 *db, void *p){ if( p==0 ) return; if( db ){ if( db->pnBytesFreed ){ - *db->pnBytesFreed += sqlite3DbMallocSize(db, p); + measureAllocationSize(db, p); return; } if( isLookaside(db, p) ){ @@ -503,8 +522,8 @@ void sqlite3DbFree(sqlite3 *db, void *p){ return; } } - assert( sqlite3MemdebugHasType(p, MEMTYPE_DB) ); - assert( sqlite3MemdebugHasType(p, MEMTYPE_LOOKASIDE|MEMTYPE_HEAP) ); + assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + assert( sqlite3MemdebugNoType(p, ~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); assert( db!=0 || sqlite3MemdebugNoType(p, MEMTYPE_LOOKASIDE) ); sqlite3MemdebugSetType(p, MEMTYPE_HEAP); sqlite3_free(p); @@ -513,14 +532,16 @@ void sqlite3DbFree(sqlite3 *db, void *p){ /* ** Change the size of an existing memory allocation */ -void *sqlite3Realloc(void *pOld, int nBytes){ +void *sqlite3Realloc(void *pOld, u64 nBytes){ int nOld, nNew, nDiff; void *pNew; + assert( sqlite3MemdebugHasType(pOld, MEMTYPE_HEAP) ); + assert( sqlite3MemdebugNoType(pOld, ~MEMTYPE_HEAP) ); if( pOld==0 ){ - return sqlite3Malloc(nBytes); /* IMP: R-28354-25769 */ + return sqlite3Malloc(nBytes); /* IMP: R-04300-56712 */ } - if( nBytes<=0 ){ - sqlite3_free(pOld); /* IMP: R-31593-10574 */ + if( nBytes==0 ){ + sqlite3_free(pOld); /* IMP: R-26507-47431 */ return 0; } if( nBytes>=0x7fffff00 ){ @@ -531,22 +552,20 @@ void *sqlite3Realloc(void *pOld, int nBytes){ /* IMPLEMENTATION-OF: R-46199-30249 SQLite guarantees that the second ** argument to xRealloc is always a value returned by a prior call to ** xRoundup. */ - nNew = sqlite3GlobalConfig.m.xRoundup(nBytes); + nNew = sqlite3GlobalConfig.m.xRoundup((int)nBytes); if( nOld==nNew ){ pNew = pOld; }else if( sqlite3GlobalConfig.bMemstat ){ sqlite3_mutex_enter(mem0.mutex); - sqlite3StatusSet(SQLITE_STATUS_MALLOC_SIZE, nBytes); + sqlite3StatusSet(SQLITE_STATUS_MALLOC_SIZE, (int)nBytes); nDiff = nNew - nOld; if( sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED) >= mem0.alarmThreshold-nDiff ){ sqlite3MallocAlarm(nDiff); } - assert( sqlite3MemdebugHasType(pOld, MEMTYPE_HEAP) ); - assert( sqlite3MemdebugNoType(pOld, ~MEMTYPE_HEAP) ); pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); if( pNew==0 && mem0.alarmCallback ){ - sqlite3MallocAlarm(nBytes); + sqlite3MallocAlarm((int)nBytes); pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); } if( pNew ){ @@ -557,7 +576,7 @@ void *sqlite3Realloc(void *pOld, int nBytes){ }else{ pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); } - assert( EIGHT_BYTE_ALIGNMENT(pNew) ); /* IMP: R-04675-44850 */ + assert( EIGHT_BYTE_ALIGNMENT(pNew) ); /* IMP: R-11148-40995 */ return pNew; } @@ -566,6 +585,13 @@ void *sqlite3Realloc(void *pOld, int nBytes){ ** subsystem is initialized prior to invoking sqliteRealloc. */ void *sqlite3_realloc(void *pOld, int n){ +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + if( n<0 ) n = 0; /* IMP: R-26507-47431 */ + return sqlite3Realloc(pOld, n); +} +void *sqlite3_realloc64(void *pOld, sqlite3_uint64 n){ #ifndef SQLITE_OMIT_AUTOINIT if( sqlite3_initialize() ) return 0; #endif @@ -576,10 +602,10 @@ void *sqlite3_realloc(void *pOld, int n){ /* ** Allocate and zero memory. */ -void *sqlite3MallocZero(int n){ +void *sqlite3MallocZero(u64 n){ void *p = sqlite3Malloc(n); if( p ){ - memset(p, 0, n); + memset(p, 0, (size_t)n); } return p; } @@ -588,10 +614,10 @@ void *sqlite3MallocZero(int n){ ** Allocate and zero memory. If the allocation fails, make ** the mallocFailed flag in the connection pointer. */ -void *sqlite3DbMallocZero(sqlite3 *db, int n){ +void *sqlite3DbMallocZero(sqlite3 *db, u64 n){ void *p = sqlite3DbMallocRaw(db, n); if( p ){ - memset(p, 0, n); + memset(p, 0, (size_t)n); } return p; } @@ -614,7 +640,7 @@ void *sqlite3DbMallocZero(sqlite3 *db, int n){ ** In other words, if a subsequent malloc (ex: "b") worked, it is assumed ** that all prior mallocs (ex: "a") worked too. */ -void *sqlite3DbMallocRaw(sqlite3 *db, int n){ +void *sqlite3DbMallocRaw(sqlite3 *db, u64 n){ void *p; assert( db==0 || sqlite3_mutex_held(db->mutex) ); assert( db==0 || db->pnBytesFreed==0 ); @@ -649,8 +675,8 @@ void *sqlite3DbMallocRaw(sqlite3 *db, int n){ if( !p && db ){ db->mallocFailed = 1; } - sqlite3MemdebugSetType(p, MEMTYPE_DB | - ((db && db->lookaside.bEnabled) ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP)); + sqlite3MemdebugSetType(p, + (db && db->lookaside.bEnabled) ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP); return p; } @@ -658,7 +684,7 @@ void *sqlite3DbMallocRaw(sqlite3 *db, int n){ ** Resize the block of memory pointed to by p to n bytes. If the ** resize fails, set the mallocFailed flag in the connection object. */ -void *sqlite3DbRealloc(sqlite3 *db, void *p, int n){ +void *sqlite3DbRealloc(sqlite3 *db, void *p, u64 n){ void *pNew = 0; assert( db!=0 ); assert( sqlite3_mutex_held(db->mutex) ); @@ -676,15 +702,14 @@ void *sqlite3DbRealloc(sqlite3 *db, void *p, int n){ sqlite3DbFree(db, p); } }else{ - assert( sqlite3MemdebugHasType(p, MEMTYPE_DB) ); - assert( sqlite3MemdebugHasType(p, MEMTYPE_LOOKASIDE|MEMTYPE_HEAP) ); + assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + assert( sqlite3MemdebugNoType(p, ~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); sqlite3MemdebugSetType(p, MEMTYPE_HEAP); - pNew = sqlite3_realloc(p, n); + pNew = sqlite3_realloc64(p, n); if( !pNew ){ - sqlite3MemdebugSetType(p, MEMTYPE_DB|MEMTYPE_HEAP); db->mallocFailed = 1; } - sqlite3MemdebugSetType(pNew, MEMTYPE_DB | + sqlite3MemdebugSetType(pNew, (db->lookaside.bEnabled ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP)); } } @@ -695,7 +720,7 @@ void *sqlite3DbRealloc(sqlite3 *db, void *p, int n){ ** Attempt to reallocate p. If the reallocation fails, then free p ** and set the mallocFailed flag in the database connection. */ -void *sqlite3DbReallocOrFree(sqlite3 *db, void *p, int n){ +void *sqlite3DbReallocOrFree(sqlite3 *db, void *p, u64 n){ void *pNew; pNew = sqlite3DbRealloc(db, p, n); if( !pNew ){ @@ -725,7 +750,7 @@ char *sqlite3DbStrDup(sqlite3 *db, const char *z){ } return zNew; } -char *sqlite3DbStrNDup(sqlite3 *db, const char *z, int n){ +char *sqlite3DbStrNDup(sqlite3 *db, const char *z, u64 n){ char *zNew; if( z==0 ){ return 0; @@ -733,7 +758,7 @@ char *sqlite3DbStrNDup(sqlite3 *db, const char *z, int n){ assert( (n&0x7fffffff)==n ); zNew = sqlite3DbMallocRaw(db, n+1); if( zNew ){ - memcpy(zNew, z, n); + memcpy(zNew, z, (size_t)n); zNew[n] = 0; } return zNew; @@ -755,6 +780,14 @@ void sqlite3SetString(char **pz, sqlite3 *db, const char *zFormat, ...){ *pz = z; } +/* +** Take actions at the end of an API call to indicate an OOM error +*/ +static SQLITE_NOINLINE int apiOomError(sqlite3 *db){ + db->mallocFailed = 0; + sqlite3Error(db, SQLITE_NOMEM); + return SQLITE_NOMEM; +} /* ** This function must be called before exiting any API function (i.e. @@ -775,10 +808,9 @@ int sqlite3ApiExit(sqlite3* db, int rc){ ** is unsafe, as is the call to sqlite3Error(). */ assert( !db || sqlite3_mutex_held(db->mutex) ); - if( db && (db->mallocFailed || rc==SQLITE_IOERR_NOMEM) ){ - sqlite3Error(db, SQLITE_NOMEM, 0); - db->mallocFailed = 0; - rc = SQLITE_NOMEM; + if( db==0 ) return rc & 0xff; + if( db->mallocFailed || rc==SQLITE_IOERR_NOMEM ){ + return apiOomError(db); } - return rc & (db ? db->errMask : 0xff); + return rc & db->errMask; } diff --git a/src/mem1.c b/src/mem1.c index 6dbf1058ea..11fc1771ed 100644 --- a/src/mem1.c +++ b/src/mem1.c @@ -188,7 +188,7 @@ static int sqlite3MemSize(void *pPrior){ ** ** For this low-level interface, we know that pPrior!=0. Cases where ** pPrior==0 while have been intercepted by higher-level routine and -** redirected to xMalloc. Similarly, we know that nByte>0 becauses +** redirected to xMalloc. Similarly, we know that nByte>0 because ** cases where nByte<=0 will have been intercepted by higher-level ** routines and redirected to xFree. */ diff --git a/src/mem2.c b/src/mem2.c index 99ea42517e..51ea297c6a 100644 --- a/src/mem2.c +++ b/src/mem2.c @@ -394,7 +394,7 @@ void sqlite3MemdebugSetType(void *p, u8 eType){ ** This routine is designed for use within an assert() statement, to ** verify the type of an allocation. For example: ** -** assert( sqlite3MemdebugHasType(p, MEMTYPE_DB) ); +** assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); */ int sqlite3MemdebugHasType(void *p, u8 eType){ int rc = 1; @@ -416,7 +416,7 @@ int sqlite3MemdebugHasType(void *p, u8 eType){ ** This routine is designed for use within an assert() statement, to ** verify the type of an allocation. For example: ** -** assert( sqlite3MemdebugNoType(p, MEMTYPE_DB) ); +** assert( sqlite3MemdebugNoType(p, MEMTYPE_LOOKASIDE) ); */ int sqlite3MemdebugNoType(void *p, u8 eType){ int rc = 1; diff --git a/src/mem5.c b/src/mem5.c index 67615bb964..1479ddd0d0 100644 --- a/src/mem5.c +++ b/src/mem5.c @@ -28,7 +28,7 @@ ** 1. All memory allocations sizes are rounded up to a power of 2. ** ** 2. If two adjacent free blocks are the halves of a larger block, -** then the two blocks are coalesed into the single larger block. +** then the two blocks are coalesced into the single larger block. ** ** 3. New memory is allocated from the first available free block. ** diff --git a/src/memjournal.c b/src/memjournal.c index 65ed378b38..6452cecc37 100644 --- a/src/memjournal.c +++ b/src/memjournal.c @@ -26,7 +26,7 @@ typedef struct FileChunk FileChunk; ** ** The size chosen is a little less than a power of two. That way, ** the FileChunk object will have a size that almost exactly fills -** a power-of-two allocation. This mimimizes wasted space in power-of-two +** a power-of-two allocation. This minimizes wasted space in power-of-two ** memory allocators. */ #define JOURNAL_CHUNKSIZE ((int)(1024-sizeof(FileChunk*))) diff --git a/src/mutex.c b/src/mutex.c index b567e7c27e..2b45036289 100644 --- a/src/mutex.c +++ b/src/mutex.c @@ -81,7 +81,8 @@ int sqlite3MutexEnd(void){ */ sqlite3_mutex *sqlite3_mutex_alloc(int id){ #ifndef SQLITE_OMIT_AUTOINIT - if( sqlite3_initialize() ) return 0; + if( id<=SQLITE_MUTEX_RECURSIVE && sqlite3_initialize() ) return 0; + if( id>SQLITE_MUTEX_RECURSIVE && sqlite3MutexInit() ) return 0; #endif return sqlite3GlobalConfig.mutex.xMutexAlloc(id); } diff --git a/src/mutex.h b/src/mutex.h index 0978812252..03eb1faadb 100644 --- a/src/mutex.h +++ b/src/mutex.h @@ -25,7 +25,7 @@ ** Figure out what version of the code to use. The choices are ** ** SQLITE_MUTEX_OMIT No mutex logic. Not even stubs. The -** mutexes implemention cannot be overridden +** mutexes implementation cannot be overridden ** at start-time. ** ** SQLITE_MUTEX_NOOP For single-threaded applications. No diff --git a/src/mutex_noop.c b/src/mutex_noop.c index 456e82a25e..1a900c225a 100644 --- a/src/mutex_noop.c +++ b/src/mutex_noop.c @@ -107,7 +107,7 @@ static int debugMutexEnd(void){ return SQLITE_OK; } ** that means that a mutex could not be allocated. */ static sqlite3_mutex *debugMutexAlloc(int id){ - static sqlite3_debug_mutex aStatic[6]; + static sqlite3_debug_mutex aStatic[SQLITE_MUTEX_STATIC_APP3 - 1]; sqlite3_debug_mutex *pNew = 0; switch( id ){ case SQLITE_MUTEX_FAST: diff --git a/src/mutex_unix.c b/src/mutex_unix.c index eca7295831..c936914d8a 100644 --- a/src/mutex_unix.c +++ b/src/mutex_unix.c @@ -96,10 +96,13 @@ static int pthreadMutexEnd(void){ return SQLITE_OK; } **
  • SQLITE_MUTEX_RECURSIVE **
  • SQLITE_MUTEX_STATIC_MASTER **
  • SQLITE_MUTEX_STATIC_MEM -**
  • SQLITE_MUTEX_STATIC_MEM2 +**
  • SQLITE_MUTEX_STATIC_OPEN **
  • SQLITE_MUTEX_STATIC_PRNG **
  • SQLITE_MUTEX_STATIC_LRU **
  • SQLITE_MUTEX_STATIC_PMEM +**
  • SQLITE_MUTEX_STATIC_APP1 +**
  • SQLITE_MUTEX_STATIC_APP2 +**
  • SQLITE_MUTEX_STATIC_APP3 ** ** ** The first two constants cause sqlite3_mutex_alloc() to create @@ -133,6 +136,9 @@ static sqlite3_mutex *pthreadMutexAlloc(int iType){ SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER }; sqlite3_mutex *p; @@ -169,8 +175,12 @@ static sqlite3_mutex *pthreadMutexAlloc(int iType){ break; } default: { - assert( iType-2 >= 0 ); - assert( iType-2 < ArraySize(staticMutexes) ); +#ifdef SQLITE_ENABLE_API_ARMOR + if( iType-2<0 || iType-2>=ArraySize(staticMutexes) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif p = &staticMutexes[iType-2]; #if SQLITE_MUTEX_NREF p->id = iType; diff --git a/src/mutex_w32.c b/src/mutex_w32.c index f43a152389..a799c86159 100644 --- a/src/mutex_w32.c +++ b/src/mutex_w32.c @@ -9,11 +9,16 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** This file contains the C functions that implement mutexes for win32 +** This file contains the C functions that implement mutexes for Win32. */ #include "sqliteInt.h" #if SQLITE_OS_WIN +/* +** Include code that is common to all os_*.c files +*/ +#include "os_common.h" + /* ** Include the header file for the Windows VFS. */ @@ -22,7 +27,7 @@ /* ** The code in this file is only used if we are compiling multithreaded -** on a win32 system. +** on a Win32 system. */ #ifdef SQLITE_MUTEX_W32 @@ -35,48 +40,22 @@ struct sqlite3_mutex { #ifdef SQLITE_DEBUG volatile int nRef; /* Number of enterances */ volatile DWORD owner; /* Thread holding this mutex */ - int trace; /* True to trace changes */ + volatile int trace; /* True to trace changes */ #endif }; -#define SQLITE_W32_MUTEX_INITIALIZER { 0 } -#ifdef SQLITE_DEBUG -#define SQLITE3_MUTEX_INITIALIZER { SQLITE_W32_MUTEX_INITIALIZER, 0, 0L, (DWORD)0, 0 } -#else -#define SQLITE3_MUTEX_INITIALIZER { SQLITE_W32_MUTEX_INITIALIZER, 0 } -#endif /* -** Return true (non-zero) if we are running under WinNT, Win2K, WinXP, -** or WinCE. Return false (zero) for Win95, Win98, or WinME. -** -** Here is an interesting observation: Win95, Win98, and WinME lack -** the LockFileEx() API. But we can still statically link against that -** API as long as we don't call it win running Win95/98/ME. A call to -** this routine is used to determine if the host is Win95/98/ME or -** WinNT/2K/XP so that we will know whether or not we can safely call -** the LockFileEx() API. -** -** mutexIsNT() is only used for the TryEnterCriticalSection() API call, -** which is only available if your application was compiled with -** _WIN32_WINNT defined to a value >= 0x0400. Currently, the only -** call to TryEnterCriticalSection() is #ifdef'ed out, so #ifdef -** this out as well. +** These are the initializer values used when declaring a "static" mutex +** on Win32. It should be noted that all mutexes require initialization +** on the Win32 platform. */ -#if 0 -#if SQLITE_OS_WINCE || SQLITE_OS_WINRT -# define mutexIsNT() (1) +#define SQLITE_W32_MUTEX_INITIALIZER { 0 } + +#ifdef SQLITE_DEBUG +#define SQLITE3_MUTEX_INITIALIZER { SQLITE_W32_MUTEX_INITIALIZER, 0, \ + 0L, (DWORD)0, 0 } #else - static int mutexIsNT(void){ - static int osType = 0; - if( osType==0 ){ - OSVERSIONINFO sInfo; - sInfo.dwOSVersionInfoSize = sizeof(sInfo); - GetVersionEx(&sInfo); - osType = sInfo.dwPlatformId==VER_PLATFORM_WIN32_NT ? 2 : 1; - } - return osType==2; - } -#endif /* SQLITE_OS_WINCE || SQLITE_OS_WINRT */ +#define SQLITE3_MUTEX_INITIALIZER { SQLITE_W32_MUTEX_INITIALIZER, 0 } #endif #ifdef SQLITE_DEBUG @@ -87,20 +66,24 @@ struct sqlite3_mutex { static int winMutexHeld(sqlite3_mutex *p){ return p->nRef!=0 && p->owner==GetCurrentThreadId(); } + static int winMutexNotheld2(sqlite3_mutex *p, DWORD tid){ return p->nRef==0 || p->owner!=tid; } + static int winMutexNotheld(sqlite3_mutex *p){ - DWORD tid = GetCurrentThreadId(); + DWORD tid = GetCurrentThreadId(); return winMutexNotheld2(p, tid); } #endif - /* ** Initialize and deinitialize the mutex subsystem. */ -static sqlite3_mutex winMutex_staticMutexes[6] = { +static sqlite3_mutex winMutex_staticMutexes[] = { + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, @@ -108,17 +91,20 @@ static sqlite3_mutex winMutex_staticMutexes[6] = { SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER }; -static int winMutex_isInit = 0; -/* As winMutexInit() and winMutexEnd() are called as part -** of the sqlite3_initialize and sqlite3_shutdown() -** processing, the "interlocked" magic is probably not -** strictly necessary. -*/ -static LONG winMutex_lock = 0; +static int winMutex_isInit = 0; +static int winMutex_isNt = -1; /* <0 means "need to query" */ + +/* As the winMutexInit() and winMutexEnd() functions are called as part +** of the sqlite3_initialize() and sqlite3_shutdown() processing, the +** "interlocked" magic used here is probably not strictly necessary. +*/ +static LONG SQLITE_WIN32_VOLATILE winMutex_lock = 0; + +int sqlite3_win32_is_nt(void); /* os_win.c */ void sqlite3_win32_sleep(DWORD milliseconds); /* os_win.c */ -static int winMutexInit(void){ +static int winMutexInit(void){ /* The first to increment to 1 does actual initialization */ if( InterlockedCompareExchange(&winMutex_lock, 1, 0)==0 ){ int i; @@ -131,16 +117,17 @@ static int winMutexInit(void){ } winMutex_isInit = 1; }else{ - /* Someone else is in the process of initing the static mutexes */ + /* Another thread is (in the process of) initializing the static + ** mutexes */ while( !winMutex_isInit ){ sqlite3_win32_sleep(1); } } - return SQLITE_OK; + return SQLITE_OK; } -static int winMutexEnd(void){ - /* The first to decrement to 0 does actual shutdown +static int winMutexEnd(void){ + /* The first to decrement to 0 does actual shutdown ** (which should be the last to shutdown.) */ if( InterlockedCompareExchange(&winMutex_lock, 0, 1)==1 ){ if( winMutex_isInit==1 ){ @@ -151,7 +138,7 @@ static int winMutexEnd(void){ winMutex_isInit = 0; } } - return SQLITE_OK; + return SQLITE_OK; } /* @@ -166,10 +153,13 @@ static int winMutexEnd(void){ **
  • SQLITE_MUTEX_RECURSIVE **
  • SQLITE_MUTEX_STATIC_MASTER **
  • SQLITE_MUTEX_STATIC_MEM -**
  • SQLITE_MUTEX_STATIC_MEM2 +**
  • SQLITE_MUTEX_STATIC_OPEN **
  • SQLITE_MUTEX_STATIC_PRNG **
  • SQLITE_MUTEX_STATIC_LRU **
  • SQLITE_MUTEX_STATIC_PMEM +**
  • SQLITE_MUTEX_STATIC_APP1 +**
  • SQLITE_MUTEX_STATIC_APP2 +**
  • SQLITE_MUTEX_STATIC_APP3 ** ** ** The first two constants cause sqlite3_mutex_alloc() to create @@ -192,7 +182,7 @@ static int winMutexEnd(void){ ** ** Note that if one of the dynamic mutex parameters (SQLITE_MUTEX_FAST ** or SQLITE_MUTEX_RECURSIVE) is used then sqlite3_mutex_alloc() -** returns a different mutex on every call. But for the static +** returns a different mutex on every call. But for the static ** mutex types, the same mutex is returned on every call that has ** the same type number. */ @@ -203,9 +193,12 @@ static sqlite3_mutex *winMutexAlloc(int iType){ case SQLITE_MUTEX_FAST: case SQLITE_MUTEX_RECURSIVE: { p = sqlite3MallocZero( sizeof(*p) ); - if( p ){ + if( p ){ #ifdef SQLITE_DEBUG p->id = iType; +#ifdef SQLITE_WIN32_MUTEX_TRACE_DYNAMIC + p->trace = 1; +#endif #endif #if SQLITE_OS_WINRT InitializeCriticalSectionEx(&p->mutex, 0, 0); @@ -216,12 +209,21 @@ static sqlite3_mutex *winMutexAlloc(int iType){ break; } default: { - assert( winMutex_isInit==1 ); +#ifdef SQLITE_ENABLE_API_ARMOR + if( iType-2<0 || iType-2>=ArraySize(winMutex_staticMutexes) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif assert( iType-2 >= 0 ); assert( iType-2 < ArraySize(winMutex_staticMutexes) ); + assert( winMutex_isInit==1 ); p = &winMutex_staticMutexes[iType-2]; #ifdef SQLITE_DEBUG p->id = iType; +#ifdef SQLITE_WIN32_MUTEX_TRACE_STATIC + p->trace = 1; +#endif #endif break; } @@ -237,8 +239,11 @@ static sqlite3_mutex *winMutexAlloc(int iType){ */ static void winMutexFree(sqlite3_mutex *p){ assert( p ); +#ifdef SQLITE_DEBUG assert( p->nRef==0 && p->owner==0 ); assert( p->id==SQLITE_MUTEX_FAST || p->id==SQLITE_MUTEX_RECURSIVE ); +#endif + assert( winMutex_isInit==1 ); DeleteCriticalSection(&p->mutex); sqlite3_free(p); } @@ -255,30 +260,39 @@ static void winMutexFree(sqlite3_mutex *p){ ** more than once, the behavior is undefined. */ static void winMutexEnter(sqlite3_mutex *p){ -#ifdef SQLITE_DEBUG - DWORD tid = GetCurrentThreadId(); - assert( p->id==SQLITE_MUTEX_RECURSIVE || winMutexNotheld2(p, tid) ); +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) + DWORD tid = GetCurrentThreadId(); #endif +#ifdef SQLITE_DEBUG + assert( p ); + assert( p->id==SQLITE_MUTEX_RECURSIVE || winMutexNotheld2(p, tid) ); +#else + assert( p ); +#endif + assert( winMutex_isInit==1 ); EnterCriticalSection(&p->mutex); #ifdef SQLITE_DEBUG assert( p->nRef>0 || p->owner==0 ); - p->owner = tid; + p->owner = tid; p->nRef++; if( p->trace ){ - printf("enter mutex %p (%d) with nRef=%d\n", p, p->trace, p->nRef); + OSTRACE(("ENTER-MUTEX tid=%lu, mutex=%p (%d), nRef=%d\n", + tid, p, p->trace, p->nRef)); } #endif } + static int winMutexTry(sqlite3_mutex *p){ -#ifndef NDEBUG - DWORD tid = GetCurrentThreadId(); +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) + DWORD tid = GetCurrentThreadId(); #endif int rc = SQLITE_BUSY; + assert( p ); assert( p->id==SQLITE_MUTEX_RECURSIVE || winMutexNotheld2(p, tid) ); /* ** The sqlite3_mutex_try() routine is very rarely used, and when it ** is used it is merely an optimization. So it is OK for it to always - ** fail. + ** fail. ** ** The TryEnterCriticalSection() interface is only available on WinNT. ** And some windows compilers complain if you try to use it without @@ -286,18 +300,27 @@ static int winMutexTry(sqlite3_mutex *p){ ** For that reason, we will omit this optimization for now. See ** ticket #2685. */ -#if 0 - if( mutexIsNT() && TryEnterCriticalSection(&p->mutex) ){ +#if defined(_WIN32_WINNT) && _WIN32_WINNT >= 0x0400 + assert( winMutex_isInit==1 ); + assert( winMutex_isNt>=-1 && winMutex_isNt<=1 ); + if( winMutex_isNt<0 ){ + winMutex_isNt = sqlite3_win32_is_nt(); + } + assert( winMutex_isNt==0 || winMutex_isNt==1 ); + if( winMutex_isNt && TryEnterCriticalSection(&p->mutex) ){ +#ifdef SQLITE_DEBUG p->owner = tid; p->nRef++; +#endif rc = SQLITE_OK; } #else UNUSED_PARAMETER(p); #endif #ifdef SQLITE_DEBUG - if( rc==SQLITE_OK && p->trace ){ - printf("try mutex %p (%d) with nRef=%d\n", p, p->trace, p->nRef); + if( p->trace ){ + OSTRACE(("TRY-MUTEX tid=%lu, mutex=%p (%d), owner=%lu, nRef=%d, rc=%s\n", + tid, p, p->trace, p->owner, p->nRef, sqlite3ErrName(rc))); } #endif return rc; @@ -310,18 +333,23 @@ static int winMutexTry(sqlite3_mutex *p){ ** is not currently allocated. SQLite will never do either. */ static void winMutexLeave(sqlite3_mutex *p){ -#ifndef NDEBUG +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) DWORD tid = GetCurrentThreadId(); +#endif + assert( p ); +#ifdef SQLITE_DEBUG assert( p->nRef>0 ); assert( p->owner==tid ); p->nRef--; if( p->nRef==0 ) p->owner = 0; assert( p->nRef==0 || p->id==SQLITE_MUTEX_RECURSIVE ); #endif + assert( winMutex_isInit==1 ); LeaveCriticalSection(&p->mutex); #ifdef SQLITE_DEBUG if( p->trace ){ - printf("leave mutex %p (%d) with nRef=%d\n", p, p->trace, p->nRef); + OSTRACE(("LEAVE-MUTEX tid=%lu, mutex=%p (%d), nRef=%d\n", + tid, p, p->trace, p->nRef)); } #endif } @@ -343,7 +371,7 @@ sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ 0 #endif }; - return &sMutex; } + #endif /* SQLITE_MUTEX_W32 */ diff --git a/src/notify.c b/src/notify.c index fcab5bfaf0..8137226f35 100644 --- a/src/notify.c +++ b/src/notify.c @@ -184,7 +184,7 @@ int sqlite3_unlock_notify( leaveMutex(); assert( !db->mallocFailed ); - sqlite3Error(db, rc, (rc?"database is deadlocked":0)); + sqlite3ErrorWithMsg(db, rc, (rc?"database is deadlocked":0)); sqlite3_mutex_leave(db->mutex); return rc; } diff --git a/src/os.c b/src/os.c index b6c28a1dc4..2a2cf13c5e 100644 --- a/src/os.c +++ b/src/os.c @@ -361,6 +361,10 @@ int sqlite3_vfs_register(sqlite3_vfs *pVfs, int makeDflt){ int rc = sqlite3_initialize(); if( rc ) return rc; #endif +#ifdef SQLITE_ENABLE_API_ARMOR + if( pVfs==0 ) return SQLITE_MISUSE_BKPT; +#endif + MUTEX_LOGIC( mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); ) sqlite3_mutex_enter(mutex); vfsUnlink(pVfs); diff --git a/src/os.h b/src/os.h index 3920a62ee3..2c1b86f913 100644 --- a/src/os.h +++ b/src/os.h @@ -120,7 +120,7 @@ ** shared locks begins at SHARED_FIRST. ** ** The same locking strategy and -** byte ranges are used for Unix. This leaves open the possiblity of having +** byte ranges are used for Unix. This leaves open the possibility of having ** clients on win95, winNT, and unix all talking to the same shared file ** and all locking correctly. To do so would require that samba (or whatever ** tool is being used for file sharing) implements locks correctly between diff --git a/src/os_unix.c b/src/os_unix.c index 347e82220c..f802d9cd19 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -94,11 +94,10 @@ #include #include #if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 -#include +# include #endif - -#if SQLITE_ENABLE_LOCKING_STYLE +#if SQLITE_ENABLE_LOCKING_STYLE || OS_VXWORKS # include # if OS_VXWORKS # include @@ -300,6 +299,14 @@ static int randomnessPid = 0; # endif #endif +/* +** Explicitly call the 64-bit version of lseek() on Android. Otherwise, lseek() +** is the 32-bit version, even if _FILE_OFFSET_BITS=64 is defined. +*/ +#ifdef __ANDROID__ +# define lseek lseek64 +#endif + /* ** Different Unix systems declare open() in different ways. Same use ** open(const char*,int,mode_t). Others use open(const char*,int,...). @@ -318,7 +325,11 @@ static int posixOpen(const char *zFile, int flags, int mode){ ** we are not running as root. */ static int posixFchown(int fd, uid_t uid, gid_t gid){ +#if OS_VXWORKS + return 0; +#else return geteuid() ? 0 : fchown(fd,uid,gid); +#endif } /* Forward reference */ @@ -374,7 +385,7 @@ static struct unix_syscall { { "read", (sqlite3_syscall_ptr)read, 0 }, #define osRead ((ssize_t(*)(int,void*,size_t))aSyscall[8].pCurrent) -#if defined(USE_PREAD) || SQLITE_ENABLE_LOCKING_STYLE +#if defined(USE_PREAD) || (SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORKS) { "pread", (sqlite3_syscall_ptr)pread, 0 }, #else { "pread", (sqlite3_syscall_ptr)0, 0 }, @@ -391,7 +402,7 @@ static struct unix_syscall { { "write", (sqlite3_syscall_ptr)write, 0 }, #define osWrite ((ssize_t(*)(int,const void*,size_t))aSyscall[11].pCurrent) -#if defined(USE_PREAD) || SQLITE_ENABLE_LOCKING_STYLE +#if defined(USE_PREAD) || (SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORKS) { "pwrite", (sqlite3_syscall_ptr)pwrite, 0 }, #else { "pwrite", (sqlite3_syscall_ptr)0, 0 }, @@ -628,7 +639,7 @@ static int unixMutexHeld(void) { #if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) /* ** Helper function for printing out trace information from debugging -** binaries. This returns the string represetation of the supplied +** binaries. This returns the string representation of the supplied ** integer lock-type. */ static const char *azFileLock(int eFileLock){ @@ -705,9 +716,22 @@ static int lockTrace(int fd, int op, struct flock *p){ /* ** Retry ftruncate() calls that fail due to EINTR +** +** All calls to ftruncate() within this file should be made through this wrapper. +** On the Android platform, bypassing the logic below could lead to a corrupt +** database. */ static int robust_ftruncate(int h, sqlite3_int64 sz){ int rc; +#ifdef __ANDROID__ + /* On Android, ftruncate() always uses 32-bit offsets, even if + ** _FILE_OFFSET_BITS=64 is defined. This means it is unsafe to attempt to + ** truncate a file to any size larger than 2GiB. Silently ignore any + ** such attempts. */ + if( sz>(sqlite3_int64)0x7FFFFFFF ){ + rc = SQLITE_OK; + }else +#endif do{ rc = osFtruncate(h,sz); }while( rc<0 && errno==EINTR ); return rc; } @@ -761,16 +785,6 @@ static int sqliteErrorFromPosixError(int posixError, int sqliteIOErr) { case EPERM: return SQLITE_PERM; - /* EDEADLK is only possible if a call to fcntl(F_SETLKW) is made. And - ** this module never makes such a call. And the code in SQLite itself - ** asserts that SQLITE_IOERR_BLOCKED is never returned. For these reasons - ** this case is also commented out. If the system does set errno to EDEADLK, - ** the default SQLITE_IOERR_XXX code will be returned. */ -#if 0 - case EDEADLK: - return SQLITE_IOERR_BLOCKED; -#endif - #if EOPNOTSUPP!=ENOTSUP case EOPNOTSUPP: /* something went terribly awry, unless during file system support @@ -1303,9 +1317,13 @@ static int findInodeInfo( ** Return TRUE if pFile has been renamed or unlinked since it was first opened. */ static int fileHasMoved(unixFile *pFile){ +#if OS_VXWORKS + return pFile->pInode!=0 && pFile->pId!=pFile->pInode->fileId.pId; +#else struct stat buf; return pFile->pInode!=0 && - (osStat(pFile->zPath, &buf)!=0 || buf.st_ino!=pFile->pInode->fileId.ino); + (osStat(pFile->zPath, &buf)!=0 || buf.st_ino!=pFile->pInode->fileId.ino); +#endif } @@ -2448,7 +2466,6 @@ static int semCheckReservedLock(sqlite3_file *id, int *pResOut) { /* Otherwise see if some other process holds it. */ if( !reserved ){ sem_t *pSem = pFile->pInode->pSem; - struct stat statBuf; if( sem_trywait(pSem)==-1 ){ int tErrno = errno; @@ -2501,7 +2518,6 @@ static int semCheckReservedLock(sqlite3_file *id, int *pResOut) { */ static int semLock(sqlite3_file *id, int eFileLock) { unixFile *pFile = (unixFile*)id; - int fd; sem_t *pSem = pFile->pInode->pSem; int rc = SQLITE_OK; @@ -3103,7 +3119,7 @@ static int nfsUnlock(sqlite3_file *id, int eFileLock){ ** NB: If you define USE_PREAD or USE_PREAD64, then it might also ** be necessary to define _XOPEN_SOURCE to be 500. This varies from ** one system to another. Since SQLite does not define USE_PREAD -** any any form by default, we will not attempt to define _XOPEN_SOURCE. +** in any form by default, we will not attempt to define _XOPEN_SOURCE. ** See tickets #2741 and #2681. ** ** To avoid stomping the errno value on a failed read the lastErrno value @@ -3600,7 +3616,7 @@ static int unixTruncate(sqlite3_file *id, i64 nByte){ nByte = ((nByte + pFile->szChunk - 1)/pFile->szChunk) * pFile->szChunk; } - rc = robust_ftruncate(pFile->h, (off_t)nByte); + rc = robust_ftruncate(pFile->h, nByte); if( rc ){ pFile->lastErrno = errno; return unixLogError(SQLITE_IOERR_TRUNCATE, "ftruncate", pFile->zPath); @@ -3693,24 +3709,27 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ }while( err==EINTR ); if( err ) return SQLITE_IOERR_WRITE; #else - /* If the OS does not have posix_fallocate(), fake it. First use - ** ftruncate() to set the file size, then write a single byte to - ** the last byte in each block within the extended region. This - ** is the same technique used by glibc to implement posix_fallocate() - ** on systems that do not have a real fallocate() system call. + /* If the OS does not have posix_fallocate(), fake it. Write a + ** single byte to the last byte in each block that falls entirely + ** within the extended region. Then, if required, a single byte + ** at offset (nSize-1), to set the size of the file correctly. + ** This is a similar technique to that used by glibc on systems + ** that do not have a real fallocate() call. */ int nBlk = buf.st_blksize; /* File-system block size */ i64 iWrite; /* Next offset to write to */ - if( robust_ftruncate(pFile->h, nSize) ){ - pFile->lastErrno = errno; - return unixLogError(SQLITE_IOERR_TRUNCATE, "ftruncate", pFile->zPath); - } iWrite = ((buf.st_size + 2*nBlk - 1)/nBlk)*nBlk-1; - while( iWrite=buf.st_size ); + assert( (iWrite/nBlk)==((buf.st_size+nBlk-1)/nBlk) ); + assert( ((iWrite+1)%nBlk)==0 ); + for(/*no-op*/; iWritectrlFlags is set. ** ** If *pArg is 0 or 1, then clear or set the mask bit of pFile->ctrlFlags. @@ -3942,7 +3961,7 @@ static int unixSectorSize(sqlite3_file *id){ ** Return the device characteristics for the file. ** ** This VFS is set up to return SQLITE_IOCAP_POWERSAFE_OVERWRITE by default. -** However, that choice is contraversial since technically the underlying +** However, that choice is controversial since technically the underlying ** file system does not always provide powersafe overwrites. (In other ** words, after a power-loss event, parts of the file that were never ** written might end up being altered.) However, non-PSOW behavior is very, @@ -4914,7 +4933,7 @@ static int unixUnfetch(sqlite3_file *fd, i64 iOff, void *p){ ** looks at the filesystem type and tries to guess the best locking ** strategy from that. ** -** For finder-funtion F, two objects are created: +** For finder-function F, two objects are created: ** ** (1) The real finder-function named "FImpt()". ** @@ -4935,7 +4954,7 @@ static int unixUnfetch(sqlite3_file *fd, i64 iOff, void *p){ ** * An I/O method finder function called FINDER that returns a pointer ** to the METHOD object in the previous bullet. */ -#define IOMETHODS(FINDER, METHOD, VERSION, CLOSE, LOCK, UNLOCK, CKLOCK) \ +#define IOMETHODS(FINDER, METHOD, VERSION, CLOSE, LOCK, UNLOCK, CKLOCK, SHMMAP) \ static const sqlite3_io_methods METHOD = { \ VERSION, /* iVersion */ \ CLOSE, /* xClose */ \ @@ -4950,7 +4969,7 @@ static const sqlite3_io_methods METHOD = { \ unixFileControl, /* xFileControl */ \ unixSectorSize, /* xSectorSize */ \ unixDeviceCharacteristics, /* xDeviceCapabilities */ \ - unixShmMap, /* xShmMap */ \ + SHMMAP, /* xShmMap */ \ unixShmLock, /* xShmLock */ \ unixShmBarrier, /* xShmBarrier */ \ unixShmUnmap, /* xShmUnmap */ \ @@ -4976,16 +4995,18 @@ IOMETHODS( unixClose, /* xClose method */ unixLock, /* xLock method */ unixUnlock, /* xUnlock method */ - unixCheckReservedLock /* xCheckReservedLock method */ + unixCheckReservedLock, /* xCheckReservedLock method */ + unixShmMap /* xShmMap method */ ) IOMETHODS( nolockIoFinder, /* Finder function name */ nolockIoMethods, /* sqlite3_io_methods object name */ - 1, /* shared memory is disabled */ + 3, /* shared memory is disabled */ nolockClose, /* xClose method */ nolockLock, /* xLock method */ nolockUnlock, /* xUnlock method */ - nolockCheckReservedLock /* xCheckReservedLock method */ + nolockCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ ) IOMETHODS( dotlockIoFinder, /* Finder function name */ @@ -4994,7 +5015,8 @@ IOMETHODS( dotlockClose, /* xClose method */ dotlockLock, /* xLock method */ dotlockUnlock, /* xUnlock method */ - dotlockCheckReservedLock /* xCheckReservedLock method */ + dotlockCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ ) #if SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORKS @@ -5005,7 +5027,8 @@ IOMETHODS( flockClose, /* xClose method */ flockLock, /* xLock method */ flockUnlock, /* xUnlock method */ - flockCheckReservedLock /* xCheckReservedLock method */ + flockCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ ) #endif @@ -5017,7 +5040,8 @@ IOMETHODS( semClose, /* xClose method */ semLock, /* xLock method */ semUnlock, /* xUnlock method */ - semCheckReservedLock /* xCheckReservedLock method */ + semCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ ) #endif @@ -5029,7 +5053,8 @@ IOMETHODS( afpClose, /* xClose method */ afpLock, /* xLock method */ afpUnlock, /* xUnlock method */ - afpCheckReservedLock /* xCheckReservedLock method */ + afpCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ ) #endif @@ -5054,7 +5079,8 @@ IOMETHODS( proxyClose, /* xClose method */ proxyLock, /* xLock method */ proxyUnlock, /* xUnlock method */ - proxyCheckReservedLock /* xCheckReservedLock method */ + proxyCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ ) #endif @@ -5067,7 +5093,8 @@ IOMETHODS( unixClose, /* xClose method */ unixLock, /* xLock method */ nfsUnlock, /* xUnlock method */ - unixCheckReservedLock /* xCheckReservedLock method */ + unixCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ ) #endif @@ -5176,7 +5203,7 @@ static const sqlite3_io_methods #endif /* OS_VXWORKS && SQLITE_ENABLE_LOCKING_STYLE */ /* -** An abstract type for a pointer to a IO method finder function: +** An abstract type for a pointer to an IO method finder function: */ typedef const sqlite3_io_methods *(*finder_type)(const char*,unixFile*); @@ -5490,7 +5517,7 @@ static UnixUnusedFd *findReusableFd(const char *zPath, int flags){ ** descriptor on the same path, fail, and return an error to SQLite. ** ** Even if a subsequent open() call does succeed, the consequences of - ** not searching for a resusable file descriptor are not dire. */ + ** not searching for a reusable file descriptor are not dire. */ if( 0==osStat(zPath, &sStat) ){ unixInodeInfo *pInode; @@ -5521,7 +5548,7 @@ static UnixUnusedFd *findReusableFd(const char *zPath, int flags){ ** written to *pMode. If an IO error occurs, an SQLite error code is ** returned and the value of *pMode is not modified. ** -** In most cases cases, this routine sets *pMode to 0, which will become +** In most cases, this routine sets *pMode to 0, which will become ** an indication to robust_open() to create the file using ** SQLITE_DEFAULT_FILE_PERMISSIONS adjusted by the umask. ** But if the file being opened is a WAL or regular journal file, then @@ -5888,7 +5915,11 @@ static int unixDelete( UNUSED_PARAMETER(NotUsed); SimulateIOError(return SQLITE_IOERR_DELETE); if( osUnlink(zPath)==(-1) ){ - if( errno==ENOENT ){ + if( errno==ENOENT +#if OS_VXWORKS + || osAccess(zPath,0)!=0 +#endif + ){ rc = SQLITE_IOERR_DELETE_NOENT; }else{ rc = unixLogError(SQLITE_IOERR_DELETE, "unlink", zPath); @@ -6309,7 +6340,7 @@ static int unixGetLastError(sqlite3_vfs *NotUsed, int NotUsed2, char *NotUsed3){ ** proxy path against the values stored in the conch. The conch file is ** stored in the same directory as the database file and the file name ** is patterned after the database file name as ".-conch". -** If the conch file does not exist, or it's contents do not match the +** If the conch file does not exist, or its contents do not match the ** host ID and/or proxy path, then the lock is escalated to an exclusive ** lock and the conch file contents is updated with the host ID and proxy ** path and the lock is downgraded to a shared lock again. If the conch @@ -6361,7 +6392,7 @@ static int unixGetLastError(sqlite3_vfs *NotUsed, int NotUsed2, char *NotUsed3){ ** setting the environment variable SQLITE_FORCE_PROXY_LOCKING to 1 will ** force proxy locking to be used for every database file opened, and 0 ** will force automatic proxy locking to be disabled for all database -** files (explicity calling the SQLITE_SET_LOCKPROXYFILE pragma or +** files (explicitly calling the SQLITE_SET_LOCKPROXYFILE pragma or ** sqlite_file_control API is not affected by SQLITE_FORCE_PROXY_LOCKING). */ diff --git a/src/os_win.c b/src/os_win.c index 015b6c61da..e2f9aecac6 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -34,6 +34,11 @@ with SQLITE_OMIT_WAL." #endif +#if !SQLITE_OS_WINNT && SQLITE_MAX_MMAP_SIZE>0 +# error "Memory mapped files require support from the Windows NT kernel,\ + compile with SQLITE_MAX_MMAP_SIZE=0." +#endif + /* ** Are most of the Win32 ANSI APIs available (i.e. with certain exceptions ** based on the sub-platform)? @@ -72,18 +77,14 @@ #endif /* -** Check if the GetVersionEx[AW] functions should be considered deprecated -** and avoid using them in that case. It should be noted here that if the -** value of the SQLITE_WIN32_GETVERSIONEX pre-processor macro is zero -** (whether via this block or via being manually specified), that implies -** the underlying operating system will always be based on the Windows NT -** Kernel. +** Check to see if the GetVersionEx[AW] functions are deprecated on the +** target system. GetVersionEx was first deprecated in Win8.1. */ #ifndef SQLITE_WIN32_GETVERSIONEX # if defined(NTDDI_VERSION) && NTDDI_VERSION >= NTDDI_WINBLUE -# define SQLITE_WIN32_GETVERSIONEX 0 +# define SQLITE_WIN32_GETVERSIONEX 0 /* GetVersionEx() is deprecated */ # else -# define SQLITE_WIN32_GETVERSIONEX 1 +# define SQLITE_WIN32_GETVERSIONEX 1 /* GetVersionEx() is current */ # endif #endif @@ -155,7 +156,7 @@ ** [sometimes] not used by the code (e.g. via conditional compilation). */ #ifndef UNUSED_VARIABLE_VALUE -# define UNUSED_VARIABLE_VALUE(x) (void)(x) +# define UNUSED_VARIABLE_VALUE(x) (void)(x) #endif /* @@ -167,10 +168,11 @@ /* ** Do we need to manually define the Win32 file mapping APIs for use with WAL -** mode (e.g. these APIs are available in the Windows CE SDK; however, they -** are not present in the header file)? +** mode or memory mapped files (e.g. these APIs are available in the Windows +** CE SDK; however, they are not present in the header file)? */ -#if SQLITE_WIN32_FILEMAPPING_API && !defined(SQLITE_OMIT_WAL) +#if SQLITE_WIN32_FILEMAPPING_API && \ + (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) /* ** Two of the file mapping APIs are different under WinRT. Figure out which ** set we need. @@ -198,13 +200,13 @@ WINBASEAPI LPVOID WINAPI MapViewOfFile(HANDLE, DWORD, DWORD, DWORD, SIZE_T); ** This file mapping API is common to both Win32 and WinRT. */ WINBASEAPI BOOL WINAPI UnmapViewOfFile(LPCVOID); -#endif /* SQLITE_WIN32_FILEMAPPING_API && !defined(SQLITE_OMIT_WAL) */ +#endif /* SQLITE_WIN32_FILEMAPPING_API */ /* ** Some Microsoft compilers lack this definition. */ #ifndef INVALID_FILE_ATTRIBUTES -# define INVALID_FILE_ATTRIBUTES ((DWORD)-1) +# define INVALID_FILE_ATTRIBUTES ((DWORD)-1) #endif #ifndef FILE_FLAG_MASK @@ -254,7 +256,7 @@ struct winFile { int szChunk; /* Chunk size configured by FCNTL_CHUNK_SIZE */ #if SQLITE_OS_WINCE LPWSTR zDeleteOnClose; /* Name of file to delete when closing */ - HANDLE hMutex; /* Mutex used to control access to shared lock */ + HANDLE hMutex; /* Mutex used to control access to shared lock */ HANDLE hShared; /* Shared memory segment used for locking */ winceLock local; /* Locks obtained by this instance of winFile */ winceLock *shared; /* Global shared lock memory for the file */ @@ -414,10 +416,9 @@ const sqlite3_mem_methods *sqlite3MemGetWin32(void); ** can manually set this value to 1 to emulate Win98 behavior. */ #ifdef SQLITE_TEST -int sqlite3_os_type = 0; -#elif !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && \ - defined(SQLITE_WIN32_HAS_ANSI) && defined(SQLITE_WIN32_HAS_WIDE) -static int sqlite3_os_type = 0; +LONG SQLITE_WIN32_VOLATILE sqlite3_os_type = 0; +#else +static LONG SQLITE_WIN32_VOLATILE sqlite3_os_type = 0; #endif #ifndef SYSCALL @@ -492,7 +493,7 @@ static struct win_syscall { LPSECURITY_ATTRIBUTES,DWORD,DWORD,HANDLE))aSyscall[5].pCurrent) #if (!SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_ANSI) && \ - !defined(SQLITE_OMIT_WAL)) + (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0)) { "CreateFileMappingA", (SYSCALL)CreateFileMappingA, 0 }, #else { "CreateFileMappingA", (SYSCALL)0, 0 }, @@ -502,7 +503,7 @@ static struct win_syscall { DWORD,DWORD,DWORD,LPCSTR))aSyscall[6].pCurrent) #if SQLITE_OS_WINCE || (!SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) && \ - !defined(SQLITE_OMIT_WAL)) + (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0)) { "CreateFileMappingW", (SYSCALL)CreateFileMappingW, 0 }, #else { "CreateFileMappingW", (SYSCALL)0, 0 }, @@ -842,7 +843,8 @@ static struct win_syscall { LPOVERLAPPED))aSyscall[48].pCurrent) #endif -#if SQLITE_OS_WINCE || (!SQLITE_OS_WINRT && !defined(SQLITE_OMIT_WAL)) +#if SQLITE_OS_WINCE || (!SQLITE_OS_WINRT && \ + (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0)) { "MapViewOfFile", (SYSCALL)MapViewOfFile, 0 }, #else { "MapViewOfFile", (SYSCALL)0, 0 }, @@ -912,7 +914,7 @@ static struct win_syscall { #define osUnlockFileEx ((BOOL(WINAPI*)(HANDLE,DWORD,DWORD,DWORD, \ LPOVERLAPPED))aSyscall[58].pCurrent) -#if SQLITE_OS_WINCE || !defined(SQLITE_OMIT_WAL) +#if SQLITE_OS_WINCE || !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 { "UnmapViewOfFile", (SYSCALL)UnmapViewOfFile, 0 }, #else { "UnmapViewOfFile", (SYSCALL)0, 0 }, @@ -948,7 +950,7 @@ static struct win_syscall { #define osWaitForSingleObject ((DWORD(WINAPI*)(HANDLE, \ DWORD))aSyscall[63].pCurrent) -#if SQLITE_OS_WINRT +#if !SQLITE_OS_WINCE { "WaitForSingleObjectEx", (SYSCALL)WaitForSingleObjectEx, 0 }, #else { "WaitForSingleObjectEx", (SYSCALL)0, 0 }, @@ -975,7 +977,7 @@ static struct win_syscall { #define osGetFileInformationByHandleEx ((BOOL(WINAPI*)(HANDLE, \ FILE_INFO_BY_HANDLE_CLASS,LPVOID,DWORD))aSyscall[66].pCurrent) -#if SQLITE_OS_WINRT && !defined(SQLITE_OMIT_WAL) +#if SQLITE_OS_WINRT && (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) { "MapViewOfFileFromApp", (SYSCALL)MapViewOfFileFromApp, 0 }, #else { "MapViewOfFileFromApp", (SYSCALL)0, 0 }, @@ -1039,7 +1041,7 @@ static struct win_syscall { #define osGetProcessHeap ((HANDLE(WINAPI*)(VOID))aSyscall[74].pCurrent) -#if SQLITE_OS_WINRT && !defined(SQLITE_OMIT_WAL) +#if SQLITE_OS_WINRT && (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) { "CreateFileMappingFromApp", (SYSCALL)CreateFileMappingFromApp, 0 }, #else { "CreateFileMappingFromApp", (SYSCALL)0, 0 }, @@ -1048,6 +1050,22 @@ static struct win_syscall { #define osCreateFileMappingFromApp ((HANDLE(WINAPI*)(HANDLE, \ LPSECURITY_ATTRIBUTES,ULONG,ULONG64,LPCWSTR))aSyscall[75].pCurrent) +/* +** NOTE: On some sub-platforms, the InterlockedCompareExchange "function" +** is really just a macro that uses a compiler intrinsic (e.g. x64). +** So do not try to make this is into a redefinable interface. +*/ +#if defined(InterlockedCompareExchange) + { "InterlockedCompareExchange", (SYSCALL)0, 0 }, + +#define osInterlockedCompareExchange InterlockedCompareExchange +#else + { "InterlockedCompareExchange", (SYSCALL)InterlockedCompareExchange, 0 }, + +#define osInterlockedCompareExchange ((LONG(WINAPI*)(LONG \ + SQLITE_WIN32_VOLATILE*, LONG,LONG))aSyscall[76].pCurrent) +#endif /* defined(InterlockedCompareExchange) */ + }; /* End of the overrideable system calls */ /* @@ -1185,8 +1203,8 @@ int sqlite3_win32_reset_heap(){ int rc; MUTEX_LOGIC( sqlite3_mutex *pMaster; ) /* The main static mutex */ MUTEX_LOGIC( sqlite3_mutex *pMem; ) /* The memsys static mutex */ - MUTEX_LOGIC( pMaster = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); ) - MUTEX_LOGIC( pMem = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM); ) + MUTEX_LOGIC( pMaster = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER); ) + MUTEX_LOGIC( pMem = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MEM); ) sqlite3_mutex_enter(pMaster); sqlite3_mutex_enter(pMem); winMemAssertMagic(); @@ -1279,6 +1297,16 @@ void sqlite3_win32_sleep(DWORD milliseconds){ #endif } +#if SQLITE_MAX_WORKER_THREADS>0 && !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && \ + SQLITE_THREADSAFE>0 +DWORD sqlite3Win32Wait(HANDLE hObject){ + DWORD rc; + while( (rc = osWaitForSingleObjectEx(hObject, INFINITE, + TRUE))==WAIT_IO_COMPLETION ){} + return rc; +} +#endif + /* ** Return true (non-zero) if we are running under WinNT, Win2K, WinXP, ** or WinCE. Return false (zero) for Win95, Win98, or WinME. @@ -1298,22 +1326,47 @@ void sqlite3_win32_sleep(DWORD milliseconds){ #elif !defined(SQLITE_WIN32_HAS_WIDE) # define osIsNT() (0) #else - static int osIsNT(void){ - if( sqlite3_os_type==0 ){ -#if defined(NTDDI_VERSION) && NTDDI_VERSION >= NTDDI_WIN8 - OSVERSIONINFOW sInfo; - sInfo.dwOSVersionInfoSize = sizeof(sInfo); - osGetVersionExW(&sInfo); -#else - OSVERSIONINFOA sInfo; - sInfo.dwOSVersionInfoSize = sizeof(sInfo); - osGetVersionExA(&sInfo); +# define osIsNT() ((sqlite3_os_type==2) || sqlite3_win32_is_nt()) +#endif + +/* +** This function determines if the machine is running a version of Windows +** based on the NT kernel. +*/ +int sqlite3_win32_is_nt(void){ +#if SQLITE_OS_WINRT + /* + ** NOTE: The WinRT sub-platform is always assumed to be based on the NT + ** kernel. + */ + return 1; +#elif defined(SQLITE_WIN32_GETVERSIONEX) && SQLITE_WIN32_GETVERSIONEX + if( osInterlockedCompareExchange(&sqlite3_os_type, 0, 0)==0 ){ +#if defined(SQLITE_WIN32_HAS_ANSI) + OSVERSIONINFOA sInfo; + sInfo.dwOSVersionInfoSize = sizeof(sInfo); + osGetVersionExA(&sInfo); + osInterlockedCompareExchange(&sqlite3_os_type, + (sInfo.dwPlatformId == VER_PLATFORM_WIN32_NT) ? 2 : 1, 0); +#elif defined(SQLITE_WIN32_HAS_WIDE) + OSVERSIONINFOW sInfo; + sInfo.dwOSVersionInfoSize = sizeof(sInfo); + osGetVersionExW(&sInfo); + osInterlockedCompareExchange(&sqlite3_os_type, + (sInfo.dwPlatformId == VER_PLATFORM_WIN32_NT) ? 2 : 1, 0); #endif - sqlite3_os_type = sInfo.dwPlatformId==VER_PLATFORM_WIN32_NT ? 2 : 1; - } - return sqlite3_os_type==2; } + return osInterlockedCompareExchange(&sqlite3_os_type, 2, 2)==2; +#elif SQLITE_TEST + return osInterlockedCompareExchange(&sqlite3_os_type, 2, 2)==2; +#else + /* + ** NOTE: All sub-platforms where the GetVersionEx[AW] functions are + ** deprecated are always assumed to be based on the NT kernel. + */ + return 1; #endif +} #ifdef SQLITE_WIN32_MALLOC /* @@ -1521,7 +1574,7 @@ void sqlite3MemSetDefault(void){ #endif /* SQLITE_WIN32_MALLOC */ /* -** Convert a UTF-8 string to Microsoft Unicode (UTF-16?). +** Convert a UTF-8 string to Microsoft Unicode (UTF-16?). ** ** Space to hold the returned string is obtained from malloc. */ @@ -1574,7 +1627,7 @@ static char *winUnicodeToUtf8(LPCWSTR zWideFilename){ /* ** Convert an ANSI string to Microsoft Unicode, based on the ** current codepage settings for file apis. -** +** ** Space to hold the returned string is obtained ** from sqlite3_malloc. */ @@ -1648,7 +1701,7 @@ char *sqlite3_win32_mbcs_to_utf8(const char *zFilename){ } /* -** Convert UTF-8 to multibyte character string. Space to hold the +** Convert UTF-8 to multibyte character string. Space to hold the ** returned string is obtained from sqlite3_malloc(). */ char *sqlite3_win32_utf8_to_mbcs(const char *zFilename){ @@ -1788,11 +1841,11 @@ static int winGetLastErrorMsg(DWORD lastErrno, int nBuf, char *zBuf){ ** ** This routine is invoked after an error occurs in an OS function. ** It logs a message using sqlite3_log() containing the current value of -** error code and, if possible, the human-readable equivalent from +** error code and, if possible, the human-readable equivalent from ** FormatMessage. ** ** The first argument passed to the macro should be the error code that -** will be returned to SQLite (e.g. SQLITE_IOERR_DELETE, SQLITE_CANTOPEN). +** will be returned to SQLite (e.g. SQLITE_IOERR_DELETE, SQLITE_CANTOPEN). ** The two subsequent arguments should be the name of the OS function that ** failed and the associated file-system path, if any. */ @@ -1823,7 +1876,7 @@ static int winLogErrorAtLine( /* ** The number of times that a ReadFile(), WriteFile(), and DeleteFile() -** will be retried following a locking error - probably caused by +** will be retried following a locking error - probably caused by ** antivirus software. Also the initial delay before the first retry. ** The delay increases linearly with each retry. */ @@ -1898,7 +1951,7 @@ static int winRetryIoerr(int *pnRetry, DWORD *pError){ */ static void winLogIoerr(int nRetry){ if( nRetry ){ - sqlite3_log(SQLITE_IOERR, + sqlite3_log(SQLITE_IOERR, "delayed %dms for lock/sharing conflict", winIoerrRetryDelay*nRetry*(nRetry+1)/2 ); @@ -1992,17 +2045,17 @@ static int winceCreateLock(const char *zFilename, winFile *pFile){ /* Acquire the mutex before continuing */ winceMutexAcquire(pFile->hMutex); - - /* Since the names of named mutexes, semaphores, file mappings etc are + + /* Since the names of named mutexes, semaphores, file mappings etc are ** case-sensitive, take advantage of that by uppercasing the mutex name ** and using that as the shared filemapping name. */ osCharUpperW(zName); pFile->hShared = osCreateFileMappingW(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, 0, sizeof(winceLock), - zName); + zName); - /* Set a flag that indicates we're the first to create the memory so it + /* Set a flag that indicates we're the first to create the memory so it ** must be zero-initialized */ lastErrno = osGetLastError(); if (lastErrno == ERROR_ALREADY_EXISTS){ @@ -2013,7 +2066,7 @@ static int winceCreateLock(const char *zFilename, winFile *pFile){ /* If we succeeded in making the shared memory handle, map it. */ if( pFile->hShared ){ - pFile->shared = (winceLock*)osMapViewOfFile(pFile->hShared, + pFile->shared = (winceLock*)osMapViewOfFile(pFile->hShared, FILE_MAP_READ|FILE_MAP_WRITE, 0, 0, sizeof(winceLock)); /* If mapping failed, close the shared memory handle and erase it */ if( !pFile->shared ){ @@ -2039,7 +2092,7 @@ static int winceCreateLock(const char *zFilename, winFile *pFile){ pFile->hMutex = NULL; return SQLITE_IOERR; } - + /* Initialize the shared memory if we're supposed to */ if( bInit ){ memset(pFile->shared, 0, sizeof(winceLock)); @@ -2077,13 +2130,13 @@ static void winceDestroyLock(winFile *pFile){ osCloseHandle(pFile->hShared); /* Done with the mutex */ - winceMutexRelease(pFile->hMutex); + winceMutexRelease(pFile->hMutex); osCloseHandle(pFile->hMutex); pFile->hMutex = NULL; } } -/* +/* ** An implementation of the LockFile() API of Windows for CE */ static BOOL winceLockFile( @@ -2294,8 +2347,8 @@ static BOOL winUnlockFile( #endif /* -** Move the current position of the file handle passed as the first -** argument to offset iOffset within the file. If successful, return 0. +** Move the current position of the file handle passed as the first +** argument to offset iOffset within the file. If successful, return 0. ** Otherwise, set pFile->lastErrno and return non-zero. */ static int winSeekFile(winFile *pFile, sqlite3_int64 iOffset){ @@ -2310,11 +2363,11 @@ static int winSeekFile(winFile *pFile, sqlite3_int64 iOffset){ upperBits = (LONG)((iOffset>>32) & 0x7fffffff); lowerBits = (LONG)(iOffset & 0xffffffff); - /* API oddity: If successful, SetFilePointer() returns a dword + /* API oddity: If successful, SetFilePointer() returns a dword ** containing the lower 32-bits of the new file-offset. Or, if it fails, - ** it returns INVALID_SET_FILE_POINTER. However according to MSDN, - ** INVALID_SET_FILE_POINTER may also be a valid new offset. So to determine - ** whether an error has actually occurred, it is also necessary to call + ** it returns INVALID_SET_FILE_POINTER. However according to MSDN, + ** INVALID_SET_FILE_POINTER may also be a valid new offset. So to determine + ** whether an error has actually occurred, it is also necessary to call ** GetLastError(). */ dwRet = osSetFilePointer(pFile->h, lowerBits, &upperBits, FILE_BEGIN); @@ -2397,7 +2450,7 @@ static int winClose(sqlite3_file *id){ int cnt = 0; while( osDeleteFileW(pFile->zDeleteOnClose)==0 - && osGetFileAttributesW(pFile->zDeleteOnClose)!=0xffffffff + && osGetFileAttributesW(pFile->zDeleteOnClose)!=0xffffffff && cnt++ < WINCE_DELETION_ATTEMPTS ){ sqlite3_win32_sleep(100); /* Wait a little before trying again */ @@ -2426,7 +2479,7 @@ static int winRead( int amt, /* Number of bytes to read */ sqlite3_int64 offset /* Begin reading at this offset */ ){ -#if !SQLITE_OS_WINCE +#if !SQLITE_OS_WINCE && !defined(SQLITE_WIN32_NO_OVERLAPPED) OVERLAPPED overlapped; /* The offset for ReadFile. */ #endif winFile *pFile = (winFile*)id; /* file handle */ @@ -2458,7 +2511,7 @@ static int winRead( } #endif -#if SQLITE_OS_WINCE +#if SQLITE_OS_WINCE || defined(SQLITE_WIN32_NO_OVERLAPPED) if( winSeekFile(pFile, offset) ){ OSTRACE(("READ file=%p, rc=SQLITE_FULL\n", pFile->h)); return SQLITE_FULL; @@ -2530,13 +2583,13 @@ static int winWrite( } #endif -#if SQLITE_OS_WINCE +#if SQLITE_OS_WINCE || defined(SQLITE_WIN32_NO_OVERLAPPED) rc = winSeekFile(pFile, offset); if( rc==0 ){ #else { #endif -#if !SQLITE_OS_WINCE +#if !SQLITE_OS_WINCE && !defined(SQLITE_WIN32_NO_OVERLAPPED) OVERLAPPED overlapped; /* The offset for WriteFile. */ #endif u8 *aRem = (u8 *)pBuf; /* Data yet to be written */ @@ -2544,14 +2597,14 @@ static int winWrite( DWORD nWrite; /* Bytes written by each WriteFile() call */ DWORD lastErrno = NO_ERROR; /* Value returned by GetLastError() */ -#if !SQLITE_OS_WINCE +#if !SQLITE_OS_WINCE && !defined(SQLITE_WIN32_NO_OVERLAPPED) memset(&overlapped, 0, sizeof(OVERLAPPED)); overlapped.Offset = (LONG)(offset & 0xffffffff); overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff); #endif while( nRem>0 ){ -#if SQLITE_OS_WINCE +#if SQLITE_OS_WINCE || defined(SQLITE_WIN32_NO_OVERLAPPED) if( !osWriteFile(pFile->h, aRem, nRem, &nWrite, 0) ){ #else if( !osWriteFile(pFile->h, aRem, nRem, &nWrite, &overlapped) ){ @@ -2564,7 +2617,7 @@ static int winWrite( lastErrno = osGetLastError(); break; } -#if !SQLITE_OS_WINCE +#if !SQLITE_OS_WINCE && !defined(SQLITE_WIN32_NO_OVERLAPPED) offset += nWrite; overlapped.Offset = (LONG)(offset & 0xffffffff); overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff); @@ -3087,7 +3140,7 @@ static int winUnlock(sqlite3_file *id, int locktype){ } /* -** If *pArg is inititially negative then this is a query. Set *pArg to +** If *pArg is initially negative then this is a query. Set *pArg to ** 1 or 0 depending on whether or not bit mask of pFile->ctrlFlags is set. ** ** If *pArg is 0 or 1, then clear or set the mask bit of pFile->ctrlFlags. @@ -3245,7 +3298,7 @@ static int winDeviceCharacteristics(sqlite3_file *id){ ((p->ctrlFlags & WINFILE_PSOW)?SQLITE_IOCAP_POWERSAFE_OVERWRITE:0); } -/* +/* ** Windows will only let you create file view mappings ** on allocation size granularity boundaries. ** During sqlite3_os_init() we do a GetSystemInfo() @@ -3257,11 +3310,11 @@ static SYSTEM_INFO winSysInfo; /* ** Helper functions to obtain and relinquish the global mutex. The -** global mutex is used to protect the winLockInfo objects used by +** global mutex is used to protect the winLockInfo objects used by ** this file, all of which may be shared by multiple threads. ** -** Function winShmMutexHeld() is used to assert() that the global mutex -** is held when required. This function is only used as part of assert() +** Function winShmMutexHeld() is used to assert() that the global mutex +** is held when required. This function is only used as part of assert() ** statements. e.g. ** ** winShmEnterMutex() @@ -3291,10 +3344,10 @@ static int winShmMutexHeld(void) { ** this object or while reading or writing the following fields: ** ** nRef -** pNext +** pNext ** ** The following fields are read-only after the object is created: -** +** ** fid ** zFilename ** @@ -3390,7 +3443,7 @@ static int winShmSystemLock( if( lockType == _SHM_WRLCK ) dwFlags |= LOCKFILE_EXCLUSIVE_LOCK; rc = winLockFile(&pFile->hFile.h, dwFlags, ofst, 0, nByte, 0); } - + if( rc!= 0 ){ rc = SQLITE_OK; }else{ @@ -3486,7 +3539,7 @@ static int winOpenSharedMemory(winFile *pDbFd){ } pNew->zFilename = (char*)&pNew[1]; sqlite3_snprintf(nName+15, pNew->zFilename, "%s-shm", pDbFd->zPath); - sqlite3FileSuffix3(pDbFd->zPath, pNew->zFilename); + sqlite3FileSuffix3(pDbFd->zPath, pNew->zFilename); /* Look to see if there is an existing winShmNode that can be used. ** If no matching winShmNode currently exists, create a new one. @@ -3523,7 +3576,7 @@ static int winOpenSharedMemory(winFile *pDbFd){ } /* Check to see if another process is holding the dead-man switch. - ** If not, truncate the file to zero length. + ** If not, truncate the file to zero length. */ if( winShmSystemLock(pShmNode, _SHM_WRLCK, WIN_SHM_DMS, 1)==SQLITE_OK ){ rc = winTruncate((sqlite3_file *)&pShmNode->hFile, 0); @@ -3552,7 +3605,7 @@ static int winOpenSharedMemory(winFile *pDbFd){ ** the cover of the winShmEnterMutex() mutex and the pointer from the ** new (struct winShm) object to the pShmNode has been set. All that is ** left to do is to link the new object into the linked list starting - ** at pShmNode->pFirst. This must be done while holding the pShmNode->mutex + ** at pShmNode->pFirst. This must be done while holding the pShmNode->mutex ** mutex. */ sqlite3_mutex_enter(pShmNode->mutex); @@ -3572,7 +3625,7 @@ shm_open_err: } /* -** Close a connection to shared-memory. Delete the underlying +** Close a connection to shared-memory. Delete the underlying ** storage if deleteFlag is true. */ static int winShmUnmap( @@ -3661,7 +3714,7 @@ static int winShmLock( if( rc==SQLITE_OK ){ p->exclMask &= ~mask; p->sharedMask &= ~mask; - } + } }else if( flags & SQLITE_SHM_SHARED ){ u16 allShared = 0; /* Union of locks held by connections other than "p" */ @@ -3700,7 +3753,7 @@ static int winShmLock( break; } } - + /* Get the exclusive locks at the system level. Then if successful ** also mark the local connection as being locked. */ @@ -3720,7 +3773,7 @@ static int winShmLock( } /* -** Implement a memory barrier or memory fence on shared memory. +** Implement a memory barrier or memory fence on shared memory. ** ** All loads and stores begun before the barrier must complete before ** any load or store begun after the barrier. @@ -3735,22 +3788,22 @@ static void winShmBarrier( } /* -** This function is called to obtain a pointer to region iRegion of the -** shared-memory associated with the database file fd. Shared-memory regions -** are numbered starting from zero. Each shared-memory region is szRegion +** This function is called to obtain a pointer to region iRegion of the +** shared-memory associated with the database file fd. Shared-memory regions +** are numbered starting from zero. Each shared-memory region is szRegion ** bytes in size. ** ** If an error occurs, an error code is returned and *pp is set to NULL. ** ** Otherwise, if the isWrite parameter is 0 and the requested shared-memory ** region has not been allocated (by any client, including one running in a -** separate process), then *pp is set to NULL and SQLITE_OK returned. If -** isWrite is non-zero and the requested shared-memory region has not yet +** separate process), then *pp is set to NULL and SQLITE_OK returned. If +** isWrite is non-zero and the requested shared-memory region has not yet ** been allocated, it is allocated by this function. ** ** If the shared-memory region has already been allocated or is allocated by -** this call as described above, then it is mapped into this processes -** address space (if it is not already), *pp is set to point to the mapped +** this call as described above, then it is mapped into this processes +** address space (if it is not already), *pp is set to point to the mapped ** memory and SQLITE_OK returned. */ static int winShmMap( @@ -3822,17 +3875,17 @@ static int winShmMap( while( pShmNode->nRegion<=iRegion ){ HANDLE hMap = NULL; /* file-mapping handle */ void *pMap = 0; /* Mapped memory region */ - + #if SQLITE_OS_WINRT hMap = osCreateFileMappingFromApp(pShmNode->hFile.h, NULL, PAGE_READWRITE, nByte, NULL ); #elif defined(SQLITE_WIN32_HAS_WIDE) - hMap = osCreateFileMappingW(pShmNode->hFile.h, + hMap = osCreateFileMappingW(pShmNode->hFile.h, NULL, PAGE_READWRITE, 0, nByte, NULL ); #elif defined(SQLITE_WIN32_HAS_ANSI) - hMap = osCreateFileMappingA(pShmNode->hFile.h, + hMap = osCreateFileMappingA(pShmNode->hFile.h, NULL, PAGE_READWRITE, 0, nByte, NULL ); #endif @@ -3929,14 +3982,14 @@ static int winUnmapfile(winFile *pFile){ /* ** Memory map or remap the file opened by file-descriptor pFd (if the file -** is already mapped, the existing mapping is replaced by the new). Or, if -** there already exists a mapping for this file, and there are still +** is already mapped, the existing mapping is replaced by the new). Or, if +** there already exists a mapping for this file, and there are still ** outstanding xFetch() references to it, this function is a no-op. ** -** If parameter nByte is non-negative, then it is the requested size of -** the mapping to create. Otherwise, if nByte is less than zero, then the +** If parameter nByte is non-negative, then it is the requested size of +** the mapping to create. Otherwise, if nByte is less than zero, then the ** requested size is the size of the file on disk. The actual size of the -** created mapping is either the requested size or the value configured +** created mapping is either the requested size or the value configured ** using SQLITE_FCNTL_MMAP_SIZE, whichever is smaller. ** ** SQLITE_OK is returned if no error occurs (even if the mapping is not @@ -3965,7 +4018,7 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ nMap = pFd->mmapSizeMax; } nMap &= ~(sqlite3_int64)(winSysInfo.dwPageSize - 1); - + if( nMap==0 && pFd->mmapSize>0 ){ winUnmapfile(pFd); } @@ -4037,7 +4090,7 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ ** Finally, if an error does occur, return an SQLite error code. The final ** value of *pp is undefined in this case. ** -** If this function does return a pointer, the caller must eventually +** If this function does return a pointer, the caller must eventually ** release the reference by calling winUnfetch(). */ static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ @@ -4072,20 +4125,20 @@ static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ } /* -** If the third argument is non-NULL, then this function releases a +** If the third argument is non-NULL, then this function releases a ** reference obtained by an earlier call to winFetch(). The second ** argument passed to this function must be the same as the corresponding -** argument that was passed to the winFetch() invocation. +** argument that was passed to the winFetch() invocation. ** -** Or, if the third argument is NULL, then this function is being called -** to inform the VFS layer that, according to POSIX, any existing mapping +** Or, if the third argument is NULL, then this function is being called +** to inform the VFS layer that, according to POSIX, any existing mapping ** may now be invalid and should be unmapped. */ static int winUnfetch(sqlite3_file *fd, i64 iOff, void *p){ #if SQLITE_MAX_MMAP_SIZE>0 winFile *pFd = (winFile*)fd; /* The underlying database file */ - /* If p==0 (unmap the entire file) then there must be no outstanding + /* If p==0 (unmap the entire file) then there must be no outstanding ** xFetch references. Or, if p!=0 (meaning it is an xFetch reference), ** then there must be at least one outstanding. */ assert( (p==0)==(pFd->nFetchOut==0) ); @@ -4101,7 +4154,7 @@ static int winUnfetch(sqlite3_file *fd, i64 iOff, void *p){ }else{ /* FIXME: If Windows truly always prevents truncating or deleting a ** file while a mapping is held, then the following winUnmapfile() call - ** is unnecessary can can be omitted - potentially improving + ** is unnecessary can be omitted - potentially improving ** performance. */ winUnmapfile(pFd); } @@ -4231,7 +4284,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ /* It's odd to simulate an io-error here, but really this is just ** using the io-error infrastructure to test that SQLite handles this - ** function failing. + ** function failing. */ SimulateIOError( return SQLITE_IOERR ); @@ -4413,7 +4466,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ } /* - ** Check that the output buffer is large enough for the temporary file + ** Check that the output buffer is large enough for the temporary file ** name in the following format: ** ** "/etilqs_XXXXXXXXXXXXXXX\0\0" @@ -4516,8 +4569,8 @@ static int winOpen( #ifndef NDEBUG int isOpenJournal = (isCreate && ( - eType==SQLITE_OPEN_MASTER_JOURNAL - || eType==SQLITE_OPEN_MAIN_JOURNAL + eType==SQLITE_OPEN_MASTER_JOURNAL + || eType==SQLITE_OPEN_MAIN_JOURNAL || eType==SQLITE_OPEN_WAL )); #endif @@ -4525,9 +4578,9 @@ static int winOpen( OSTRACE(("OPEN name=%s, pFile=%p, flags=%x, pOutFlags=%p\n", zUtf8Name, id, flags, pOutFlags)); - /* Check the following statements are true: + /* Check the following statements are true: ** - ** (a) Exactly one of the READWRITE and READONLY flags must be set, and + ** (a) Exactly one of the READWRITE and READONLY flags must be set, and ** (b) if CREATE is set, then READWRITE must also be set, and ** (c) if EXCLUSIVE is set, then CREATE must also be set. ** (d) if DELETEONCLOSE is set, then CREATE must also be set. @@ -4537,7 +4590,7 @@ static int winOpen( assert(isExclusive==0 || isCreate); assert(isDelete==0 || isCreate); - /* The main DB, main journal, WAL file and master journal are never + /* The main DB, main journal, WAL file and master journal are never ** automatically deleted. Nor are they ever temporary files. */ assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MAIN_DB ); assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MAIN_JOURNAL ); @@ -4545,9 +4598,9 @@ static int winOpen( assert( (!isDelete && zName) || eType!=SQLITE_OPEN_WAL ); /* Assert that the upper layer has set one of the "file-type" flags. */ - assert( eType==SQLITE_OPEN_MAIN_DB || eType==SQLITE_OPEN_TEMP_DB - || eType==SQLITE_OPEN_MAIN_JOURNAL || eType==SQLITE_OPEN_TEMP_JOURNAL - || eType==SQLITE_OPEN_SUBJOURNAL || eType==SQLITE_OPEN_MASTER_JOURNAL + assert( eType==SQLITE_OPEN_MAIN_DB || eType==SQLITE_OPEN_TEMP_DB + || eType==SQLITE_OPEN_MAIN_JOURNAL || eType==SQLITE_OPEN_TEMP_JOURNAL + || eType==SQLITE_OPEN_SUBJOURNAL || eType==SQLITE_OPEN_MASTER_JOURNAL || eType==SQLITE_OPEN_TRANSIENT_DB || eType==SQLITE_OPEN_WAL ); @@ -4562,8 +4615,8 @@ static int winOpen( } #endif - /* If the second argument to this function is NULL, generate a - ** temporary file name to use + /* If the second argument to this function is NULL, generate a + ** temporary file name to use */ if( !zUtf8Name ){ assert( isDelete && !isOpenJournal ); @@ -4603,8 +4656,8 @@ static int winOpen( dwDesiredAccess = GENERIC_READ; } - /* SQLITE_OPEN_EXCLUSIVE is used to make sure that a new file is - ** created. SQLite doesn't use it to indicate "exclusive access" + /* SQLITE_OPEN_EXCLUSIVE is used to make sure that a new file is + ** created. SQLite doesn't use it to indicate "exclusive access" ** as it is usually understood. */ if( isExclusive ){ @@ -4693,7 +4746,7 @@ static int winOpen( sqlite3_free(zConverted); sqlite3_free(zTmpname); if( isReadWrite && !isExclusive ){ - return winOpen(pVfs, zName, id, + return winOpen(pVfs, zName, id, ((flags|SQLITE_OPEN_READONLY) & ~(SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE)), pOutFlags); @@ -4902,14 +4955,14 @@ static int winAccess( WIN32_FILE_ATTRIBUTE_DATA sAttrData; memset(&sAttrData, 0, sizeof(sAttrData)); while( !(rc = osGetFileAttributesExW((LPCWSTR)zConverted, - GetFileExInfoStandard, + GetFileExInfoStandard, &sAttrData)) && winRetryIoerr(&cnt, &lastErrno) ){} if( rc ){ /* For an SQLITE_ACCESS_EXISTS query, treat a zero-length file ** as if it does not exist. */ if( flags==SQLITE_ACCESS_EXISTS - && sAttrData.nFileSizeHigh==0 + && sAttrData.nFileSizeHigh==0 && sAttrData.nFileSizeLow==0 ){ attr = INVALID_FILE_ATTRIBUTES; }else{ @@ -5008,7 +5061,7 @@ static int winFullPathname( int nFull, /* Size of output buffer in bytes */ char *zFull /* Output buffer */ ){ - + #if defined(__CYGWIN__) SimulateIOError( return SQLITE_ERROR ); UNUSED_PARAMETER(nFull); @@ -5321,12 +5374,12 @@ int sqlite3_current_time = 0; /* Fake system time in seconds since 1970. */ ** epoch of noon in Greenwich on November 24, 4714 B.C according to the ** proleptic Gregorian calendar. ** -** On success, return SQLITE_OK. Return SQLITE_ERROR if the time and date +** On success, return SQLITE_OK. Return SQLITE_ERROR if the time and date ** cannot be found. */ static int winCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *piNow){ - /* FILETIME structure is a 64-bit value representing the number of - 100-nanosecond intervals since January 1, 1601 (= JD 2305813.5). + /* FILETIME structure is a 64-bit value representing the number of + 100-nanosecond intervals since January 1, 1601 (= JD 2305813.5). */ FILETIME ft; static const sqlite3_int64 winFiletimeEpoch = 23058135*(sqlite3_int64)8640000; @@ -5334,7 +5387,7 @@ static int winCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *piNow){ static const sqlite3_int64 unixEpoch = 24405875*(sqlite3_int64)8640000; #endif /* 2^32 - to avoid use of LL and warnings in gcc */ - static const sqlite3_int64 max32BitValue = + static const sqlite3_int64 max32BitValue = (sqlite3_int64)2000000000 + (sqlite3_int64)2000000000 + (sqlite3_int64)294967296; @@ -5350,7 +5403,7 @@ static int winCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *piNow){ #endif *piNow = winFiletimeEpoch + - ((((sqlite3_int64)ft.dwHighDateTime)*max32BitValue) + + ((((sqlite3_int64)ft.dwHighDateTime)*max32BitValue) + (sqlite3_int64)ft.dwLowDateTime)/(sqlite3_int64)10000; #ifdef SQLITE_TEST @@ -5469,7 +5522,7 @@ int sqlite3_os_init(void){ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==76 ); + assert( ArraySize(aSyscall)==77 ); /* get memory map allocation granularity */ memset(&winSysInfo, 0, sizeof(SYSTEM_INFO)); @@ -5487,10 +5540,10 @@ int sqlite3_os_init(void){ sqlite3_vfs_register(&winLongPathVfs, 0); #endif - return SQLITE_OK; + return SQLITE_OK; } -int sqlite3_os_end(void){ +int sqlite3_os_end(void){ #if SQLITE_OS_WINRT if( sleepObj!=NULL ){ osCloseHandle(sleepObj); diff --git a/src/os_win.h b/src/os_win.h index d662cd467d..5174ac7781 100644 --- a/src/os_win.h +++ b/src/os_win.h @@ -64,4 +64,14 @@ # define SQLITE_OS_WINRT 0 #endif +/* +** For WinCE, some API function parameters do not appear to be declared as +** volatile. +*/ +#if SQLITE_OS_WINCE +# define SQLITE_WIN32_VOLATILE +#else +# define SQLITE_WIN32_VOLATILE volatile +#endif + #endif /* _OS_WIN_H_ */ diff --git a/src/pager.c b/src/pager.c index a77dcecc4c..2230174e5c 100644 --- a/src/pager.c +++ b/src/pager.c @@ -76,12 +76,12 @@ ** Definition: Two databases (or the same database at two points it time) ** are said to be "logically equivalent" if they give the same answer to ** all queries. Note in particular the content of freelist leaf -** pages can be changed arbitarily without effecting the logical equivalence +** pages can be changed arbitrarily without affecting the logical equivalence ** of the database. ** ** (7) At any time, if any subset, including the empty set and the total set, ** of the unsynced changes to a rollback journal are removed and the -** journal is rolled back, the resulting database file will be logical +** journal is rolled back, the resulting database file will be logically ** equivalent to the database file at the beginning of the transaction. ** ** (8) When a transaction is rolled back, the xTruncate method of the VFS @@ -378,7 +378,7 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** ** The exception is when the database file is unlocked as the pager moves ** from ERROR to OPEN state. At this point there may be a hot-journal file -** in the file-system that needs to be rolled back (as part of a OPEN->SHARED +** in the file-system that needs to be rolled back (as part of an OPEN->SHARED ** transition, by the same pager or any other). If the call to xUnlock() ** fails at this point and the pager is left holding an EXCLUSIVE lock, this ** can confuse the call to xCheckReservedLock() call made later as part @@ -461,7 +461,7 @@ struct PagerSavepoint { #define SPILLFLAG_NOSYNC 0x04 /* Spill is ok, but do not sync */ /* -** A open page cache is an instance of struct Pager. A description of +** An open page cache is an instance of struct Pager. A description of ** some of the more important member variables follows: ** ** eState @@ -633,7 +633,7 @@ struct Pager { /************************************************************************** ** The following block contains those class members that change during - ** routine opertion. Class members not in this block are either fixed + ** routine operation. Class members not in this block are either fixed ** when the pager is first created or else only change when there is a ** significant mode change (such as changing the page_size, locking_mode, ** or the journal_mode). From another view, these class members describe @@ -646,6 +646,8 @@ struct Pager { u8 setMaster; /* True if a m-j name has been written to jrnl */ u8 doNotSpill; /* Do not spill the cache when non-zero */ u8 subjInMemory; /* True to use in-memory sub-journals */ + u8 bUseFetch; /* True to use xFetch() */ + u8 hasBeenUsed; /* True if any content previously read from this pager*/ Pgno dbSize; /* Number of pages in the database */ Pgno dbOrigSize; /* dbSize before the current transaction */ Pgno dbFileSize; /* Number of pages in the database file */ @@ -663,9 +665,9 @@ struct Pager { sqlite3_backup *pBackup; /* Pointer to list of ongoing backup processes */ PagerSavepoint *aSavepoint; /* Array of active savepoints */ int nSavepoint; /* Number of elements in aSavepoint[] */ + u32 iDataVersion; /* Changes whenever database content changes */ char dbFileVers[16]; /* Changes whenever database file changes */ - u8 bUseFetch; /* True to use xFetch() */ int nMmapOut; /* Number of mmap pages currently outstanding */ sqlite3_int64 szMmap; /* Desired maximum mmap size */ PgHdr *pMmapFreelist; /* List of free mmap page headers (pDirty) */ @@ -1677,29 +1679,23 @@ static int writeMasterJournal(Pager *pPager, const char *zMaster){ return rc; } -/* -** Find a page in the hash table given its page number. Return -** a pointer to the page or NULL if the requested page is not -** already in memory. -*/ -static PgHdr *pager_lookup(Pager *pPager, Pgno pgno){ - PgHdr *p = 0; /* Return value */ - - /* It is not possible for a call to PcacheFetch() with createFlag==0 to - ** fail, since no attempt to allocate dynamic memory will be made. - */ - (void)sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &p); - return p; -} - /* ** Discard the entire contents of the in-memory page-cache. */ static void pager_reset(Pager *pPager){ + pPager->iDataVersion++; sqlite3BackupRestart(pPager->pBackup); sqlite3PcacheClear(pPager->pPCache); } +/* +** Return the pPager->iDataVersion value +*/ +u32 sqlite3PagerDataVersion(Pager *pPager){ + assert( pPager->eState>PAGER_OPEN ); + return pPager->iDataVersion; +} + /* ** Free all structures in the Pager.aSavepoint[] array and set both ** Pager.aSavepoint and Pager.nSavepoint to zero. Close the sub-journal @@ -1956,6 +1952,14 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ rc = SQLITE_OK; }else{ rc = sqlite3OsTruncate(pPager->jfd, 0); + if( rc==SQLITE_OK && pPager->fullSync ){ + /* Make sure the new file size is written into the inode right away. + ** Otherwise the journal might resurrect following a power loss and + ** cause the last transaction to roll back. See + ** https://bugzilla.mozilla.org/show_bug.cgi?id=1072773 + */ + rc = sqlite3OsSync(pPager->jfd, pPager->syncFlags); + } } pPager->journalOff = 0; }else if( pPager->journalMode==PAGER_JOURNALMODE_PERSIST @@ -1984,7 +1988,7 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ #ifdef SQLITE_CHECK_PAGES sqlite3PcacheIterateDirty(pPager->pPCache, pager_set_pagehash); if( pPager->dbSize==0 && sqlite3PcacheRefCount(pPager->pPCache)>0 ){ - PgHdr *p = pager_lookup(pPager, 1); + PgHdr *p = sqlite3PagerLookup(pPager, 1); if( p ){ p->pageHash = 0; sqlite3PagerUnrefNotNull(p); @@ -2263,7 +2267,7 @@ static int pager_playback_one_page( if( pagerUseWal(pPager) ){ pPg = 0; }else{ - pPg = pager_lookup(pPager, pgno); + pPg = sqlite3PagerLookup(pPager, pgno); } assert( pPg || !MEMDB ); assert( pPager->eState!=PAGER_OPEN || pPg==0 ); @@ -2443,7 +2447,7 @@ static int pager_delmaster(Pager *pPager, const char *zMaster){ rc = sqlite3OsFileSize(pMaster, &nMasterJournal); if( rc!=SQLITE_OK ) goto delmaster_out; nMasterPtr = pVfs->mxPathname+1; - zMasterJournal = sqlite3Malloc((int)nMasterJournal + nMasterPtr + 1); + zMasterJournal = sqlite3Malloc(nMasterJournal + nMasterPtr + 1); if( !zMasterJournal ){ rc = SQLITE_NOMEM; goto delmaster_out; @@ -2512,7 +2516,7 @@ delmaster_out: ** If the file on disk is currently larger than nPage pages, then use the VFS ** xTruncate() method to truncate it. ** -** Or, it might might be the case that the file on disk is smaller than +** Or, it might be the case that the file on disk is smaller than ** nPage pages. Some operating system implementations can get confused if ** you try to truncate a file to some size that is larger than it ** currently is, so detect this case and write a single zero byte to @@ -2571,7 +2575,7 @@ int sqlite3SectorSize(sqlite3_file *pFile){ /* ** Set the value of the Pager.sectorSize variable for the given ** pager based on the value returned by the xSectorSize method -** of the open database file. The sector size will be used used +** of the open database file. The sector size will be used ** to determine the size and alignment of journal header and ** master journal pointers within created journal files. ** @@ -2906,7 +2910,7 @@ static int readDbPage(PgHdr *pPg, u32 iFrame){ ** ** For an encrypted database, the situation is more complex: bytes ** 24..39 of the database are white noise. But the probability of - ** white noising equaling 16 bytes of 0xff is vanishingly small so + ** white noise equaling 16 bytes of 0xff is vanishingly small so ** we should still be ok. */ memset(pPager->dbFileVers, 0xff, sizeof(pPager->dbFileVers)); @@ -3633,11 +3637,15 @@ int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nReserve){ if( rc==SQLITE_OK ){ pager_reset(pPager); - pPager->dbSize = (Pgno)((nByte+pageSize-1)/pageSize); - pPager->pageSize = pageSize; + rc = sqlite3PcacheSetPageSize(pPager->pPCache, pageSize); + } + if( rc==SQLITE_OK ){ sqlite3PageFree(pPager->pTmpSpace); pPager->pTmpSpace = pNew; - sqlite3PcacheSetPageSize(pPager->pPCache, pageSize); + pPager->dbSize = (Pgno)((nByte+pageSize-1)/pageSize); + pPager->pageSize = pageSize; + }else{ + sqlite3PageFree(pNew); } } @@ -3771,7 +3779,7 @@ static int pager_wait_on_lock(Pager *pPager, int locktype){ int rc; /* Return code */ /* Check that this is either a no-op (because the requested lock is - ** already held, or one of the transistions that the busy-handler + ** already held), or one of the transitions that the busy-handler ** may be invoked during, according to the comment above ** sqlite3PagerSetBusyhandler(). */ @@ -3890,7 +3898,7 @@ static int pagerAcquireMapPage( PgHdr **ppPage /* OUT: Acquired page object */ ){ PgHdr *p; /* Memory mapped page to return */ - + if( pPager->pMmapFreelist ){ *ppPage = p = pPager->pMmapFreelist; pPager->pMmapFreelist = p->pDirty; @@ -4399,8 +4407,8 @@ static int pagerStress(void *p, PgHdr *pPg){ ** a rollback or by user request, respectively. ** ** Spilling is also prohibited when in an error state since that could - ** lead to database corruption. In the current implementaton it - ** is impossible for sqlite3PcacheFetch() to be called with createFlag==1 + ** lead to database corruption. In the current implementation it + ** is impossible for sqlite3PcacheFetch() to be called with createFlag==3 ** while in the error state, hence it is impossible for this routine to ** be called in the error state. Nevertheless, we include a NEVER() ** test for the error state as a safeguard against future changes. @@ -4736,22 +4744,23 @@ act_like_temp_file: testcase( rc!=SQLITE_OK ); } - /* If an error occurred in either of the blocks above, free the - ** Pager structure and close the file. + /* Initialize the PCache object. */ + if( rc==SQLITE_OK ){ + assert( nExtra<1000 ); + nExtra = ROUND8(nExtra); + rc = sqlite3PcacheOpen(szPageDflt, nExtra, !memDb, + !memDb?pagerStress:0, (void *)pPager, pPager->pPCache); + } + + /* If an error occurred above, free the Pager structure and close the file. */ if( rc!=SQLITE_OK ){ - assert( !pPager->pTmpSpace ); sqlite3OsClose(pPager->fd); + sqlite3PageFree(pPager->pTmpSpace); sqlite3_free(pPager); return rc; } - /* Initialize the PCache object. */ - assert( nExtra<1000 ); - nExtra = ROUND8(nExtra); - sqlite3PcacheOpen(szPageDflt, nExtra, !memDb, - !memDb?pagerStress:0, (void *)pPager, pPager->pPCache); - PAGERTRACE(("OPEN %d %s\n", FILEHANDLEID(pPager->fd), pPager->zFilename)); IOTRACE(("OPEN %p %s\n", pPager, pPager->zFilename)) @@ -4938,7 +4947,7 @@ static int hasHotJournal(Pager *pPager, int *pExists){ *pExists = (first!=0); }else if( rc==SQLITE_CANTOPEN ){ /* If we cannot open the rollback journal file in order to see if - ** its has a zero header, that might be due to an I/O error, or + ** it has a zero header, that might be due to an I/O error, or ** it might be due to the race condition described above and in ** ticket #3883. Either way, assume that the journal is hot. ** This might be a false positive. But if it is, then the @@ -5120,16 +5129,12 @@ int sqlite3PagerSharedLock(Pager *pPager){ ); } - if( !pPager->tempFile && ( - pPager->pBackup - || sqlite3PcachePagecount(pPager->pPCache)>0 - || USEFETCH(pPager) - )){ - /* The shared-lock has just been acquired on the database file - ** and there are already pages in the cache (from a previous - ** read or write transaction). Check to see if the database - ** has been modified. If the database has changed, flush the - ** cache. + if( !pPager->tempFile && pPager->hasBeenUsed ){ + /* The shared-lock has just been acquired then check to + ** see if the database has been modified. If the database has changed, + ** flush the cache. The pPager->hasBeenUsed flag prevents this from + ** occurring on the very first access to a file, in order to save a + ** single unnecessary sqlite3OsRead() call at the start-up. ** ** Database changes is detected by looking at 15 bytes beginning ** at offset 24 into the file. The first 4 of these 16 bytes are @@ -5294,13 +5299,13 @@ int sqlite3PagerAcquire( if( pgno==0 ){ return SQLITE_CORRUPT_BKPT; } + pPager->hasBeenUsed = 1; /* If the pager is in the error state, return an error immediately. ** Otherwise, request the page from the PCache layer. */ if( pPager->errCode!=SQLITE_OK ){ rc = pPager->errCode; }else{ - if( bMmapOk && pagerUseWal(pPager) ){ rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iFrame); if( rc!=SQLITE_OK ) goto pager_acquire_err; @@ -5315,7 +5320,7 @@ int sqlite3PagerAcquire( if( rc==SQLITE_OK && pData ){ if( pPager->eState>PAGER_READER ){ - (void)sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &pPg); + pPg = sqlite3PagerLookup(pPager, pgno); } if( pPg==0 ){ rc = pagerAcquireMapPage(pPager, pgno, pData, &pPg); @@ -5333,7 +5338,16 @@ int sqlite3PagerAcquire( } } - rc = sqlite3PcacheFetch(pPager->pPCache, pgno, 1, ppPage); + { + sqlite3_pcache_page *pBase; + pBase = sqlite3PcacheFetch(pPager->pPCache, pgno, 3); + if( pBase==0 ){ + rc = sqlite3PcacheFetchStress(pPager->pPCache, pgno, &pBase); + if( rc!=SQLITE_OK ) goto pager_acquire_err; + } + pPg = *ppPage = sqlite3PcacheFetchFinish(pPager->pPCache, pgno, pBase); + if( pPg==0 ) rc = SQLITE_NOMEM; + } } if( rc!=SQLITE_OK ){ @@ -5430,13 +5444,13 @@ pager_acquire_err: ** has ever happened. */ DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ - PgHdr *pPg = 0; + sqlite3_pcache_page *pPage; assert( pPager!=0 ); assert( pgno!=0 ); assert( pPager->pPCache!=0 ); - assert( pPager->eState>=PAGER_READER && pPager->eState!=PAGER_ERROR ); - sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &pPg); - return pPg; + pPage = sqlite3PcacheFetch(pPager->pPCache, pgno, 0); + assert( pPage==0 || pPager->hasBeenUsed ); + return sqlite3PcacheFetchFinish(pPager->pPCache, pgno, pPage); } /* @@ -5772,6 +5786,97 @@ static int pager_write(PgHdr *pPg){ return rc; } +/* +** This is a variant of sqlite3PagerWrite() that runs when the sector size +** is larger than the page size. SQLite makes the (reasonable) assumption that +** all bytes of a sector are written together by hardware. Hence, all bytes of +** a sector need to be journalled in case of a power loss in the middle of +** a write. +** +** Usually, the sector size is less than or equal to the page size, in which +** case pages can be individually written. This routine only runs in the exceptional +** case where the page size is smaller than the sector size. +*/ +static SQLITE_NOINLINE int pagerWriteLargeSector(PgHdr *pPg){ + int rc = SQLITE_OK; /* Return code */ + Pgno nPageCount; /* Total number of pages in database file */ + Pgno pg1; /* First page of the sector pPg is located on. */ + int nPage = 0; /* Number of pages starting at pg1 to journal */ + int ii; /* Loop counter */ + int needSync = 0; /* True if any page has PGHDR_NEED_SYNC */ + Pager *pPager = pPg->pPager; /* The pager that owns pPg */ + Pgno nPagePerSector = (pPager->sectorSize/pPager->pageSize); + + /* Set the doNotSpill NOSYNC bit to 1. This is because we cannot allow + ** a journal header to be written between the pages journaled by + ** this function. + */ + assert( !MEMDB ); + assert( (pPager->doNotSpill & SPILLFLAG_NOSYNC)==0 ); + pPager->doNotSpill |= SPILLFLAG_NOSYNC; + + /* This trick assumes that both the page-size and sector-size are + ** an integer power of 2. It sets variable pg1 to the identifier + ** of the first page of the sector pPg is located on. + */ + pg1 = ((pPg->pgno-1) & ~(nPagePerSector-1)) + 1; + + nPageCount = pPager->dbSize; + if( pPg->pgno>nPageCount ){ + nPage = (pPg->pgno - pg1)+1; + }else if( (pg1+nPagePerSector-1)>nPageCount ){ + nPage = nPageCount+1-pg1; + }else{ + nPage = nPagePerSector; + } + assert(nPage>0); + assert(pg1<=pPg->pgno); + assert((pg1+nPage)>pPg->pgno); + + for(ii=0; iipgno || !sqlite3BitvecTest(pPager->pInJournal, pg) ){ + if( pg!=PAGER_MJ_PGNO(pPager) ){ + rc = sqlite3PagerGet(pPager, pg, &pPage); + if( rc==SQLITE_OK ){ + rc = pager_write(pPage); + if( pPage->flags&PGHDR_NEED_SYNC ){ + needSync = 1; + } + sqlite3PagerUnrefNotNull(pPage); + } + } + }else if( (pPage = sqlite3PagerLookup(pPager, pg))!=0 ){ + if( pPage->flags&PGHDR_NEED_SYNC ){ + needSync = 1; + } + sqlite3PagerUnrefNotNull(pPage); + } + } + + /* If the PGHDR_NEED_SYNC flag is set for any of the nPage pages + ** starting at pg1, then it needs to be set for all of them. Because + ** writing to any of these nPage pages may damage the others, the + ** journal file must contain sync()ed copies of all of them + ** before any of them can be written out to the database file. + */ + if( rc==SQLITE_OK && needSync ){ + assert( !MEMDB ); + for(ii=0; iiflags |= PGHDR_NEED_SYNC; + sqlite3PagerUnrefNotNull(pPage); + } + } + } + + assert( (pPager->doNotSpill & SPILLFLAG_NOSYNC)!=0 ); + pPager->doNotSpill &= ~SPILLFLAG_NOSYNC; + return rc; +} + /* ** Mark a data page as writeable. This routine must be called before ** making changes to a page. The caller must check the return value @@ -5786,96 +5891,16 @@ static int pager_write(PgHdr *pPg){ ** If an error occurs, SQLITE_NOMEM or an IO error code is returned ** as appropriate. Otherwise, SQLITE_OK. */ -int sqlite3PagerWrite(DbPage *pDbPage){ - int rc = SQLITE_OK; - - PgHdr *pPg = pDbPage; - Pager *pPager = pPg->pPager; - +int sqlite3PagerWrite(PgHdr *pPg){ assert( (pPg->flags & PGHDR_MMAP)==0 ); - assert( pPager->eState>=PAGER_WRITER_LOCKED ); - assert( pPager->eState!=PAGER_ERROR ); - assert( assert_pager_state(pPager) ); - - if( pPager->sectorSize > (u32)pPager->pageSize ){ - Pgno nPageCount; /* Total number of pages in database file */ - Pgno pg1; /* First page of the sector pPg is located on. */ - int nPage = 0; /* Number of pages starting at pg1 to journal */ - int ii; /* Loop counter */ - int needSync = 0; /* True if any page has PGHDR_NEED_SYNC */ - Pgno nPagePerSector = (pPager->sectorSize/pPager->pageSize); - - /* Set the doNotSpill NOSYNC bit to 1. This is because we cannot allow - ** a journal header to be written between the pages journaled by - ** this function. - */ - assert( !MEMDB ); - assert( (pPager->doNotSpill & SPILLFLAG_NOSYNC)==0 ); - pPager->doNotSpill |= SPILLFLAG_NOSYNC; - - /* This trick assumes that both the page-size and sector-size are - ** an integer power of 2. It sets variable pg1 to the identifier - ** of the first page of the sector pPg is located on. - */ - pg1 = ((pPg->pgno-1) & ~(nPagePerSector-1)) + 1; - - nPageCount = pPager->dbSize; - if( pPg->pgno>nPageCount ){ - nPage = (pPg->pgno - pg1)+1; - }else if( (pg1+nPagePerSector-1)>nPageCount ){ - nPage = nPageCount+1-pg1; - }else{ - nPage = nPagePerSector; - } - assert(nPage>0); - assert(pg1<=pPg->pgno); - assert((pg1+nPage)>pPg->pgno); - - for(ii=0; iipgno || !sqlite3BitvecTest(pPager->pInJournal, pg) ){ - if( pg!=PAGER_MJ_PGNO(pPager) ){ - rc = sqlite3PagerGet(pPager, pg, &pPage); - if( rc==SQLITE_OK ){ - rc = pager_write(pPage); - if( pPage->flags&PGHDR_NEED_SYNC ){ - needSync = 1; - } - sqlite3PagerUnrefNotNull(pPage); - } - } - }else if( (pPage = pager_lookup(pPager, pg))!=0 ){ - if( pPage->flags&PGHDR_NEED_SYNC ){ - needSync = 1; - } - sqlite3PagerUnrefNotNull(pPage); - } - } - - /* If the PGHDR_NEED_SYNC flag is set for any of the nPage pages - ** starting at pg1, then it needs to be set for all of them. Because - ** writing to any of these nPage pages may damage the others, the - ** journal file must contain sync()ed copies of all of them - ** before any of them can be written out to the database file. - */ - if( rc==SQLITE_OK && needSync ){ - assert( !MEMDB ); - for(ii=0; iiflags |= PGHDR_NEED_SYNC; - sqlite3PagerUnrefNotNull(pPage); - } - } - } - - assert( (pPager->doNotSpill & SPILLFLAG_NOSYNC)!=0 ); - pPager->doNotSpill &= ~SPILLFLAG_NOSYNC; + assert( pPg->pPager->eState>=PAGER_WRITER_LOCKED ); + assert( pPg->pPager->eState!=PAGER_ERROR ); + assert( assert_pager_state(pPg->pPager) ); + if( pPg->pPager->sectorSize > (u32)pPg->pPager->pageSize ){ + return pagerWriteLargeSector(pPg); }else{ - rc = pager_write(pDbPage); + return pager_write(pPg); } - return rc; } /* @@ -6291,6 +6316,7 @@ int sqlite3PagerCommitPhaseTwo(Pager *pPager){ } PAGERTRACE(("COMMIT %d\n", PAGERID(pPager))); + pPager->iDataVersion++; rc = pager_end_transaction(pPager, pPager->setMaster, 1); return pager_error(pPager, rc); } @@ -6771,7 +6797,7 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ ** for the page moved there. */ pPg->flags &= ~PGHDR_NEED_SYNC; - pPgOld = pager_lookup(pPager, pgno); + pPgOld = sqlite3PagerLookup(pPager, pgno); assert( !pPgOld || pPgOld->nRef==1 ); if( pPgOld ){ pPg->flags |= (pPgOld->flags&PGHDR_NEED_SYNC); @@ -6831,6 +6857,18 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ } #endif +/* +** The page handle passed as the first argument refers to a dirty page +** with a page number other than iNew. This function changes the page's +** page number to iNew and sets the value of the PgHdr.flags field to +** the value passed as the third parameter. +*/ +void sqlite3PagerRekey(DbPage *pPg, Pgno iNew, u16 flags){ + assert( pPg->pgno!=iNew ); + pPg->flags = flags; + sqlite3PcacheMove(pPg, iNew); +} + /* ** Return a pointer to the data for the specified page. */ @@ -7047,7 +7085,8 @@ int sqlite3PagerCheckpoint(Pager *pPager, int eMode, int *pnLog, int *pnCkpt){ int rc = SQLITE_OK; if( pPager->pWal ){ rc = sqlite3WalCheckpoint(pPager->pWal, eMode, - pPager->xBusyHandler, pPager->pBusyHandlerArg, + (eMode==SQLITE_CHECKPOINT_PASSIVE ? 0 : pPager->xBusyHandler), + pPager->pBusyHandlerArg, pPager->ckptSyncFlags, pPager->pageSize, (u8 *)pPager->pTmpSpace, pnLog, pnCkpt ); @@ -7224,9 +7263,10 @@ int sqlite3PagerCloseWal(Pager *pPager){ ** is empty, return 0. */ int sqlite3PagerWalFramesize(Pager *pPager){ - assert( pPager->eState==PAGER_READER ); + assert( pPager->eState>=PAGER_READER ); return sqlite3WalFramesize(pPager->pWal); } #endif + #endif /* SQLITE_OMIT_DISKIO */ diff --git a/src/pager.h b/src/pager.h index c9ca8553b9..e07fa88fc5 100644 --- a/src/pager.h +++ b/src/pager.h @@ -172,6 +172,7 @@ int sqlite3PagerSharedLock(Pager *pPager); /* Functions used to query pager state and configuration. */ u8 sqlite3PagerIsreadonly(Pager*); +u32 sqlite3PagerDataVersion(Pager*); int sqlite3PagerRefcount(Pager*); int sqlite3PagerMemUsed(Pager*); const char *sqlite3PagerFilename(Pager*, int); @@ -188,6 +189,8 @@ int sqlite3SectorSize(sqlite3_file *); /* Functions used to truncate the database file. */ void sqlite3PagerTruncateImage(Pager*,Pgno); +void sqlite3PagerRekey(DbPage*, Pgno, u16); + #if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL) void *sqlite3PagerCodec(DbPage *); #endif diff --git a/src/parse.y b/src/parse.y index dbc129ce63..877827e68d 100644 --- a/src/parse.y +++ b/src/parse.y @@ -399,9 +399,6 @@ cmd ::= DROP VIEW ifexists(E) fullname(X). { cmd ::= select(X). { SelectDest dest = {SRT_Output, 0, 0, 0, 0, 0}; sqlite3Select(pParse, X, &dest); - sqlite3ExplainBegin(pParse->pVdbe); - sqlite3ExplainSelect(pParse->pVdbe, X); - sqlite3ExplainFinish(pParse->pVdbe); sqlite3SelectDelete(pParse->db, X); } @@ -459,9 +456,33 @@ multiselect_op(A) ::= UNION(OP). {A = @OP;} multiselect_op(A) ::= UNION ALL. {A = TK_ALL;} multiselect_op(A) ::= EXCEPT|INTERSECT(OP). {A = @OP;} %endif SQLITE_OMIT_COMPOUND_SELECT -oneselect(A) ::= SELECT distinct(D) selcollist(W) from(X) where_opt(Y) +oneselect(A) ::= SELECT(S) distinct(D) selcollist(W) from(X) where_opt(Y) groupby_opt(P) having_opt(Q) orderby_opt(Z) limit_opt(L). { A = sqlite3SelectNew(pParse,W,X,Y,P,Q,Z,D,L.pLimit,L.pOffset); +#if SELECTTRACE_ENABLED + /* Populate the Select.zSelName[] string that is used to help with + ** query planner debugging, to differentiate between multiple Select + ** objects in a complex query. + ** + ** If the SELECT keyword is immediately followed by a C-style comment + ** then extract the first few alphanumeric characters from within that + ** comment to be the zSelName value. Otherwise, the label is #N where + ** is an integer that is incremented with each SELECT statement seen. + */ + if( A!=0 ){ + const char *z = S.z+6; + int i; + sqlite3_snprintf(sizeof(A->zSelName), A->zSelName, "#%d", + ++pParse->nSelect); + while( z[0]==' ' ) z++; + if( z[0]=='/' && z[1]=='*' ){ + z += 2; + while( z[0]==' ' ) z++; + for(i=0; sqlite3Isalnum(z[i]); i++){} + sqlite3_snprintf(sizeof(A->zSelName), A->zSelName, "%.*s", i, z); + } + } +#endif /* SELECTRACE_ENABLED */ } oneselect(A) ::= values(X). {A = X;} @@ -940,7 +961,7 @@ expr(A) ::= expr(X) NOT NULL(E). {spanUnaryPostfix(&A,pParse,TK_NOTNULL,&X,&E);} ** unary TK_ISNULL or TK_NOTNULL expression. */ static void binaryToUnaryIfNull(Parse *pParse, Expr *pY, Expr *pA, int op){ sqlite3 *db = pParse->db; - if( db->mallocFailed==0 && pY->op==TK_NULL ){ + if( pY && pA && pY->op==TK_NULL ){ pA->op = (u8)op; sqlite3ExprDelete(db, pA->pRight); pA->pRight = 0; diff --git a/src/pcache.c b/src/pcache.c index e18bf93be0..0194f63bb9 100644 --- a/src/pcache.c +++ b/src/pcache.c @@ -45,88 +45,71 @@ struct PCache { /********************************** Linked List Management ********************/ -#if !defined(NDEBUG) && defined(SQLITE_ENABLE_EXPENSIVE_ASSERT) -/* -** Check that the pCache->pSynced variable is set correctly. If it -** is not, either fail an assert or return zero. Otherwise, return -** non-zero. This is only used in debugging builds, as follows: -** -** expensive_assert( pcacheCheckSynced(pCache) ); -*/ -static int pcacheCheckSynced(PCache *pCache){ - PgHdr *p; - for(p=pCache->pDirtyTail; p!=pCache->pSynced; p=p->pDirtyPrev){ - assert( p->nRef || (p->flags&PGHDR_NEED_SYNC) ); - } - return (p==0 || p->nRef || (p->flags&PGHDR_NEED_SYNC)==0); -} -#endif /* !NDEBUG && SQLITE_ENABLE_EXPENSIVE_ASSERT */ +/* Allowed values for second argument to pcacheManageDirtyList() */ +#define PCACHE_DIRTYLIST_REMOVE 1 /* Remove pPage from dirty list */ +#define PCACHE_DIRTYLIST_ADD 2 /* Add pPage to the dirty list */ +#define PCACHE_DIRTYLIST_FRONT 3 /* Move pPage to the front of the list */ /* -** Remove page pPage from the list of dirty pages. +** Manage pPage's participation on the dirty list. Bits of the addRemove +** argument determines what operation to do. The 0x01 bit means first +** remove pPage from the dirty list. The 0x02 means add pPage back to +** the dirty list. Doing both moves pPage to the front of the dirty list. */ -static void pcacheRemoveFromDirtyList(PgHdr *pPage){ +static void pcacheManageDirtyList(PgHdr *pPage, u8 addRemove){ PCache *p = pPage->pCache; - assert( pPage->pDirtyNext || pPage==p->pDirtyTail ); - assert( pPage->pDirtyPrev || pPage==p->pDirty ); - - /* Update the PCache1.pSynced variable if necessary. */ - if( p->pSynced==pPage ){ - PgHdr *pSynced = pPage->pDirtyPrev; - while( pSynced && (pSynced->flags&PGHDR_NEED_SYNC) ){ - pSynced = pSynced->pDirtyPrev; + if( addRemove & PCACHE_DIRTYLIST_REMOVE ){ + assert( pPage->pDirtyNext || pPage==p->pDirtyTail ); + assert( pPage->pDirtyPrev || pPage==p->pDirty ); + + /* Update the PCache1.pSynced variable if necessary. */ + if( p->pSynced==pPage ){ + PgHdr *pSynced = pPage->pDirtyPrev; + while( pSynced && (pSynced->flags&PGHDR_NEED_SYNC) ){ + pSynced = pSynced->pDirtyPrev; + } + p->pSynced = pSynced; } - p->pSynced = pSynced; + + if( pPage->pDirtyNext ){ + pPage->pDirtyNext->pDirtyPrev = pPage->pDirtyPrev; + }else{ + assert( pPage==p->pDirtyTail ); + p->pDirtyTail = pPage->pDirtyPrev; + } + if( pPage->pDirtyPrev ){ + pPage->pDirtyPrev->pDirtyNext = pPage->pDirtyNext; + }else{ + assert( pPage==p->pDirty ); + p->pDirty = pPage->pDirtyNext; + if( p->pDirty==0 && p->bPurgeable ){ + assert( p->eCreate==1 ); + p->eCreate = 2; + } + } + pPage->pDirtyNext = 0; + pPage->pDirtyPrev = 0; } - - if( pPage->pDirtyNext ){ - pPage->pDirtyNext->pDirtyPrev = pPage->pDirtyPrev; - }else{ - assert( pPage==p->pDirtyTail ); - p->pDirtyTail = pPage->pDirtyPrev; - } - if( pPage->pDirtyPrev ){ - pPage->pDirtyPrev->pDirtyNext = pPage->pDirtyNext; - }else{ - assert( pPage==p->pDirty ); - p->pDirty = pPage->pDirtyNext; - if( p->pDirty==0 && p->bPurgeable ){ - assert( p->eCreate==1 ); - p->eCreate = 2; + if( addRemove & PCACHE_DIRTYLIST_ADD ){ + assert( pPage->pDirtyNext==0 && pPage->pDirtyPrev==0 && p->pDirty!=pPage ); + + pPage->pDirtyNext = p->pDirty; + if( pPage->pDirtyNext ){ + assert( pPage->pDirtyNext->pDirtyPrev==0 ); + pPage->pDirtyNext->pDirtyPrev = pPage; + }else{ + p->pDirtyTail = pPage; + if( p->bPurgeable ){ + assert( p->eCreate==2 ); + p->eCreate = 1; + } + } + p->pDirty = pPage; + if( !p->pSynced && 0==(pPage->flags&PGHDR_NEED_SYNC) ){ + p->pSynced = pPage; } } - pPage->pDirtyNext = 0; - pPage->pDirtyPrev = 0; - - expensive_assert( pcacheCheckSynced(p) ); -} - -/* -** Add page pPage to the head of the dirty list (PCache1.pDirty is set to -** pPage). -*/ -static void pcacheAddToDirtyList(PgHdr *pPage){ - PCache *p = pPage->pCache; - - assert( pPage->pDirtyNext==0 && pPage->pDirtyPrev==0 && p->pDirty!=pPage ); - - pPage->pDirtyNext = p->pDirty; - if( pPage->pDirtyNext ){ - assert( pPage->pDirtyNext->pDirtyPrev==0 ); - pPage->pDirtyNext->pDirtyPrev = pPage; - }else if( p->bPurgeable ){ - assert( p->eCreate==2 ); - p->eCreate = 1; - } - p->pDirty = pPage; - if( !p->pDirtyTail ){ - p->pDirtyTail = pPage; - } - if( !p->pSynced && 0==(pPage->flags&PGHDR_NEED_SYNC) ){ - p->pSynced = pPage; - } - expensive_assert( pcacheCheckSynced(p) ); } /* @@ -134,12 +117,22 @@ static void pcacheAddToDirtyList(PgHdr *pPage){ ** being used for an in-memory database, this function is a no-op. */ static void pcacheUnpin(PgHdr *p){ - PCache *pCache = p->pCache; - if( pCache->bPurgeable ){ + if( p->pCache->bPurgeable ){ if( p->pgno==1 ){ - pCache->pPage1 = 0; + p->pCache->pPage1 = 0; } - sqlite3GlobalConfig.pcache2.xUnpin(pCache->pCache, p->pPage, 0); + sqlite3GlobalConfig.pcache2.xUnpin(p->pCache->pCache, p->pPage, 0); + } +} + +/* +** Compute the number of pages of cache requested. +*/ +static int numberOfCachePages(PCache *p){ + if( p->szCache>=0 ){ + return p->szCache; + }else{ + return (int)((-1024*(i64)p->szCache)/(p->szPage+p->szExtra)); } } @@ -175,7 +168,7 @@ int sqlite3PcacheSize(void){ return sizeof(PCache); } ** The caller discovers how much space needs to be allocated by ** calling sqlite3PcacheSize(). */ -void sqlite3PcacheOpen( +int sqlite3PcacheOpen( int szPage, /* Size of every page */ int szExtra, /* Extra space associated with each page */ int bPurgeable, /* True if pages are on backing store */ @@ -184,76 +177,76 @@ void sqlite3PcacheOpen( PCache *p /* Preallocated space for the PCache */ ){ memset(p, 0, sizeof(PCache)); - p->szPage = szPage; + p->szPage = 1; p->szExtra = szExtra; p->bPurgeable = bPurgeable; p->eCreate = 2; p->xStress = xStress; p->pStress = pStress; p->szCache = 100; + return sqlite3PcacheSetPageSize(p, szPage); } /* ** Change the page size for PCache object. The caller must ensure that there ** are no outstanding page references when this function is called. */ -void sqlite3PcacheSetPageSize(PCache *pCache, int szPage){ +int sqlite3PcacheSetPageSize(PCache *pCache, int szPage){ assert( pCache->nRef==0 && pCache->pDirty==0 ); - if( pCache->pCache ){ - sqlite3GlobalConfig.pcache2.xDestroy(pCache->pCache); - pCache->pCache = 0; + if( pCache->szPage ){ + sqlite3_pcache *pNew; + pNew = sqlite3GlobalConfig.pcache2.xCreate( + szPage, pCache->szExtra + ROUND8(sizeof(PgHdr)), + pCache->bPurgeable + ); + if( pNew==0 ) return SQLITE_NOMEM; + sqlite3GlobalConfig.pcache2.xCachesize(pNew, numberOfCachePages(pCache)); + if( pCache->pCache ){ + sqlite3GlobalConfig.pcache2.xDestroy(pCache->pCache); + } + pCache->pCache = pNew; pCache->pPage1 = 0; + pCache->szPage = szPage; } - pCache->szPage = szPage; -} - -/* -** Compute the number of pages of cache requested. -*/ -static int numberOfCachePages(PCache *p){ - if( p->szCache>=0 ){ - return p->szCache; - }else{ - return (int)((-1024*(i64)p->szCache)/(p->szPage+p->szExtra)); - } + return SQLITE_OK; } /* ** Try to obtain a page from the cache. +** +** This routine returns a pointer to an sqlite3_pcache_page object if +** such an object is already in cache, or if a new one is created. +** This routine returns a NULL pointer if the object was not in cache +** and could not be created. +** +** The createFlags should be 0 to check for existing pages and should +** be 3 (not 1, but 3) to try to create a new page. +** +** If the createFlag is 0, then NULL is always returned if the page +** is not already in the cache. If createFlag is 1, then a new page +** is created only if that can be done without spilling dirty pages +** and without exceeding the cache size limit. +** +** The caller needs to invoke sqlite3PcacheFetchFinish() to properly +** initialize the sqlite3_pcache_page object and convert it into a +** PgHdr object. The sqlite3PcacheFetch() and sqlite3PcacheFetchFinish() +** routines are split this way for performance reasons. When separated +** they can both (usually) operate without having to push values to +** the stack on entry and pop them back off on exit, which saves a +** lot of pushing and popping. */ -int sqlite3PcacheFetch( +sqlite3_pcache_page *sqlite3PcacheFetch( PCache *pCache, /* Obtain the page from this cache */ Pgno pgno, /* Page number to obtain */ - int createFlag, /* If true, create page if it does not exist already */ - PgHdr **ppPage /* Write the page here */ + int createFlag /* If true, create page if it does not exist already */ ){ - sqlite3_pcache_page *pPage; - PgHdr *pPgHdr = 0; int eCreate; assert( pCache!=0 ); - assert( createFlag==1 || createFlag==0 ); + assert( pCache->pCache!=0 ); + assert( createFlag==3 || createFlag==0 ); assert( pgno>0 ); - /* If the pluggable cache (sqlite3_pcache*) has not been allocated, - ** allocate it now. - */ - if( !pCache->pCache ){ - sqlite3_pcache *p; - if( !createFlag ){ - *ppPage = 0; - return SQLITE_OK; - } - p = sqlite3GlobalConfig.pcache2.xCreate( - pCache->szPage, pCache->szExtra + sizeof(PgHdr), pCache->bPurgeable - ); - if( !p ){ - return SQLITE_NOMEM; - } - sqlite3GlobalConfig.pcache2.xCachesize(p, numberOfCachePages(pCache)); - pCache->pCache = p; - } - /* eCreate defines what to do if the page does not exist. ** 0 Do not allocate a new page. (createFlag==0) ** 1 Allocate a new page if doing so is inexpensive. @@ -261,89 +254,135 @@ int sqlite3PcacheFetch( ** 2 Allocate a new page even it doing so is difficult. ** (createFlag==1 AND !(bPurgeable AND pDirty) */ - eCreate = createFlag==0 ? 0 : pCache->eCreate; - assert( (createFlag*(1+(!pCache->bPurgeable||!pCache->pDirty)))==eCreate ); - pPage = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, eCreate); - if( !pPage && eCreate==1 ){ - PgHdr *pPg; + eCreate = createFlag & pCache->eCreate; + assert( eCreate==0 || eCreate==1 || eCreate==2 ); + assert( createFlag==0 || pCache->eCreate==eCreate ); + assert( createFlag==0 || eCreate==1+(!pCache->bPurgeable||!pCache->pDirty) ); + return sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, eCreate); +} - /* Find a dirty page to write-out and recycle. First try to find a - ** page that does not require a journal-sync (one with PGHDR_NEED_SYNC - ** cleared), but if that is not possible settle for any other - ** unreferenced dirty page. - */ - expensive_assert( pcacheCheckSynced(pCache) ); - for(pPg=pCache->pSynced; - pPg && (pPg->nRef || (pPg->flags&PGHDR_NEED_SYNC)); - pPg=pPg->pDirtyPrev - ); - pCache->pSynced = pPg; - if( !pPg ){ - for(pPg=pCache->pDirtyTail; pPg && pPg->nRef; pPg=pPg->pDirtyPrev); - } - if( pPg ){ - int rc; +/* +** If the sqlite3PcacheFetch() routine is unable to allocate a new +** page because new clean pages are available for reuse and the cache +** size limit has been reached, then this routine can be invoked to +** try harder to allocate a page. This routine might invoke the stress +** callback to spill dirty pages to the journal. It will then try to +** allocate the new page and will only fail to allocate a new page on +** an OOM error. +** +** This routine should be invoked only after sqlite3PcacheFetch() fails. +*/ +int sqlite3PcacheFetchStress( + PCache *pCache, /* Obtain the page from this cache */ + Pgno pgno, /* Page number to obtain */ + sqlite3_pcache_page **ppPage /* Write result here */ +){ + PgHdr *pPg; + if( pCache->eCreate==2 ) return 0; + + + /* Find a dirty page to write-out and recycle. First try to find a + ** page that does not require a journal-sync (one with PGHDR_NEED_SYNC + ** cleared), but if that is not possible settle for any other + ** unreferenced dirty page. + */ + for(pPg=pCache->pSynced; + pPg && (pPg->nRef || (pPg->flags&PGHDR_NEED_SYNC)); + pPg=pPg->pDirtyPrev + ); + pCache->pSynced = pPg; + if( !pPg ){ + for(pPg=pCache->pDirtyTail; pPg && pPg->nRef; pPg=pPg->pDirtyPrev); + } + if( pPg ){ + int rc; #ifdef SQLITE_LOG_CACHE_SPILL - sqlite3_log(SQLITE_FULL, - "spill page %d making room for %d - cache used: %d/%d", - pPg->pgno, pgno, - sqlite3GlobalConfig.pcache.xPagecount(pCache->pCache), - numberOfCachePages(pCache)); + sqlite3_log(SQLITE_FULL, + "spill page %d making room for %d - cache used: %d/%d", + pPg->pgno, pgno, + sqlite3GlobalConfig.pcache.xPagecount(pCache->pCache), + numberOfCachePages(pCache)); #endif - rc = pCache->xStress(pCache->pStress, pPg); - if( rc!=SQLITE_OK && rc!=SQLITE_BUSY ){ - return rc; - } - } - - pPage = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, 2); - } - - if( pPage ){ - pPgHdr = (PgHdr *)pPage->pExtra; - - if( !pPgHdr->pPage ){ - memset(pPgHdr, 0, sizeof(PgHdr)); - pPgHdr->pPage = pPage; - pPgHdr->pData = pPage->pBuf; - pPgHdr->pExtra = (void *)&pPgHdr[1]; - memset(pPgHdr->pExtra, 0, pCache->szExtra); - pPgHdr->pCache = pCache; - pPgHdr->pgno = pgno; - } - assert( pPgHdr->pCache==pCache ); - assert( pPgHdr->pgno==pgno ); - assert( pPgHdr->pData==pPage->pBuf ); - assert( pPgHdr->pExtra==(void *)&pPgHdr[1] ); - - if( 0==pPgHdr->nRef ){ - pCache->nRef++; - } - pPgHdr->nRef++; - if( pgno==1 ){ - pCache->pPage1 = pPgHdr; + rc = pCache->xStress(pCache->pStress, pPg); + if( rc!=SQLITE_OK && rc!=SQLITE_BUSY ){ + return rc; } } - *ppPage = pPgHdr; - return (pPgHdr==0 && eCreate) ? SQLITE_NOMEM : SQLITE_OK; + *ppPage = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, 2); + return *ppPage==0 ? SQLITE_NOMEM : SQLITE_OK; +} + +/* +** This is a helper routine for sqlite3PcacheFetchFinish() +** +** In the uncommon case where the page being fetched has not been +** initialized, this routine is invoked to do the initialization. +** This routine is broken out into a separate function since it +** requires extra stack manipulation that can be avoided in the common +** case. +*/ +static SQLITE_NOINLINE PgHdr *pcacheFetchFinishWithInit( + PCache *pCache, /* Obtain the page from this cache */ + Pgno pgno, /* Page number obtained */ + sqlite3_pcache_page *pPage /* Page obtained by prior PcacheFetch() call */ +){ + PgHdr *pPgHdr; + assert( pPage!=0 ); + pPgHdr = (PgHdr*)pPage->pExtra; + assert( pPgHdr->pPage==0 ); + memset(pPgHdr, 0, sizeof(PgHdr)); + pPgHdr->pPage = pPage; + pPgHdr->pData = pPage->pBuf; + pPgHdr->pExtra = (void *)&pPgHdr[1]; + memset(pPgHdr->pExtra, 0, pCache->szExtra); + pPgHdr->pCache = pCache; + pPgHdr->pgno = pgno; + return sqlite3PcacheFetchFinish(pCache,pgno,pPage); +} + +/* +** This routine converts the sqlite3_pcache_page object returned by +** sqlite3PcacheFetch() into an initialized PgHdr object. This routine +** must be called after sqlite3PcacheFetch() in order to get a usable +** result. +*/ +PgHdr *sqlite3PcacheFetchFinish( + PCache *pCache, /* Obtain the page from this cache */ + Pgno pgno, /* Page number obtained */ + sqlite3_pcache_page *pPage /* Page obtained by prior PcacheFetch() call */ +){ + PgHdr *pPgHdr; + + if( pPage==0 ) return 0; + pPgHdr = (PgHdr *)pPage->pExtra; + + if( !pPgHdr->pPage ){ + return pcacheFetchFinishWithInit(pCache, pgno, pPage); + } + if( 0==pPgHdr->nRef ){ + pCache->nRef++; + } + pPgHdr->nRef++; + if( pgno==1 ){ + pCache->pPage1 = pPgHdr; + } + return pPgHdr; } /* ** Decrement the reference count on a page. If the page is clean and the -** reference count drops to 0, then it is made elible for recycling. +** reference count drops to 0, then it is made eligible for recycling. */ -void sqlite3PcacheRelease(PgHdr *p){ +void SQLITE_NOINLINE sqlite3PcacheRelease(PgHdr *p){ assert( p->nRef>0 ); p->nRef--; if( p->nRef==0 ){ - PCache *pCache = p->pCache; - pCache->nRef--; + p->pCache->nRef--; if( (p->flags&PGHDR_DIRTY)==0 ){ pcacheUnpin(p); - }else{ + }else if( p->pDirtyPrev!=0 ){ /* Move the page to the head of the dirty list. */ - pcacheRemoveFromDirtyList(p); - pcacheAddToDirtyList(p); + pcacheManageDirtyList(p, PCACHE_DIRTYLIST_FRONT); } } } @@ -362,17 +401,15 @@ void sqlite3PcacheRef(PgHdr *p){ ** page pointed to by p is invalid. */ void sqlite3PcacheDrop(PgHdr *p){ - PCache *pCache; assert( p->nRef==1 ); if( p->flags&PGHDR_DIRTY ){ - pcacheRemoveFromDirtyList(p); + pcacheManageDirtyList(p, PCACHE_DIRTYLIST_REMOVE); } - pCache = p->pCache; - pCache->nRef--; + p->pCache->nRef--; if( p->pgno==1 ){ - pCache->pPage1 = 0; + p->pCache->pPage1 = 0; } - sqlite3GlobalConfig.pcache2.xUnpin(pCache->pCache, p->pPage, 1); + sqlite3GlobalConfig.pcache2.xUnpin(p->pCache->pCache, p->pPage, 1); } /* @@ -384,7 +421,7 @@ void sqlite3PcacheMakeDirty(PgHdr *p){ assert( p->nRef>0 ); if( 0==(p->flags & PGHDR_DIRTY) ){ p->flags |= PGHDR_DIRTY; - pcacheAddToDirtyList( p); + pcacheManageDirtyList(p, PCACHE_DIRTYLIST_ADD); } } @@ -394,7 +431,7 @@ void sqlite3PcacheMakeDirty(PgHdr *p){ */ void sqlite3PcacheMakeClean(PgHdr *p){ if( (p->flags & PGHDR_DIRTY) ){ - pcacheRemoveFromDirtyList(p); + pcacheManageDirtyList(p, PCACHE_DIRTYLIST_REMOVE); p->flags &= ~(PGHDR_DIRTY|PGHDR_NEED_SYNC); if( p->nRef==0 ){ pcacheUnpin(p); @@ -433,8 +470,7 @@ void sqlite3PcacheMove(PgHdr *p, Pgno newPgno){ sqlite3GlobalConfig.pcache2.xRekey(pCache->pCache, p->pPage, p->pgno,newPgno); p->pgno = newPgno; if( (p->flags&PGHDR_DIRTY) && (p->flags&PGHDR_NEED_SYNC) ){ - pcacheRemoveFromDirtyList(p); - pcacheAddToDirtyList(p); + pcacheManageDirtyList(p, PCACHE_DIRTYLIST_FRONT); } } @@ -475,9 +511,8 @@ void sqlite3PcacheTruncate(PCache *pCache, Pgno pgno){ ** Close a cache. */ void sqlite3PcacheClose(PCache *pCache){ - if( pCache->pCache ){ - sqlite3GlobalConfig.pcache2.xDestroy(pCache->pCache); - } + assert( pCache->pCache!=0 ); + sqlite3GlobalConfig.pcache2.xDestroy(pCache->pCache); } /* @@ -586,11 +621,8 @@ int sqlite3PcachePageRefcount(PgHdr *p){ ** Return the total number of pages in the cache. */ int sqlite3PcachePagecount(PCache *pCache){ - int nPage = 0; - if( pCache->pCache ){ - nPage = sqlite3GlobalConfig.pcache2.xPagecount(pCache->pCache); - } - return nPage; + assert( pCache->pCache!=0 ); + return sqlite3GlobalConfig.pcache2.xPagecount(pCache->pCache); } #ifdef SQLITE_TEST @@ -606,22 +638,27 @@ int sqlite3PcacheGetCachesize(PCache *pCache){ ** Set the suggested cache-size value. */ void sqlite3PcacheSetCachesize(PCache *pCache, int mxPage){ + assert( pCache->pCache!=0 ); pCache->szCache = mxPage; - if( pCache->pCache ){ - sqlite3GlobalConfig.pcache2.xCachesize(pCache->pCache, - numberOfCachePages(pCache)); - } + sqlite3GlobalConfig.pcache2.xCachesize(pCache->pCache, + numberOfCachePages(pCache)); } /* ** Free up as much memory as possible from the page cache. */ void sqlite3PcacheShrink(PCache *pCache){ - if( pCache->pCache ){ - sqlite3GlobalConfig.pcache2.xShrink(pCache->pCache); - } + assert( pCache->pCache!=0 ); + sqlite3GlobalConfig.pcache2.xShrink(pCache->pCache); } +/* +** Return the size of the header added by this middleware layer +** in the page-cache hierarchy. +*/ +int sqlite3HeaderSizePcache(void){ return ROUND8(sizeof(PgHdr)); } + + #if defined(SQLITE_CHECK_PAGES) || defined(SQLITE_DEBUG) /* ** For all dirty pages currently in the cache, invoke the specified diff --git a/src/pcache.h b/src/pcache.h index f4d4ad71c1..9ed62a88ff 100644 --- a/src/pcache.h +++ b/src/pcache.h @@ -68,7 +68,7 @@ void sqlite3PCacheBufferSetup(void *, int sz, int n); ** Under memory stress, invoke xStress to try to make pages clean. ** Only clean and unpinned pages can be reclaimed. */ -void sqlite3PcacheOpen( +int sqlite3PcacheOpen( int szPage, /* Size of every page */ int szExtra, /* Extra space associated with each page */ int bPurgeable, /* True if pages are on backing store */ @@ -78,7 +78,7 @@ void sqlite3PcacheOpen( ); /* Modify the page-size after the cache has been created. */ -void sqlite3PcacheSetPageSize(PCache *, int); +int sqlite3PcacheSetPageSize(PCache *, int); /* Return the size in bytes of a PCache object. Used to preallocate ** storage space. @@ -88,7 +88,9 @@ int sqlite3PcacheSize(void); /* One release per successful fetch. Page is pinned until released. ** Reference counted. */ -int sqlite3PcacheFetch(PCache*, Pgno, int createFlag, PgHdr**); +sqlite3_pcache_page *sqlite3PcacheFetch(PCache*, Pgno, int createFlag); +int sqlite3PcacheFetchStress(PCache*, Pgno, sqlite3_pcache_page**); +PgHdr *sqlite3PcacheFetchFinish(PCache*, Pgno, sqlite3_pcache_page *pPage); void sqlite3PcacheRelease(PgHdr*); void sqlite3PcacheDrop(PgHdr*); /* Remove page from cache */ @@ -158,4 +160,8 @@ void sqlite3PcacheStats(int*,int*,int*,int*); void sqlite3PCacheSetDefault(void); +/* Return the header size */ +int sqlite3HeaderSizePcache(void); +int sqlite3HeaderSizePcache1(void); + #endif /* _PCACHE_H_ */ diff --git a/src/pcache1.c b/src/pcache1.c index 1644b0693c..f5f7893714 100644 --- a/src/pcache1.c +++ b/src/pcache1.c @@ -13,7 +13,7 @@ ** This file implements the default page cache implementation (the ** sqlite3_pcache interface). It also contains part of the implementation ** of the SQLITE_CONFIG_PAGECACHE and sqlite3_release_memory() features. -** If the default page cache implementation is overriden, then neither of +** If the default page cache implementation is overridden, then neither of ** these two features are available. */ @@ -25,7 +25,7 @@ typedef struct PgFreeslot PgFreeslot; typedef struct PGroup PGroup; /* Each page cache (or PCache) belongs to a PGroup. A PGroup is a set -** of one or more PCaches that are able to recycle each others unpinned +** of one or more PCaches that are able to recycle each other's unpinned ** pages when they are under memory pressure. A PGroup is an instance of ** the following object. ** @@ -296,7 +296,7 @@ static PgHdr1 *pcache1AllocPage(PCache1 *pCache){ pPg = 0; } #else - pPg = pcache1Alloc(sizeof(PgHdr1) + pCache->szPage + pCache->szExtra); + pPg = pcache1Alloc(ROUND8(sizeof(PgHdr1)) + pCache->szPage + pCache->szExtra); p = (PgHdr1 *)&((u8 *)pPg)[pCache->szPage]; #endif pcache1EnterMutex(pCache->pGroup); @@ -383,7 +383,7 @@ static int pcache1UnderMemoryPressure(PCache1 *pCache){ ** ** The PCache mutex must be held when this function is called. */ -static int pcache1ResizeHash(PCache1 *p){ +static void pcache1ResizeHash(PCache1 *p){ PgHdr1 **apNew; unsigned int nNew; unsigned int i; @@ -415,8 +415,6 @@ static int pcache1ResizeHash(PCache1 *p){ p->apHash = apNew; p->nHash = nNew; } - - return (p->apHash ? SQLITE_OK : SQLITE_NOMEM); } /* @@ -551,6 +549,9 @@ static void pcache1Shutdown(void *NotUsed){ memset(&pcache1, 0, sizeof(pcache1)); } +/* forward declaration */ +static void pcache1Destroy(sqlite3_pcache *p); + /* ** Implementation of the sqlite3_pcache.xCreate method. ** @@ -595,12 +596,17 @@ static sqlite3_pcache *pcache1Create(int szPage, int szExtra, int bPurgeable){ pCache->szPage = szPage; pCache->szExtra = szExtra; pCache->bPurgeable = (bPurgeable ? 1 : 0); + pcache1EnterMutex(pGroup); + pcache1ResizeHash(pCache); if( bPurgeable ){ pCache->nMin = 10; - pcache1EnterMutex(pGroup); pGroup->nMinPage += pCache->nMin; pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; - pcache1LeaveMutex(pGroup); + } + pcache1LeaveMutex(pGroup); + if( pCache->nHash==0 ){ + pcache1Destroy((sqlite3_pcache*)pCache); + pCache = 0; } } return (sqlite3_pcache *)pCache; @@ -656,6 +662,95 @@ static int pcache1Pagecount(sqlite3_pcache *p){ return n; } + +/* +** Implement steps 3, 4, and 5 of the pcache1Fetch() algorithm described +** in the header of the pcache1Fetch() procedure. +** +** This steps are broken out into a separate procedure because they are +** usually not needed, and by avoiding the stack initialization required +** for these steps, the main pcache1Fetch() procedure can run faster. +*/ +static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( + PCache1 *pCache, + unsigned int iKey, + int createFlag +){ + unsigned int nPinned; + PGroup *pGroup = pCache->pGroup; + PgHdr1 *pPage = 0; + + /* Step 3: Abort if createFlag is 1 but the cache is nearly full */ + assert( pCache->nPage >= pCache->nRecyclable ); + nPinned = pCache->nPage - pCache->nRecyclable; + assert( pGroup->mxPinned == pGroup->nMaxPage + 10 - pGroup->nMinPage ); + assert( pCache->n90pct == pCache->nMax*9/10 ); + if( createFlag==1 && ( + nPinned>=pGroup->mxPinned + || nPinned>=pCache->n90pct + || (pcache1UnderMemoryPressure(pCache) && pCache->nRecyclablenPage>=pCache->nHash ) pcache1ResizeHash(pCache); + assert( pCache->nHash>0 && pCache->apHash ); + + /* Step 4. Try to recycle a page. */ + if( pCache->bPurgeable && pGroup->pLruTail && ( + (pCache->nPage+1>=pCache->nMax) + || pGroup->nCurrentPage>=pGroup->nMaxPage + || pcache1UnderMemoryPressure(pCache) + )){ + PCache1 *pOther; + pPage = pGroup->pLruTail; + assert( pPage->isPinned==0 ); + pcache1RemoveFromHash(pPage); + pcache1PinPage(pPage); + pOther = pPage->pCache; + + /* We want to verify that szPage and szExtra are the same for pOther + ** and pCache. Assert that we can verify this by comparing sums. */ + assert( (pCache->szPage & (pCache->szPage-1))==0 && pCache->szPage>=512 ); + assert( pCache->szExtra<512 ); + assert( (pOther->szPage & (pOther->szPage-1))==0 && pOther->szPage>=512 ); + assert( pOther->szExtra<512 ); + + if( pOther->szPage+pOther->szExtra != pCache->szPage+pCache->szExtra ){ + pcache1FreePage(pPage); + pPage = 0; + }else{ + pGroup->nCurrentPage -= (pOther->bPurgeable - pCache->bPurgeable); + } + } + + /* Step 5. If a usable page buffer has still not been found, + ** attempt to allocate a new one. + */ + if( !pPage ){ + if( createFlag==1 ) sqlite3BeginBenignMalloc(); + pPage = pcache1AllocPage(pCache); + if( createFlag==1 ) sqlite3EndBenignMalloc(); + } + + if( pPage ){ + unsigned int h = iKey % pCache->nHash; + pCache->nPage++; + pPage->iKey = iKey; + pPage->pNext = pCache->apHash[h]; + pPage->pCache = pCache; + pPage->pLruPrev = 0; + pPage->pLruNext = 0; + pPage->isPinned = 1; + *(void **)pPage->page.pExtra = 0; + pCache->apHash[h] = pPage; + if( iKey>pCache->iMaxKey ){ + pCache->iMaxKey = iKey; + } + } + return pPage; +} + /* ** Implementation of the sqlite3_pcache.xFetch method. ** @@ -715,9 +810,7 @@ static sqlite3_pcache_page *pcache1Fetch( unsigned int iKey, int createFlag ){ - unsigned int nPinned; PCache1 *pCache = (PCache1 *)p; - PGroup *pGroup; PgHdr1 *pPage = 0; assert( offsetof(PgHdr1,page)==0 ); @@ -725,107 +818,22 @@ static sqlite3_pcache_page *pcache1Fetch( assert( pCache->bPurgeable || pCache->nMin==0 ); assert( pCache->bPurgeable==0 || pCache->nMin==10 ); assert( pCache->nMin==0 || pCache->bPurgeable ); - pcache1EnterMutex(pGroup = pCache->pGroup); + assert( pCache->nHash>0 ); + pcache1EnterMutex(pCache->pGroup); /* Step 1: Search the hash table for an existing entry. */ - if( pCache->nHash>0 ){ - unsigned int h = iKey % pCache->nHash; - for(pPage=pCache->apHash[h]; pPage&&pPage->iKey!=iKey; pPage=pPage->pNext); - } + pPage = pCache->apHash[iKey % pCache->nHash]; + while( pPage && pPage->iKey!=iKey ){ pPage = pPage->pNext; } /* Step 2: Abort if no existing page is found and createFlag is 0 */ if( pPage ){ if( !pPage->isPinned ) pcache1PinPage(pPage); - goto fetch_out; + }else if( createFlag ){ + /* Steps 3, 4, and 5 implemented by this subroutine */ + pPage = pcache1FetchStage2(pCache, iKey, createFlag); } - if( createFlag==0 ){ - goto fetch_out; - } - - /* The pGroup local variable will normally be initialized by the - ** pcache1EnterMutex() macro above. But if SQLITE_MUTEX_OMIT is defined, - ** then pcache1EnterMutex() is a no-op, so we have to initialize the - ** local variable here. Delaying the initialization of pGroup is an - ** optimization: The common case is to exit the module before reaching - ** this point. - */ -#ifdef SQLITE_MUTEX_OMIT - pGroup = pCache->pGroup; -#endif - - /* Step 3: Abort if createFlag is 1 but the cache is nearly full */ - assert( pCache->nPage >= pCache->nRecyclable ); - nPinned = pCache->nPage - pCache->nRecyclable; - assert( pGroup->mxPinned == pGroup->nMaxPage + 10 - pGroup->nMinPage ); - assert( pCache->n90pct == pCache->nMax*9/10 ); - if( createFlag==1 && ( - nPinned>=pGroup->mxPinned - || nPinned>=pCache->n90pct - || pcache1UnderMemoryPressure(pCache) - )){ - goto fetch_out; - } - - if( pCache->nPage>=pCache->nHash && pcache1ResizeHash(pCache) ){ - goto fetch_out; - } - assert( pCache->nHash>0 && pCache->apHash ); - - /* Step 4. Try to recycle a page. */ - if( pCache->bPurgeable && pGroup->pLruTail && ( - (pCache->nPage+1>=pCache->nMax) - || pGroup->nCurrentPage>=pGroup->nMaxPage - || pcache1UnderMemoryPressure(pCache) - )){ - PCache1 *pOther; - pPage = pGroup->pLruTail; - assert( pPage->isPinned==0 ); - pcache1RemoveFromHash(pPage); - pcache1PinPage(pPage); - pOther = pPage->pCache; - - /* We want to verify that szPage and szExtra are the same for pOther - ** and pCache. Assert that we can verify this by comparing sums. */ - assert( (pCache->szPage & (pCache->szPage-1))==0 && pCache->szPage>=512 ); - assert( pCache->szExtra<512 ); - assert( (pOther->szPage & (pOther->szPage-1))==0 && pOther->szPage>=512 ); - assert( pOther->szExtra<512 ); - - if( pOther->szPage+pOther->szExtra != pCache->szPage+pCache->szExtra ){ - pcache1FreePage(pPage); - pPage = 0; - }else{ - pGroup->nCurrentPage -= (pOther->bPurgeable - pCache->bPurgeable); - } - } - - /* Step 5. If a usable page buffer has still not been found, - ** attempt to allocate a new one. - */ - if( !pPage ){ - if( createFlag==1 ) sqlite3BeginBenignMalloc(); - pPage = pcache1AllocPage(pCache); - if( createFlag==1 ) sqlite3EndBenignMalloc(); - } - - if( pPage ){ - unsigned int h = iKey % pCache->nHash; - pCache->nPage++; - pPage->iKey = iKey; - pPage->pNext = pCache->apHash[h]; - pPage->pCache = pCache; - pPage->pLruPrev = 0; - pPage->pLruNext = 0; - pPage->isPinned = 1; - *(void **)pPage->page.pExtra = 0; - pCache->apHash[h] = pPage; - } - -fetch_out: - if( pPage && iKey>pCache->iMaxKey ){ - pCache->iMaxKey = iKey; - } - pcache1LeaveMutex(pGroup); + assert( pPage==0 || pCache->iMaxKey>=iKey ); + pcache1LeaveMutex(pCache->pGroup); return (sqlite3_pcache_page*)pPage; } @@ -973,6 +981,11 @@ void sqlite3PCacheSetDefault(void){ sqlite3_config(SQLITE_CONFIG_PCACHE2, &defaultMethods); } +/* +** Return the size of the header on each page of this PCACHE implementation. +*/ +int sqlite3HeaderSizePcache1(void){ return ROUND8(sizeof(PgHdr1)); } + #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT /* ** This function is called to free superfluous dynamically allocated memory diff --git a/src/pragma.c b/src/pragma.c index 4c69ceb4fd..1312beef04 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -61,15 +61,17 @@ #define PragTyp_TABLE_INFO 30 #define PragTyp_TEMP_STORE 31 #define PragTyp_TEMP_STORE_DIRECTORY 32 -#define PragTyp_WAL_AUTOCHECKPOINT 33 -#define PragTyp_WAL_CHECKPOINT 34 -#define PragTyp_ACTIVATE_EXTENSIONS 35 -#define PragTyp_HEXKEY 36 -#define PragTyp_KEY 37 -#define PragTyp_REKEY 38 -#define PragTyp_LOCK_STATUS 39 -#define PragTyp_PARSER_TRACE 40 +#define PragTyp_THREADS 33 +#define PragTyp_WAL_AUTOCHECKPOINT 34 +#define PragTyp_WAL_CHECKPOINT 35 +#define PragTyp_ACTIVATE_EXTENSIONS 36 +#define PragTyp_HEXKEY 37 +#define PragTyp_KEY 38 +#define PragTyp_REKEY 39 +#define PragTyp_LOCK_STATUS 40 +#define PragTyp_PARSER_TRACE 41 #define PragFlag_NeedSchema 0x01 +#define PragFlag_ReadOnly 0x02 static const struct sPragmaNames { const char *const zName; /* Name of pragma */ u8 ePragTyp; /* PragTyp_XXX value */ @@ -86,7 +88,7 @@ static const struct sPragmaNames { { /* zName: */ "application_id", /* ePragTyp: */ PragTyp_HEADER_VALUE, /* ePragFlag: */ 0, - /* iArg: */ 0 }, + /* iArg: */ BTREE_APPLICATION_ID }, #endif #if !defined(SQLITE_OMIT_AUTOVACUUM) { /* zName: */ "auto_vacuum", @@ -152,6 +154,12 @@ static const struct sPragmaNames { /* ePragFlag: */ 0, /* iArg: */ 0 }, #endif +#if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) + { /* zName: */ "data_version", + /* ePragTyp: */ PragTyp_HEADER_VALUE, + /* ePragFlag: */ PragFlag_ReadOnly, + /* iArg: */ BTREE_DATA_VERSION }, +#endif #if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) { /* zName: */ "database_list", /* ePragTyp: */ PragTyp_DATABASE_LIST, @@ -207,8 +215,8 @@ static const struct sPragmaNames { #if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) { /* zName: */ "freelist_count", /* ePragTyp: */ PragTyp_HEADER_VALUE, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, + /* ePragFlag: */ PragFlag_ReadOnly, + /* iArg: */ BTREE_FREE_PAGE_COUNT }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) { /* zName: */ "full_column_names", @@ -360,7 +368,7 @@ static const struct sPragmaNames { { /* zName: */ "schema_version", /* ePragTyp: */ PragTyp_HEADER_VALUE, /* ePragFlag: */ 0, - /* iArg: */ 0 }, + /* iArg: */ BTREE_SCHEMA_VERSION }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) { /* zName: */ "secure_delete", @@ -418,11 +426,15 @@ static const struct sPragmaNames { /* ePragFlag: */ 0, /* iArg: */ 0 }, #endif + { /* zName: */ "threads", + /* ePragTyp: */ PragTyp_THREADS, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, #if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) { /* zName: */ "user_version", /* ePragTyp: */ PragTyp_HEADER_VALUE, /* ePragFlag: */ 0, - /* iArg: */ 0 }, + /* iArg: */ BTREE_USER_VERSION }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) #if defined(SQLITE_DEBUG) @@ -465,7 +477,7 @@ static const struct sPragmaNames { /* iArg: */ SQLITE_WriteSchema|SQLITE_RecoveryMode }, #endif }; -/* Number of pragmas: 56 on by default, 69 total. */ +/* Number of pragmas: 58 on by default, 71 total. */ /* End of the automatically generated pragma table. ***************************************************************************/ @@ -480,7 +492,7 @@ static const struct sPragmaNames { ** to support legacy SQL code. The safety level used to be boolean ** and older scripts may have used numbers 0 for OFF and 1 for ON. */ -static u8 getSafetyLevel(const char *z, int omitFull, int dflt){ +static u8 getSafetyLevel(const char *z, int omitFull, u8 dflt){ /* 123456789 123456789 */ static const char zText[] = "onoffalseyestruefull"; static const u8 iOffset[] = {0, 1, 2, 4, 9, 12, 16}; @@ -502,7 +514,7 @@ static u8 getSafetyLevel(const char *z, int omitFull, int dflt){ /* ** Interpret the given string as a boolean value. */ -u8 sqlite3GetBoolean(const char *z, int dflt){ +u8 sqlite3GetBoolean(const char *z, u8 dflt){ return getSafetyLevel(z,1,dflt)!=0; } @@ -1048,7 +1060,7 @@ void sqlite3Pragma( Pager *pPager = sqlite3BtreePager(pDb->pBt); i64 iLimit = -2; if( zRight ){ - sqlite3Atoi64(zRight, &iLimit, sqlite3Strlen30(zRight), SQLITE_UTF8); + sqlite3DecOrHexToI64(zRight, &iLimit); if( iLimit<-1 ) iLimit = -1; } iLimit = sqlite3PagerJournalSizeLimit(pPager, iLimit); @@ -1176,7 +1188,7 @@ void sqlite3Pragma( assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( zRight ){ int ii; - sqlite3Atoi64(zRight, &sz, sqlite3Strlen30(zRight), SQLITE_UTF8); + sqlite3DecOrHexToI64(zRight, &sz); if( sz<0 ) sz = sqlite3GlobalConfig.szMmap; if( pId2->n==0 ) db->szMmap = sz; for(ii=db->nDb-1; ii>=0; ii--){ @@ -1392,6 +1404,12 @@ void sqlite3Pragma( ** in auto-commit mode. */ mask &= ~(SQLITE_ForeignKeys); } +#if SQLITE_USER_AUTHENTICATION + if( db->auth.authLevel==UAUTH_User ){ + /* Do not allow non-admin users to modify the schema arbitrarily */ + mask &= ~(SQLITE_WriteSchema); + } +#endif if( sqlite3GetBoolean(zRight, 0) ){ db->flags |= mask; @@ -1544,7 +1562,7 @@ void sqlite3Pragma( for(pIdx=pTab->pIndex, i=0; pIdx; pIdx=pIdx->pNext, i++){ sqlite3VdbeAddOp2(v, OP_Integer, i, 1); sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pIdx->zName, 0); - sqlite3VdbeAddOp2(v, OP_Integer, pIdx->onError!=OE_None, 3); + sqlite3VdbeAddOp2(v, OP_Integer, IsUniqueIndex(pIdx), 3); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); } } @@ -1794,9 +1812,8 @@ void sqlite3Pragma( */ static const int iLn = VDBE_OFFSET_LINENO(2); static const VdbeOpList endCode[] = { - { OP_AddImm, 1, 0, 0}, /* 0 */ - { OP_IfNeg, 1, 0, 0}, /* 1 */ - { OP_String8, 0, 3, 0}, /* 2 */ + { OP_IfNeg, 1, 0, 0}, /* 0 */ + { OP_String8, 0, 3, 0}, /* 1 */ { OP_ResultRow, 3, 1, 0}, }; @@ -1908,28 +1925,76 @@ void sqlite3Pragma( pParse->nMem = MAX(pParse->nMem, 8+j); sqlite3VdbeAddOp2(v, OP_Rewind, iDataCur, 0); VdbeCoverage(v); loopTop = sqlite3VdbeAddOp2(v, OP_AddImm, 7, 1); + /* Verify that all NOT NULL columns really are NOT NULL */ + for(j=0; jnCol; j++){ + char *zErr; + int jmp2, jmp3; + if( j==pTab->iPKey ) continue; + if( pTab->aCol[j].notNull==0 ) continue; + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, j, 3); + sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); + jmp2 = sqlite3VdbeAddOp1(v, OP_NotNull, 3); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); /* Decrement error limit */ + zErr = sqlite3MPrintf(db, "NULL value in %s.%s", pTab->zName, + pTab->aCol[j].zName); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); + sqlite3VdbeAddOp2(v, OP_ResultRow, 3, 1); + jmp3 = sqlite3VdbeAddOp1(v, OP_IfPos, 1); VdbeCoverage(v); + sqlite3VdbeAddOp0(v, OP_Halt); + sqlite3VdbeJumpHere(v, jmp2); + sqlite3VdbeJumpHere(v, jmp3); + } + /* Validate index entries for the current row */ for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ - int jmp2, jmp3, jmp4; + int jmp2, jmp3, jmp4, jmp5; + int ckUniq = sqlite3VdbeMakeLabel(v); if( pPk==pIdx ) continue; r1 = sqlite3GenerateIndexKey(pParse, pIdx, iDataCur, 0, 0, &jmp3, pPrior, r1); pPrior = pIdx; sqlite3VdbeAddOp2(v, OP_AddImm, 8+j, 1); /* increment entry count */ - jmp2 = sqlite3VdbeAddOp4Int(v, OP_Found, iIdxCur+j, 0, r1, + /* Verify that an index entry exists for the current table row */ + jmp2 = sqlite3VdbeAddOp4Int(v, OP_Found, iIdxCur+j, ckUniq, r1, pIdx->nColumn); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); /* Decrement error limit */ sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, "row ", P4_STATIC); sqlite3VdbeAddOp3(v, OP_Concat, 7, 3, 3); - sqlite3VdbeAddOp4(v, OP_String8, 0, 4, 0, " missing from index ", - P4_STATIC); + sqlite3VdbeAddOp4(v, OP_String8, 0, 4, 0, + " missing from index ", P4_STATIC); sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); - sqlite3VdbeAddOp4(v, OP_String8, 0, 4, 0, pIdx->zName, P4_TRANSIENT); + jmp5 = sqlite3VdbeAddOp4(v, OP_String8, 0, 4, 0, + pIdx->zName, P4_TRANSIENT); sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); sqlite3VdbeAddOp2(v, OP_ResultRow, 3, 1); jmp4 = sqlite3VdbeAddOp1(v, OP_IfPos, 1); VdbeCoverage(v); sqlite3VdbeAddOp0(v, OP_Halt); - sqlite3VdbeJumpHere(v, jmp4); sqlite3VdbeJumpHere(v, jmp2); + /* For UNIQUE indexes, verify that only one entry exists with the + ** current key. The entry is unique if (1) any column is NULL + ** or (2) the next entry has a different key */ + if( IsUniqueIndex(pIdx) ){ + int uniqOk = sqlite3VdbeMakeLabel(v); + int jmp6; + int kk; + for(kk=0; kknKeyCol; kk++){ + int iCol = pIdx->aiColumn[kk]; + assert( iCol>=0 && iColnCol ); + if( pTab->aCol[iCol].notNull ) continue; + sqlite3VdbeAddOp2(v, OP_IsNull, r1+kk, uniqOk); + VdbeCoverage(v); + } + jmp6 = sqlite3VdbeAddOp1(v, OP_Next, iIdxCur+j); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Goto, 0, uniqOk); + sqlite3VdbeJumpHere(v, jmp6); + sqlite3VdbeAddOp4Int(v, OP_IdxGT, iIdxCur+j, uniqOk, r1, + pIdx->nKeyCol); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); /* Decrement error limit */ + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, + "non-unique entry in index ", P4_STATIC); + sqlite3VdbeAddOp2(v, OP_Goto, 0, jmp5); + sqlite3VdbeResolveLabel(v, uniqOk); + } + sqlite3VdbeJumpHere(v, jmp4); sqlite3ResolvePartIdxLabel(pParse, jmp3); } sqlite3VdbeAddOp2(v, OP_Next, iDataCur, loopTop); VdbeCoverage(v); @@ -1954,9 +2019,9 @@ void sqlite3Pragma( } } addr = sqlite3VdbeAddOpList(v, ArraySize(endCode), endCode, iLn); - sqlite3VdbeChangeP2(v, addr, -mxErr); - sqlite3VdbeJumpHere(v, addr+1); - sqlite3VdbeChangeP4(v, addr+2, "ok", P4_STATIC); + sqlite3VdbeChangeP3(v, addr, -mxErr); + sqlite3VdbeJumpHere(v, addr); + sqlite3VdbeChangeP4(v, addr+1, "ok", P4_STATIC); } break; #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ @@ -2022,7 +2087,8 @@ void sqlite3Pragma( ){ for(pEnc=&encnames[0]; pEnc->zName; pEnc++){ if( 0==sqlite3StrICmp(zRight, pEnc->zName) ){ - ENC(pParse->db) = pEnc->enc ? pEnc->enc : SQLITE_UTF16NATIVE; + SCHEMA_ENC(db) = ENC(db) = + pEnc->enc ? pEnc->enc : SQLITE_UTF16NATIVE; break; } } @@ -2067,24 +2133,9 @@ void sqlite3Pragma( ** applications for any purpose. */ case PragTyp_HEADER_VALUE: { - int iCookie; /* Cookie index. 1 for schema-cookie, 6 for user-cookie. */ + int iCookie = aPragmaNames[mid].iArg; /* Which cookie to read or write */ sqlite3VdbeUsesBtree(v, iDb); - switch( zLeft[0] ){ - case 'a': case 'A': - iCookie = BTREE_APPLICATION_ID; - break; - case 'f': case 'F': - iCookie = BTREE_FREE_PAGE_COUNT; - break; - case 's': case 'S': - iCookie = BTREE_SCHEMA_VERSION; - break; - default: - iCookie = BTREE_USER_VERSION; - break; - } - - if( zRight && iCookie!=BTREE_FREE_PAGE_COUNT ){ + if( zRight && (aPragmaNames[mid].mPragFlag & PragFlag_ReadOnly)==0 ){ /* Write the specified cookie value */ static const VdbeOpList setCookie[] = { { OP_Transaction, 0, 1, 0}, /* 0 */ @@ -2137,7 +2188,7 @@ void sqlite3Pragma( #ifndef SQLITE_OMIT_WAL /* - ** PRAGMA [database.]wal_checkpoint = passive|full|restart + ** PRAGMA [database.]wal_checkpoint = passive|full|restart|truncate ** ** Checkpoint the database. */ @@ -2149,6 +2200,8 @@ void sqlite3Pragma( eMode = SQLITE_CHECKPOINT_FULL; }else if( sqlite3StrICmp(zRight, "restart")==0 ){ eMode = SQLITE_CHECKPOINT_RESTART; + }else if( sqlite3StrICmp(zRight, "truncate")==0 ){ + eMode = SQLITE_CHECKPOINT_TRUNCATE; } } sqlite3VdbeSetNumCols(v, 3); @@ -2219,13 +2272,33 @@ void sqlite3Pragma( */ case PragTyp_SOFT_HEAP_LIMIT: { sqlite3_int64 N; - if( zRight && sqlite3Atoi64(zRight, &N, 1000000, SQLITE_UTF8)==SQLITE_OK ){ + if( zRight && sqlite3DecOrHexToI64(zRight, &N)==SQLITE_OK ){ sqlite3_soft_heap_limit64(N); } returnSingleInt(pParse, "soft_heap_limit", sqlite3_soft_heap_limit64(-1)); break; } + /* + ** PRAGMA threads + ** PRAGMA threads = N + ** + ** Configure the maximum number of worker threads. Return the new + ** maximum, which might be less than requested. + */ + case PragTyp_THREADS: { + sqlite3_int64 N; + if( zRight + && sqlite3DecOrHexToI64(zRight, &N)==SQLITE_OK + && N>=0 + ){ + sqlite3_limit(db, SQLITE_LIMIT_WORKER_THREADS, (int)(N&0x7fffffff)); + } + returnSingleInt(pParse, "threads", + sqlite3_limit(db, SQLITE_LIMIT_WORKER_THREADS, -1)); + break; + } + #if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) /* ** Report the current state of file logs for all databases diff --git a/src/prepare.c b/src/prepare.c index c7ba53a1f5..97be900d68 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -328,7 +328,7 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ db->aDb[iDb].zName, zMasterName); #ifndef SQLITE_OMIT_AUTHORIZATION { - int (*xAuth)(void*,int,const char*,const char*,const char*,const char*); + sqlite3_xauth xAuth; xAuth = db->xAuth; db->xAuth = 0; #endif @@ -394,8 +394,11 @@ int sqlite3Init(sqlite3 *db, char **pzErrMsg){ int commit_internal = !(db->flags&SQLITE_InternChanges); assert( sqlite3_mutex_held(db->mutex) ); + assert( sqlite3BtreeHoldsMutex(db->aDb[0].pBt) ); + assert( db->init.busy==0 ); rc = SQLITE_OK; db->init.busy = 1; + ENC(db) = SCHEMA_ENC(db); for(i=0; rc==SQLITE_OK && inDb; i++){ if( DbHasProperty(db, i, DB_SchemaLoaded) || i==1 ) continue; rc = sqlite3InitOne(db, i, pzErrMsg); @@ -409,8 +412,8 @@ int sqlite3Init(sqlite3 *db, char **pzErrMsg){ ** schema may contain references to objects in other databases. */ #ifndef SQLITE_OMIT_TEMPDB - if( rc==SQLITE_OK && ALWAYS(db->nDb>1) - && !DbHasProperty(db, 1, DB_SchemaLoaded) ){ + assert( db->nDb>1 ); + if( rc==SQLITE_OK && !DbHasProperty(db, 1, DB_SchemaLoaded) ){ rc = sqlite3InitOne(db, 1, pzErrMsg); if( rc ){ sqlite3ResetOneSchema(db, 1); @@ -593,7 +596,7 @@ static int sqlite3Prepare( rc = sqlite3BtreeSchemaLocked(pBt); if( rc ){ const char *zDb = db->aDb[i].zName; - sqlite3Error(db, rc, "database schema is locked: %s", zDb); + sqlite3ErrorWithMsg(db, rc, "database schema is locked: %s", zDb); testcase( db->flags & SQLITE_ReadUncommitted ); goto end_prepare; } @@ -610,7 +613,7 @@ static int sqlite3Prepare( testcase( nBytes==mxLen ); testcase( nBytes==mxLen+1 ); if( nBytes>mxLen ){ - sqlite3Error(db, SQLITE_TOOBIG, "statement too long"); + sqlite3ErrorWithMsg(db, SQLITE_TOOBIG, "statement too long"); rc = sqlite3ApiExit(db, SQLITE_TOOBIG); goto end_prepare; } @@ -677,10 +680,10 @@ static int sqlite3Prepare( } if( zErrMsg ){ - sqlite3Error(db, rc, "%s", zErrMsg); + sqlite3ErrorWithMsg(db, rc, "%s", zErrMsg); sqlite3DbFree(db, zErrMsg); }else{ - sqlite3Error(db, rc, 0); + sqlite3Error(db, rc); } /* Delete any TriggerPrg structures allocated while parsing this statement. */ @@ -708,9 +711,12 @@ static int sqlite3LockAndPrepare( const char **pzTail /* OUT: End of parsed string */ ){ int rc; - assert( ppStmt!=0 ); + +#ifdef SQLITE_ENABLE_API_ARMOR + if( ppStmt==0 ) return SQLITE_MISUSE_BKPT; +#endif *ppStmt = 0; - if( !sqlite3SafetyCheckOk(db) ){ + if( !sqlite3SafetyCheckOk(db)||zSql==0 ){ return SQLITE_MISUSE_BKPT; } sqlite3_mutex_enter(db->mutex); @@ -817,9 +823,11 @@ static int sqlite3Prepare16( const char *zTail8 = 0; int rc = SQLITE_OK; - assert( ppStmt ); +#ifdef SQLITE_ENABLE_API_ARMOR + if( ppStmt==0 ) return SQLITE_MISUSE_BKPT; +#endif *ppStmt = 0; - if( !sqlite3SafetyCheckOk(db) ){ + if( !sqlite3SafetyCheckOk(db)||zSql==0 ){ return SQLITE_MISUSE_BKPT; } if( nBytes>=0 ){ diff --git a/src/printf.c b/src/printf.c index 37910804d2..ac7b051631 100644 --- a/src/printf.c +++ b/src/printf.c @@ -14,6 +14,17 @@ */ #include "sqliteInt.h" +/* +** If the strchrnul() library function is available, then set +** HAVE_STRCHRNUL. If that routine is not available, this module +** will supply its own. The built-in version is slower than +** the glibc version so the glibc version is definitely preferred. +*/ +#if !defined(HAVE_STRCHRNUL) +# define HAVE_STRCHRNUL 0 +#endif + + /* ** Conversion types fall into various categories as defined by the ** following enumeration. @@ -201,7 +212,7 @@ void sqlite3VXPrintf( const et_info *infop; /* Pointer to the appropriate info structure */ char *zOut; /* Rendering buffer */ int nOut; /* Size of the rendering buffer */ - char *zExtra; /* Malloced memory used by some conversion */ + char *zExtra = 0; /* Malloced memory used by some conversion */ #ifndef SQLITE_OMIT_FLOATING_POINT int exp, e2; /* exponent of real numbers */ int nsd; /* Number of significant digits returned */ @@ -212,6 +223,13 @@ void sqlite3VXPrintf( PrintfArguments *pArgList = 0; /* Arguments for SQLITE_PRINTF_SQLFUNC */ char buf[etBUFSIZE]; /* Conversion buffer */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( ap==0 ){ + (void)SQLITE_MISUSE_BKPT; + sqlite3StrAccumReset(pAccum); + return; + } +#endif bufpt = 0; if( bFlags ){ if( (bArgList = (bFlags & SQLITE_PRINTF_SQLFUNC))!=0 ){ @@ -224,9 +242,13 @@ void sqlite3VXPrintf( for(; (c=(*fmt))!=0; ++fmt){ if( c!='%' ){ bufpt = (char *)fmt; - while( (c=(*++fmt))!='%' && c!=0 ){}; +#if HAVE_STRCHRNUL + fmt = strchrnul(fmt, '%'); +#else + do{ fmt++; }while( *fmt && *fmt != '%' ); +#endif sqlite3StrAccumAppend(pAccum, bufpt, (int)(fmt - bufpt)); - if( c==0 ) break; + if( *fmt==0 ) break; } if( (c=(*++fmt))==0 ){ sqlite3StrAccumAppend(pAccum, "%", 1); @@ -314,7 +336,6 @@ void sqlite3VXPrintf( break; } } - zExtra = 0; /* ** At this point, variables are initialized as follows: @@ -605,13 +626,16 @@ void sqlite3VXPrintf( }else{ c = va_arg(ap,int); } - buf[0] = (char)c; - if( precision>=0 ){ - for(idx=1; idx1 ){ + width -= precision-1; + if( width>1 && !flag_leftjustify ){ + sqlite3AppendChar(pAccum, width-1, ' '); + width = 0; + } + sqlite3AppendChar(pAccum, precision-1, c); } + length = 1; + buf[0] = c; bufpt = buf; break; case etSTRING: @@ -712,11 +736,14 @@ void sqlite3VXPrintf( ** the output. */ width -= length; - if( width>0 && !flag_leftjustify ) sqlite3AppendSpace(pAccum, width); + if( width>0 && !flag_leftjustify ) sqlite3AppendChar(pAccum, width, ' '); sqlite3StrAccumAppend(pAccum, bufpt, length); - if( width>0 && flag_leftjustify ) sqlite3AppendSpace(pAccum, width); + if( width>0 && flag_leftjustify ) sqlite3AppendChar(pAccum, width, ' '); - if( zExtra ) sqlite3_free(zExtra); + if( zExtra ){ + sqlite3_free(zExtra); + zExtra = 0; + } }/* End for loop over the format string */ } /* End of function */ @@ -743,6 +770,11 @@ static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ char *zOld = (p->zText==p->zBase ? 0 : p->zText); i64 szNew = p->nChar; szNew += N + 1; + if( szNew+p->nChar<=p->mxAlloc ){ + /* Force exponential buffer size growth as long as it does not overflow, + ** to avoid having to call this routine too often */ + szNew += p->nChar; + } if( szNew > p->mxAlloc ){ sqlite3StrAccumReset(p); setStrAccumError(p, STRACCUM_TOOBIG); @@ -759,6 +791,7 @@ static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ assert( p->zText!=0 || p->nChar==0 ); if( zOld==0 && p->nChar>0 ) memcpy(zNew, p->zText, p->nChar); p->zText = zNew; + p->nAlloc = sqlite3DbMallocSize(p->db, zNew); }else{ sqlite3StrAccumReset(p); setStrAccumError(p, STRACCUM_NOMEM); @@ -769,11 +802,11 @@ static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ } /* -** Append N space characters to the given string buffer. +** Append N copies of character c to the given string buffer. */ -void sqlite3AppendSpace(StrAccum *p, int N){ +void sqlite3AppendChar(StrAccum *p, int N, char c){ if( p->nChar+N >= p->nAlloc && (N = sqlite3StrAccumEnlarge(p, N))<=0 ) return; - while( (N--)>0 ) p->zText[p->nChar++] = ' '; + while( (N--)>0 ) p->zText[p->nChar++] = c; } /* @@ -784,7 +817,7 @@ void sqlite3AppendSpace(StrAccum *p, int N){ ** work (enlarging the buffer) using tail recursion, so that the ** sqlite3StrAccumAppend() routine can use fast calling semantics. */ -static void enlargeAndAppend(StrAccum *p, const char *z, int N){ +static void SQLITE_NOINLINE enlargeAndAppend(StrAccum *p, const char *z, int N){ N = sqlite3StrAccumEnlarge(p, N); if( N>0 ){ memcpy(&p->zText[p->nChar], z, N); @@ -803,11 +836,11 @@ void sqlite3StrAccumAppend(StrAccum *p, const char *z, int N){ assert( p->accError==0 || p->nAlloc==0 ); if( p->nChar+N >= p->nAlloc ){ enlargeAndAppend(p,z,N); - return; + }else{ + assert( p->zText ); + p->nChar += N; + memcpy(&p->zText[p->nChar-N], z, N); } - assert( p->zText ); - memcpy(&p->zText[p->nChar], z, N); - p->nChar += N; } /* @@ -904,7 +937,7 @@ char *sqlite3MPrintf(sqlite3 *db, const char *zFormat, ...){ /* ** Like sqlite3MPrintf(), but call sqlite3DbFree() on zStr after formatting -** the string and before returnning. This routine is intended to be used +** the string and before returning. This routine is intended to be used ** to modify an existing string. For example: ** ** x = sqlite3MPrintf(db, x, "prefix %s suffix", x); @@ -928,6 +961,13 @@ char *sqlite3_vmprintf(const char *zFormat, va_list ap){ char *z; char zBase[SQLITE_PRINT_BUF_SIZE]; StrAccum acc; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( zFormat==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif #ifndef SQLITE_OMIT_AUTOINIT if( sqlite3_initialize() ) return 0; #endif @@ -970,6 +1010,13 @@ char *sqlite3_mprintf(const char *zFormat, ...){ char *sqlite3_vsnprintf(int n, char *zBuf, const char *zFormat, va_list ap){ StrAccum acc; if( n<=0 ) return zBuf; +#ifdef SQLITE_ENABLE_API_ARMOR + if( zBuf==0 || zFormat==0 ) { + (void)SQLITE_MISUSE_BKPT; + if( zBuf && n>0 ) zBuf[0] = 0; + return zBuf; + } +#endif sqlite3StrAccumInit(&acc, zBuf, n, 0); acc.useMalloc = 0; sqlite3VXPrintf(&acc, 0, zFormat, ap); @@ -1037,6 +1084,69 @@ void sqlite3DebugPrintf(const char *zFormat, ...){ } #endif +#ifdef SQLITE_DEBUG +/************************************************************************* +** Routines for implementing the "TreeView" display of hierarchical +** data structures for debugging. +** +** The main entry points (coded elsewhere) are: +** sqlite3TreeViewExpr(0, pExpr, 0); +** sqlite3TreeViewExprList(0, pList, 0, 0); +** sqlite3TreeViewSelect(0, pSelect, 0); +** Insert calls to those routines while debugging in order to display +** a diagram of Expr, ExprList, and Select objects. +** +*/ +/* Add a new subitem to the tree. The moreToFollow flag indicates that this +** is not the last item in the tree. */ +TreeView *sqlite3TreeViewPush(TreeView *p, u8 moreToFollow){ + if( p==0 ){ + p = sqlite3_malloc( sizeof(*p) ); + if( p==0 ) return 0; + memset(p, 0, sizeof(*p)); + }else{ + p->iLevel++; + } + assert( moreToFollow==0 || moreToFollow==1 ); + if( p->iLevelbLine) ) p->bLine[p->iLevel] = moreToFollow; + return p; +} +/* Finished with one layer of the tree */ +void sqlite3TreeViewPop(TreeView *p){ + if( p==0 ) return; + p->iLevel--; + if( p->iLevel<0 ) sqlite3_free(p); +} +/* Generate a single line of output for the tree, with a prefix that contains +** all the appropriate tree lines */ +void sqlite3TreeViewLine(TreeView *p, const char *zFormat, ...){ + va_list ap; + int i; + StrAccum acc; + char zBuf[500]; + sqlite3StrAccumInit(&acc, zBuf, sizeof(zBuf), 0); + acc.useMalloc = 0; + if( p ){ + for(i=0; iiLevel && ibLine)-1; i++){ + sqlite3StrAccumAppend(&acc, p->bLine[i] ? "| " : " ", 4); + } + sqlite3StrAccumAppend(&acc, p->bLine[i] ? "|-- " : "'-- ", 4); + } + va_start(ap, zFormat); + sqlite3VXPrintf(&acc, 0, zFormat, ap); + va_end(ap); + if( zBuf[acc.nChar-1]!='\n' ) sqlite3StrAccumAppend(&acc, "\n", 1); + sqlite3StrAccumFinish(&acc); + fprintf(stdout,"%s", zBuf); + fflush(stdout); +} +/* Shorthand for starting a new tree item that consists of a single label */ +void sqlite3TreeViewItem(TreeView *p, const char *zLabel, u8 moreToFollow){ + p = sqlite3TreeViewPush(p, moreToFollow); + sqlite3TreeViewLine(p, "%s", zLabel); +} +#endif /* SQLITE_DEBUG */ + /* ** variable-argument wrapper around sqlite3VXPrintf(). */ diff --git a/src/random.c b/src/random.c index b82566524c..179d01bef2 100644 --- a/src/random.c +++ b/src/random.c @@ -48,11 +48,19 @@ void sqlite3_randomness(int N, void *pBuf){ #endif #if SQLITE_THREADSAFE - sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_PRNG); - sqlite3_mutex_enter(mutex); + sqlite3_mutex *mutex; #endif - if( N<=0 ){ +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return; +#endif + +#if SQLITE_THREADSAFE + mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_PRNG); +#endif + + sqlite3_mutex_enter(mutex); + if( N<=0 || pBuf==0 ){ wsdPrng.isInit = 0; sqlite3_mutex_leave(mutex); return; diff --git a/src/resolve.c b/src/resolve.c index ce26f00c66..d4bd548c93 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -28,7 +28,7 @@ ** is a helper function - a callback for the tree walker. */ static int incrAggDepth(Walker *pWalker, Expr *pExpr){ - if( pExpr->op==TK_AGG_FUNCTION ) pExpr->op2 += pWalker->u.i; + if( pExpr->op==TK_AGG_FUNCTION ) pExpr->op2 += pWalker->u.n; return WRC_Continue; } static void incrAggFunctionDepth(Expr *pExpr, int N){ @@ -36,7 +36,7 @@ static void incrAggFunctionDepth(Expr *pExpr, int N){ Walker w; memset(&w, 0, sizeof(w)); w.xExprCallback = incrAggDepth; - w.u.i = N; + w.u.n = N; sqlite3WalkExpr(&w, pExpr); } } @@ -320,6 +320,10 @@ static int lookupName( if( pMatch ){ pExpr->iTable = pMatch->iCursor; pExpr->pTab = pMatch->pTab; + assert( (pMatch->jointype & JT_RIGHT)==0 ); /* RIGHT JOIN not (yet) supported */ + if( (pMatch->jointype & JT_LEFT)!=0 ){ + ExprSetProperty(pExpr, EP_CanBeNull); + } pSchema = pExpr->pTab->pSchema; } } /* if( pSrcList ) */ @@ -354,7 +358,7 @@ static int lookupName( } } if( iCol>=pTab->nCol && sqlite3IsRowid(zCol) && HasRowid(pTab) ){ - /* IMP: R-24309-18625 */ + /* IMP: R-51414-32910 */ /* IMP: R-44911-55124 */ iCol = -1; } @@ -584,7 +588,7 @@ static int exprProbability(Expr *p){ sqlite3AtoF(p->u.zToken, &r, sqlite3Strlen30(p->u.zToken), SQLITE_UTF8); assert( r>=0.0 ); if( r>1.0 ) return -1; - return (int)(r*1000.0); + return (int)(r*134217728.0); } /* @@ -710,14 +714,16 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ /* EVIDENCE-OF: R-61304-29449 The unlikely(X) function is equivalent to ** likelihood(X, 0.0625). ** EVIDENCE-OF: R-01283-11636 The unlikely(X) function is short-hand for - ** likelihood(X,0.0625). */ + ** likelihood(X,0.0625). + ** EVIDENCE-OF: R-36850-34127 The likely(X) function is short-hand for + ** likelihood(X,0.9375). + ** EVIDENCE-OF: R-53436-40973 The likely(X) function is equivalent to + ** likelihood(X,0.9375). */ /* TUNING: unlikely() probability is 0.0625. likely() is 0.9375 */ - pExpr->iTable = pDef->zName[0]=='u' ? 62 : 938; + pExpr->iTable = pDef->zName[0]=='u' ? 8388608 : 125829120; } } - } #ifndef SQLITE_OMIT_AUTHORIZATION - if( pDef ){ auth = sqlite3AuthCheck(pParse, SQLITE_FUNCTION, 0, pDef->zName, 0); if( auth!=SQLITE_OK ){ if( auth==SQLITE_DENY ){ @@ -728,9 +734,9 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ pExpr->op = TK_NULL; return WRC_Prune; } +#endif if( pDef->funcFlags & SQLITE_FUNC_CONSTANT ) ExprSetProperty(pExpr,EP_Constant); } -#endif if( is_agg && (pNC->ncFlags & NC_AllowAgg)==0 ){ sqlite3ErrorMsg(pParse, "misuse of aggregate function %.*s()", nId,zId); pNC->nErr++; @@ -753,7 +759,13 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ pExpr->op2++; pNC2 = pNC2->pNext; } - if( pNC2 ) pNC2->ncFlags |= NC_HasAgg; + assert( pDef!=0 ); + if( pNC2 ){ + assert( SQLITE_FUNC_MINMAX==NC_MinMaxAgg ); + testcase( (pDef->funcFlags & SQLITE_FUNC_MINMAX)!=0 ); + pNC2->ncFlags |= NC_HasAgg | (pDef->funcFlags & SQLITE_FUNC_MINMAX); + + } pNC->ncFlags |= NC_AllowAgg; } /* FIX ME: Compute pExpr->affinity based on the expected return @@ -1114,7 +1126,7 @@ static int resolveOrderGroupBy( } /* -** Resolve names in the SELECT statement p and all of its descendents. +** Resolve names in the SELECT statement p and all of its descendants. */ static int resolveSelectStep(Walker *pWalker, Select *p){ NameContext *pOuterNC; /* Context that contains this SELECT */ @@ -1218,7 +1230,8 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ assert( (p->selFlags & SF_Aggregate)==0 ); pGroupBy = p->pGroupBy; if( pGroupBy || (sNC.ncFlags & NC_HasAgg)!=0 ){ - p->selFlags |= SF_Aggregate; + assert( NC_MinMaxAgg==SF_MinMaxAgg ); + p->selFlags |= SF_Aggregate | (sNC.ncFlags&NC_MinMaxAgg); }else{ sNC.ncFlags &= ~NC_AllowAgg; } @@ -1346,7 +1359,7 @@ int sqlite3ResolveExprNames( NameContext *pNC, /* Namespace to resolve expressions in. */ Expr *pExpr /* The expression to be analyzed. */ ){ - u8 savedHasAgg; + u16 savedHasAgg; Walker w; if( pExpr==0 ) return 0; @@ -1359,8 +1372,8 @@ int sqlite3ResolveExprNames( pParse->nHeight += pExpr->nHeight; } #endif - savedHasAgg = pNC->ncFlags & NC_HasAgg; - pNC->ncFlags &= ~NC_HasAgg; + savedHasAgg = pNC->ncFlags & (NC_HasAgg|NC_MinMaxAgg); + pNC->ncFlags &= ~(NC_HasAgg|NC_MinMaxAgg); memset(&w, 0, sizeof(w)); w.xExprCallback = resolveExprStep; w.xSelectCallback = resolveSelectStep; @@ -1375,9 +1388,8 @@ int sqlite3ResolveExprNames( } if( pNC->ncFlags & NC_HasAgg ){ ExprSetProperty(pExpr, EP_Agg); - }else if( savedHasAgg ){ - pNC->ncFlags |= NC_HasAgg; } + pNC->ncFlags |= savedHasAgg; return ExprHasProperty(pExpr, EP_Error); } diff --git a/src/rowset.c b/src/rowset.c index ba2e056bd4..ff5593892a 100644 --- a/src/rowset.c +++ b/src/rowset.c @@ -50,7 +50,7 @@ ** No INSERTs may occurs after a SMALLEST. An assertion will fail if ** that is attempted. ** -** The cost of an INSERT is roughly constant. (Sometime new memory +** The cost of an INSERT is roughly constant. (Sometimes new memory ** has to be allocated on an INSERT.) The cost of a TEST with a new ** batch number is O(NlogN) where N is the number of elements in the RowSet. ** The cost of a TEST using the same batch number is O(logN). The cost @@ -443,8 +443,8 @@ int sqlite3RowSetNext(RowSet *p, i64 *pRowid){ ** Check to see if element iRowid was inserted into the rowset as ** part of any insert batch prior to iBatch. Return 1 or 0. ** -** If this is the first test of a new batch and if there exist entires -** on pRowSet->pEntry, then sort those entires into the forest at +** If this is the first test of a new batch and if there exist entries +** on pRowSet->pEntry, then sort those entries into the forest at ** pRowSet->pForest so that they can be tested. */ int sqlite3RowSetTest(RowSet *pRowSet, int iBatch, sqlite3_int64 iRowid){ diff --git a/src/select.c b/src/select.c index 6ceb3fe946..070ac00410 100644 --- a/src/select.c +++ b/src/select.c @@ -14,6 +14,20 @@ */ #include "sqliteInt.h" +/* +** Trace output macros +*/ +#if SELECTTRACE_ENABLED +/***/ int sqlite3SelectTrace = 0; +# define SELECTTRACE(K,P,S,X) \ + if(sqlite3SelectTrace&(K)) \ + sqlite3DebugPrintf("%*s%s.%p: ",(P)->nSelectIndent*2-2,"",(S)->zSelName,(S)),\ + sqlite3DebugPrintf X +#else +# define SELECTTRACE(K,P,S,X) +#endif + + /* ** An instance of the following object is used to record information about ** how to process the DISTINCT keyword, to simplify passing that information @@ -126,6 +140,18 @@ Select *sqlite3SelectNew( return pNew; } +#if SELECTTRACE_ENABLED +/* +** Set the name of a Select object +*/ +void sqlite3SelectSetName(Select *p, const char *zName){ + if( p && zName ){ + sqlite3_snprintf(sizeof(p->zSelName), p->zSelName, "%s", zName); + } +} +#endif + + /* ** Delete the given Select structure and all of its substructures. */ @@ -455,28 +481,43 @@ static KeyInfo *keyInfoFromExprList( ); /* -** Insert code into "v" that will push the record in register regData -** into the sorter. +** Generate code that will push the record in registers regData +** through regData+nData-1 onto the sorter. */ static void pushOntoSorter( Parse *pParse, /* Parser context */ SortCtx *pSort, /* Information about the ORDER BY clause */ Select *pSelect, /* The whole SELECT statement */ - int regData /* Register holding data to be sorted */ + int regData, /* First register holding data to be sorted */ + int nData, /* Number of elements in the data array */ + int nPrefixReg /* No. of reg prior to regData available for use */ ){ - Vdbe *v = pParse->pVdbe; - int nExpr = pSort->pOrderBy->nExpr; - int regRecord = ++pParse->nMem; - int regBase = pParse->nMem+1; - int nOBSat = pSort->nOBSat; - int op; + Vdbe *v = pParse->pVdbe; /* Stmt under construction */ + int bSeq = ((pSort->sortFlags & SORTFLAG_UseSorter)==0); + int nExpr = pSort->pOrderBy->nExpr; /* No. of ORDER BY terms */ + int nBase = nExpr + bSeq + nData; /* Fields in sorter record */ + int regBase; /* Regs for sorter record */ + int regRecord = ++pParse->nMem; /* Assembled sorter record */ + int nOBSat = pSort->nOBSat; /* ORDER BY terms to skip */ + int op; /* Opcode to add sorter record to sorter */ - pParse->nMem += nExpr+2; /* nExpr+2 registers allocated at regBase */ - sqlite3ExprCacheClear(pParse); - sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, 0); - sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); - sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+1, 1); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nExpr+2-nOBSat,regRecord); + assert( bSeq==0 || bSeq==1 ); + if( nPrefixReg ){ + assert( nPrefixReg==nExpr+bSeq ); + regBase = regData - nExpr - bSeq; + }else{ + regBase = pParse->nMem + 1; + pParse->nMem += nBase; + } + sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, SQLITE_ECEL_DUP); + if( bSeq ){ + sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); + } + if( nPrefixReg==0 ){ + sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+bSeq, nData); + } + + sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat, regRecord); if( nOBSat>0 ){ int regPrevKey; /* The first nOBSat columns of the previous row */ int addrFirst; /* Address of the OP_IfNot opcode */ @@ -487,12 +528,17 @@ static void pushOntoSorter( regPrevKey = pParse->nMem+1; pParse->nMem += pSort->nOBSat; - nKey = nExpr - pSort->nOBSat + 1; - addrFirst = sqlite3VdbeAddOp1(v, OP_IfNot, regBase+nExpr); VdbeCoverage(v); + nKey = nExpr - pSort->nOBSat + bSeq; + if( bSeq ){ + addrFirst = sqlite3VdbeAddOp1(v, OP_IfNot, regBase+nExpr); + }else{ + addrFirst = sqlite3VdbeAddOp1(v, OP_SequenceTest, pSort->iECursor); + } + VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_Compare, regPrevKey, regBase, pSort->nOBSat); pOp = sqlite3VdbeGetOp(v, pSort->addrSortIndex); if( pParse->db->mallocFailed ) return; - pOp->p2 = nKey + 1; + pOp->p2 = nKey + nData; pKI = pOp->p4.pKeyInfo; memset(pKI->aSortOrder, 0, pKI->nField); /* Makes OP_Jump below testable */ sqlite3VdbeChangeP4(v, -1, (char*)pKI, P4_KEYINFO); @@ -504,7 +550,7 @@ static void pushOntoSorter( sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); sqlite3VdbeAddOp1(v, OP_ResetSorter, pSort->iECursor); sqlite3VdbeJumpHere(v, addrFirst); - sqlite3VdbeAddOp3(v, OP_Move, regBase, regPrevKey, pSort->nOBSat); + sqlite3ExprCodeMove(pParse, regBase, regPrevKey, pSort->nOBSat); sqlite3VdbeJumpHere(v, addrJmp); } if( pSort->sortFlags & SORTFLAG_UseSorter ){ @@ -541,8 +587,7 @@ static void codeOffset( ){ if( iOffset>0 ){ int addr; - sqlite3VdbeAddOp2(v, OP_AddImm, iOffset, -1); - addr = sqlite3VdbeAddOp1(v, OP_IfNeg, iOffset); VdbeCoverage(v); + addr = sqlite3VdbeAddOp3(v, OP_IfNeg, iOffset, 0, -1); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Goto, 0, iContinue); VdbeComment((v, "skip OFFSET records")); sqlite3VdbeJumpHere(v, addr); @@ -627,6 +672,7 @@ static void selectInnerLoop( int eDest = pDest->eDest; /* How to dispose of results */ int iParm = pDest->iSDParm; /* First argument to disposal method */ int nResultCol; /* Number of result columns */ + int nPrefixReg = 0; /* Number of extra registers before regResult */ assert( v ); assert( pEList!=0 ); @@ -642,6 +688,11 @@ static void selectInnerLoop( nResultCol = pEList->nExpr; if( pDest->iSdst==0 ){ + if( pSort ){ + nPrefixReg = pSort->pOrderBy->nExpr; + if( !(pSort->sortFlags & SORTFLAG_UseSorter) ) nPrefixReg++; + pParse->nMem += nPrefixReg; + } pDest->iSdst = pParse->nMem+1; pParse->nMem += nResultCol; }else if( pDest->iSdst+nResultCol > pParse->nMem ){ @@ -707,7 +758,7 @@ static void selectInnerLoop( sqlite3VdbeChangeP4(v, -1, (const char *)pColl, P4_COLLSEQ); sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); } - assert( sqlite3VdbeCurrentAddr(v)==iJump ); + assert( sqlite3VdbeCurrentAddr(v)==iJump || pParse->db->mallocFailed ); sqlite3VdbeAddOp3(v, OP_Copy, regResult, regPrev, nResultCol-1); break; } @@ -758,10 +809,10 @@ static void selectInnerLoop( case SRT_DistFifo: case SRT_Table: case SRT_EphemTab: { - int r1 = sqlite3GetTempReg(pParse); + int r1 = sqlite3GetTempRange(pParse, nPrefixReg+1); testcase( eDest==SRT_Table ); testcase( eDest==SRT_EphemTab ); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1+nPrefixReg); #ifndef SQLITE_OMIT_CTE if( eDest==SRT_DistFifo ){ /* If the destination is DistFifo, then cursor (iParm+1) is open @@ -776,7 +827,7 @@ static void selectInnerLoop( } #endif if( pSort ){ - pushOntoSorter(pParse, pSort, p, r1); + pushOntoSorter(pParse, pSort, p, r1+nPrefixReg, 1, nPrefixReg); }else{ int r2 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, r2); @@ -784,7 +835,7 @@ static void selectInnerLoop( sqlite3VdbeChangeP5(v, OPFLAG_APPEND); sqlite3ReleaseTempReg(pParse, r2); } - sqlite3ReleaseTempReg(pParse, r1); + sqlite3ReleaseTempRange(pParse, r1, nPrefixReg+1); break; } @@ -802,7 +853,7 @@ static void selectInnerLoop( ** ORDER BY in this case since the order of entries in the set ** does not matter. But there might be a LIMIT clause, in which ** case the order does matter */ - pushOntoSorter(pParse, pSort, p, regResult); + pushOntoSorter(pParse, pSort, p, regResult, 1, nPrefixReg); }else{ int r1 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult,1,r1, &pDest->affSdst, 1); @@ -828,9 +879,9 @@ static void selectInnerLoop( case SRT_Mem: { assert( nResultCol==1 ); if( pSort ){ - pushOntoSorter(pParse, pSort, p, regResult); + pushOntoSorter(pParse, pSort, p, regResult, 1, nPrefixReg); }else{ - sqlite3ExprCodeMove(pParse, regResult, iParm, 1); + assert( regResult==iParm ); /* The LIMIT clause will jump out of the loop for us */ } break; @@ -842,10 +893,7 @@ static void selectInnerLoop( testcase( eDest==SRT_Coroutine ); testcase( eDest==SRT_Output ); if( pSort ){ - int r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); - pushOntoSorter(pParse, pSort, p, r1); - sqlite3ReleaseTempReg(pParse, r1); + pushOntoSorter(pParse, pSort, p, regResult, nResultCol, nPrefixReg); }else if( eDest==SRT_Coroutine ){ sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); }else{ @@ -988,7 +1036,7 @@ int sqlite3KeyInfoIsWriteable(KeyInfo *p){ return p->nRef==1; } ** then the KeyInfo structure is appropriate for initializing a virtual ** index to implement a DISTINCT test. ** -** Space to hold the KeyInfo structure is obtain from malloc. The calling +** Space to hold the KeyInfo structure is obtained from malloc. The calling ** function is responsible for seeing that this structure is eventually ** freed. */ @@ -1125,46 +1173,58 @@ static void generateSortTail( int addr; int addrOnce = 0; int iTab; - int pseudoTab = 0; ExprList *pOrderBy = pSort->pOrderBy; int eDest = pDest->eDest; int iParm = pDest->iSDParm; int regRow; int regRowid; int nKey; + int iSortTab; /* Sorter cursor to read from */ + int nSortData; /* Trailing values to read from sorter */ + int i; + int bSeq; /* True if sorter record includes seq. no. */ +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + struct ExprList_item *aOutEx = p->pEList->a; +#endif if( pSort->labelBkOut ){ sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); sqlite3VdbeAddOp2(v, OP_Goto, 0, addrBreak); sqlite3VdbeResolveLabel(v, pSort->labelBkOut); - addrOnce = sqlite3CodeOnce(pParse); VdbeCoverage(v); } iTab = pSort->iECursor; - regRow = sqlite3GetTempReg(pParse); if( eDest==SRT_Output || eDest==SRT_Coroutine ){ - pseudoTab = pParse->nTab++; - sqlite3VdbeAddOp3(v, OP_OpenPseudo, pseudoTab, regRow, nColumn); regRowid = 0; + regRow = pDest->iSdst; + nSortData = nColumn; }else{ regRowid = sqlite3GetTempReg(pParse); + regRow = sqlite3GetTempReg(pParse); + nSortData = 1; } nKey = pOrderBy->nExpr - pSort->nOBSat; if( pSort->sortFlags & SORTFLAG_UseSorter ){ int regSortOut = ++pParse->nMem; - int ptab2 = pParse->nTab++; - sqlite3VdbeAddOp3(v, OP_OpenPseudo, ptab2, regSortOut, nKey+2); + iSortTab = pParse->nTab++; + if( pSort->labelBkOut ){ + addrOnce = sqlite3CodeOnce(pParse); VdbeCoverage(v); + } + sqlite3VdbeAddOp3(v, OP_OpenPseudo, iSortTab, regSortOut, nKey+1+nSortData); if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak); VdbeCoverage(v); codeOffset(v, p->iOffset, addrContinue); - sqlite3VdbeAddOp2(v, OP_SorterData, iTab, regSortOut); - sqlite3VdbeAddOp3(v, OP_Column, ptab2, nKey+1, regRow); - sqlite3VdbeChangeP5(v, OPFLAG_CLEARCACHE); + sqlite3VdbeAddOp3(v, OP_SorterData, iTab, regSortOut, iSortTab); + bSeq = 0; }else{ - if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); VdbeCoverage(v); codeOffset(v, p->iOffset, addrContinue); - sqlite3VdbeAddOp3(v, OP_Column, iTab, nKey+1, regRow); + iSortTab = iTab; + bSeq = 1; + } + for(i=0; iiSdst+i ); - sqlite3VdbeAddOp3(v, OP_Column, pseudoTab, i, pDest->iSdst+i); - if( i==0 ){ - sqlite3VdbeChangeP5(v, OPFLAG_CLEARCACHE); - } - } if( eDest==SRT_Output ){ sqlite3VdbeAddOp2(v, OP_ResultRow, pDest->iSdst, nColumn); sqlite3ExprCacheAffinityChange(pParse, pDest->iSdst, nColumn); @@ -1213,9 +1265,10 @@ static void generateSortTail( break; } } - sqlite3ReleaseTempReg(pParse, regRow); - sqlite3ReleaseTempReg(pParse, regRowid); - + if( regRowid ){ + sqlite3ReleaseTempReg(pParse, regRow); + sqlite3ReleaseTempReg(pParse, regRowid); + } /* The bottom of the loop */ sqlite3VdbeResolveLabel(v, addrContinue); @@ -1510,7 +1563,7 @@ static void generateColumnNames( } /* -** Given a an expression list (which is really the list of expressions +** Given an expression list (which is really the list of expressions ** that form the result set of a SELECT statement) compute appropriate ** column names for a table that would hold the expression list. ** @@ -1583,7 +1636,7 @@ static int selectColumnsFromExprList( } /* Make sure the column name is unique. If the name is not unique, - ** append a integer to the name so that it becomes unique. + ** append an integer to the name so that it becomes unique. */ nName = sqlite3Strlen30(zName); for(j=cnt=0; j5 ** -** The code generated for this simpification gives the same result +** The code generated for this simplification gives the same result ** but only has to scan the data once. And because indices might ** exist on the table t1, a complete scan of the data might be ** avoided. @@ -3100,8 +3153,10 @@ static void substSelect( ** (9) The subquery does not use LIMIT or the outer query does not use ** aggregates. ** -** (10) The subquery does not use aggregates or the outer query does not -** use LIMIT. +** (**) Restriction (10) was removed from the code on 2005-02-05 but we +** accidently carried the comment forward until 2014-09-15. Original +** text: "The subquery does not use aggregates or the outer query does not +** use LIMIT." ** ** (11) The subquery and the outer query do not both have ORDER BY clauses. ** @@ -3164,6 +3219,11 @@ static void substSelect( ** parent to a compound query confuses the code that handles ** recursive queries in multiSelect(). ** +** (24) The subquery is not an aggregate that uses the built-in min() or +** or max() functions. (Without this restriction, a query like: +** "SELECT x FROM (SELECT max(y), x FROM t1)" would not necessarily +** return the value X for which Y was maximal.) +** ** ** In this routine, the "p" parameter is a pointer to the outer query. ** The subquery is p->pSrc->a[iFrom]. isAgg is true if the outer query @@ -3211,7 +3271,7 @@ static int flattenSubquery( pSubSrc = pSub->pSrc; assert( pSubSrc ); /* Prior to version 3.1.2, when LIMIT and OFFSET had to be simple constants, - ** not arbitrary expresssions, we allowed some combining of LIMIT and OFFSET + ** not arbitrary expressions, we allowed some combining of LIMIT and OFFSET ** because they could be computed at compile-time. But when LIMIT and OFFSET ** became arbitrary expressions, we were forced to add restrictions (13) ** and (14). */ @@ -3236,8 +3296,14 @@ static int flattenSubquery( if( pSub->pLimit && (p->selFlags & SF_Distinct)!=0 ){ return 0; /* Restriction (21) */ } - if( pSub->selFlags & SF_Recursive ) return 0; /* Restriction (22) */ - if( (p->selFlags & SF_Recursive) && pSub->pPrior ) return 0; /* (23) */ + testcase( pSub->selFlags & SF_Recursive ); + testcase( pSub->selFlags & SF_MinMaxAgg ); + if( pSub->selFlags & (SF_Recursive|SF_MinMaxAgg) ){ + return 0; /* Restrictions (22) and (24) */ + } + if( (p->selFlags & SF_Recursive) && pSub->pPrior ){ + return 0; /* Restriction (23) */ + } /* OBSOLETE COMMENT 1: ** Restriction 3: If the subquery is a join, make sure the subquery is @@ -3311,6 +3377,8 @@ static int flattenSubquery( } /***** If we reach this point, flattening is permitted. *****/ + SELECTTRACE(1,pParse,p,("flatten %s.%p from term %d\n", + pSub->zSelName, pSub, iFrom)); /* Authorize the subquery */ pParse->zAuthContext = pSubitem->zName; @@ -3363,6 +3431,7 @@ static int flattenSubquery( p->pLimit = 0; p->pOffset = 0; pNew = sqlite3SelectDup(db, p, 0); + sqlite3SelectSetName(pNew, pSub->zSelName); p->pOffset = pOffset; p->pLimit = pLimit; p->pOrderBy = pOrderBy; @@ -3375,6 +3444,9 @@ static int flattenSubquery( if( pPrior ) pPrior->pNext = pNew; pNew->pNext = p; p->pPrior = pNew; + SELECTTRACE(2,pParse,p, + ("compound-subquery flattener creates %s.%p as peer\n", + pNew->zSelName, pNew)); } if( db->mallocFailed ) return 1; } @@ -3504,8 +3576,23 @@ static int flattenSubquery( pParent->pHaving = substExpr(db, pParent->pHaving, iParent, pSub->pEList); } if( pSub->pOrderBy ){ + /* At this point, any non-zero iOrderByCol values indicate that the + ** ORDER BY column expression is identical to the iOrderByCol'th + ** expression returned by SELECT statement pSub. Since these values + ** do not necessarily correspond to columns in SELECT statement pParent, + ** zero them before transfering the ORDER BY clause. + ** + ** Not doing this may cause an error if a subsequent call to this + ** function attempts to flatten a compound sub-query into pParent + ** (the only way this can happen is if the compound sub-query is + ** currently part of pSub->pSrc). See ticket [d11a6e908f]. */ + ExprList *pOrderBy = pSub->pOrderBy; + for(i=0; inExpr; i++){ + pOrderBy->a[i].u.x.iOrderByCol = 0; + } assert( pParent->pOrderBy==0 ); - pParent->pOrderBy = pSub->pOrderBy; + assert( pSub->pPrior==0 ); + pParent->pOrderBy = pOrderBy; pSub->pOrderBy = 0; }else if( pParent->pOrderBy ){ substExprList(db, pParent->pOrderBy, iParent, pSub->pEList); @@ -3551,6 +3638,13 @@ static int flattenSubquery( */ sqlite3SelectDelete(db, pSub1); +#if SELECTTRACE_ENABLED + if( sqlite3SelectTrace & 0x100 ){ + sqlite3DebugPrintf("After flattening:\n"); + sqlite3TreeViewSelect(0, p, 0); + } +#endif + return 1; } #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ @@ -3597,7 +3691,7 @@ static u8 minMaxQuery(AggInfo *pAggInfo, ExprList **ppMinMax){ /* ** The select statement passed as the first argument is an aggregate query. -** The second argment is the associated aggregate-info object. This +** The second argument is the associated aggregate-info object. This ** function tests if the SELECT is of the form: ** ** SELECT count(*) FROM @@ -3927,10 +4021,10 @@ static void selectPopWith(Walker *pWalker, Select *p){ ** fill pTabList->a[].pSelect with a copy of the SELECT statement ** that implements the view. A copy is made of the view's SELECT ** statement so that we can freely modify or delete that statement -** without worrying about messing up the presistent representation +** without worrying about messing up the persistent representation ** of the view. ** -** (3) Add terms to the WHERE clause to accomodate the NATURAL keyword +** (3) Add terms to the WHERE clause to accommodate the NATURAL keyword ** on joins and the ON and USING clause of joins. ** ** (4) Scan the list of columns in the result set (pEList) looking @@ -4021,6 +4115,7 @@ static int selectExpander(Walker *pWalker, Select *p){ if( sqlite3ViewGetColumnNames(pParse, pTab) ) return WRC_Abort; assert( pFrom->pSelect==0 ); pFrom->pSelect = sqlite3SelectDup(db, pTab->pSelect, 0); + sqlite3SelectSetName(pFrom->pSelect, pTab->zName); sqlite3WalkSelect(pWalker, pFrom->pSelect); } #endif @@ -4555,6 +4650,13 @@ int sqlite3Select( } if( sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0) ) return 1; memset(&sAggInfo, 0, sizeof(sAggInfo)); +#if SELECTTRACE_ENABLED + pParse->nSelectIndent++; + SELECTTRACE(1,pParse,p, ("begin processing:\n")); + if( sqlite3SelectTrace & 0x100 ){ + sqlite3TreeViewSelect(0, p, 0); + } +#endif assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistFifo ); assert( p->pOrderBy==0 || pDest->eDest!=SRT_Fifo ); @@ -4711,6 +4813,10 @@ int sqlite3Select( if( p->pPrior ){ rc = multiSelect(pParse, p, pDest); explainSetInteger(pParse->iSelectId, iRestoreSelectId); +#if SELECTTRACE_ENABLED + SELECTTRACE(1,pParse,p,("end compound-select processing\n")); + pParse->nSelectIndent--; +#endif return rc; } #endif @@ -4723,7 +4829,7 @@ int sqlite3Select( ** ** is transformed to: ** - ** SELECT xyz FROM ... GROUP BY xyz + ** SELECT xyz FROM ... GROUP BY xyz ORDER BY xyz ** ** The second form is preferred as a single index (or temp-table) may be ** used for both the ORDER BY and DISTINCT processing. As originally @@ -4736,7 +4842,6 @@ int sqlite3Select( p->selFlags &= ~SF_Distinct; p->pGroupBy = sqlite3ExprListDup(db, p->pEList, 0); pGroupBy = p->pGroupBy; - sSort.pOrderBy = 0; /* Notice that even thought SF_Distinct has been cleared from p->selFlags, ** the sDistinct.isTnct is still set. Hence, isTnct represents the ** original setting of the SF_Distinct flag, not the current setting */ @@ -4756,8 +4861,9 @@ int sqlite3Select( sSort.iECursor = pParse->nTab++; sSort.addrSortIndex = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, - sSort.iECursor, sSort.pOrderBy->nExpr+2, 0, - (char*)pKeyInfo, P4_KEYINFO); + sSort.iECursor, sSort.pOrderBy->nExpr+1+pEList->nExpr, 0, + (char*)pKeyInfo, P4_KEYINFO + ); }else{ sSort.addrSortIndex = -1; } @@ -4888,7 +4994,7 @@ int sqlite3Select( sNC.pSrcList = pTabList; sNC.pAggInfo = &sAggInfo; sAggInfo.mnReg = pParse->nMem+1; - sAggInfo.nSortingColumn = pGroupBy ? pGroupBy->nExpr+1 : 0; + sAggInfo.nSortingColumn = pGroupBy ? pGroupBy->nExpr : 0; sAggInfo.pGroupBy = pGroupBy; sqlite3ExprAnalyzeAggList(&sNC, pEList); sqlite3ExprAnalyzeAggList(&sNC, sSort.pOrderBy); @@ -4981,8 +5087,8 @@ int sqlite3Select( groupBySort = 1; nGroupBy = pGroupBy->nExpr; - nCol = nGroupBy + 1; - j = nGroupBy+1; + nCol = nGroupBy; + j = nGroupBy; for(i=0; i=j ){ nCol++; @@ -4992,8 +5098,7 @@ int sqlite3Select( regBase = sqlite3GetTempRange(pParse, nCol); sqlite3ExprCacheClear(pParse); sqlite3ExprCodeExprList(pParse, pGroupBy, regBase, 0); - sqlite3VdbeAddOp2(v, OP_Sequence, sAggInfo.sortingIdx,regBase+nGroupBy); - j = nGroupBy+1; + j = nGroupBy; for(i=0; iiSorterColumn>=j ){ @@ -5046,12 +5151,11 @@ int sqlite3Select( addrTopOfLoop = sqlite3VdbeCurrentAddr(v); sqlite3ExprCacheClear(pParse); if( groupBySort ){ - sqlite3VdbeAddOp2(v, OP_SorterData, sAggInfo.sortingIdx, sortOut); + sqlite3VdbeAddOp3(v, OP_SorterData, sAggInfo.sortingIdx, sortOut,sortPTab); } for(j=0; jnExpr; j++){ if( groupBySort ){ sqlite3VdbeAddOp3(v, OP_Column, sortPTab, j, iBMem+j); - if( j==0 ) sqlite3VdbeChangeP5(v, OPFLAG_CLEARCACHE); }else{ sAggInfo.directMode = 1; sqlite3ExprCode(pParse, pGroupBy->a[j].pExpr, iBMem+j); @@ -5310,100 +5414,103 @@ select_end: sqlite3DbFree(db, sAggInfo.aCol); sqlite3DbFree(db, sAggInfo.aFunc); +#if SELECTTRACE_ENABLED + SELECTTRACE(1,pParse,p,("end processing\n")); + pParse->nSelectIndent--; +#endif return rc; } -#if defined(SQLITE_ENABLE_TREE_EXPLAIN) +#ifdef SQLITE_DEBUG /* ** Generate a human-readable description of a the Select object. */ -static void explainOneSelect(Vdbe *pVdbe, Select *p){ - sqlite3ExplainPrintf(pVdbe, "SELECT "); - if( p->selFlags & (SF_Distinct|SF_Aggregate) ){ - if( p->selFlags & SF_Distinct ){ - sqlite3ExplainPrintf(pVdbe, "DISTINCT "); - } - if( p->selFlags & SF_Aggregate ){ - sqlite3ExplainPrintf(pVdbe, "agg_flag "); - } - sqlite3ExplainNL(pVdbe); - sqlite3ExplainPrintf(pVdbe, " "); - } - sqlite3ExplainExprList(pVdbe, p->pEList); - sqlite3ExplainNL(pVdbe); +void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){ + int n = 0; + pView = sqlite3TreeViewPush(pView, moreToFollow); + sqlite3TreeViewLine(pView, "SELECT%s%s", + ((p->selFlags & SF_Distinct) ? " DISTINCT" : ""), + ((p->selFlags & SF_Aggregate) ? " agg_flag" : "") + ); + if( p->pSrc && p->pSrc->nSrc ) n++; + if( p->pWhere ) n++; + if( p->pGroupBy ) n++; + if( p->pHaving ) n++; + if( p->pOrderBy ) n++; + if( p->pLimit ) n++; + if( p->pOffset ) n++; + if( p->pPrior ) n++; + sqlite3TreeViewExprList(pView, p->pEList, (n--)>0, "result-set"); if( p->pSrc && p->pSrc->nSrc ){ int i; - sqlite3ExplainPrintf(pVdbe, "FROM "); - sqlite3ExplainPush(pVdbe); + pView = sqlite3TreeViewPush(pView, (n--)>0); + sqlite3TreeViewLine(pView, "FROM"); for(i=0; ipSrc->nSrc; i++){ struct SrcList_item *pItem = &p->pSrc->a[i]; - sqlite3ExplainPrintf(pVdbe, "{%d,*} = ", pItem->iCursor); - if( pItem->pSelect ){ - sqlite3ExplainSelect(pVdbe, pItem->pSelect); - if( pItem->pTab ){ - sqlite3ExplainPrintf(pVdbe, " (tabname=%s)", pItem->pTab->zName); - } + StrAccum x; + char zLine[100]; + sqlite3StrAccumInit(&x, zLine, sizeof(zLine), 0); + sqlite3XPrintf(&x, 0, "{%d,*}", pItem->iCursor); + if( pItem->zDatabase ){ + sqlite3XPrintf(&x, 0, " %s.%s", pItem->zDatabase, pItem->zName); }else if( pItem->zName ){ - sqlite3ExplainPrintf(pVdbe, "%s", pItem->zName); + sqlite3XPrintf(&x, 0, " %s", pItem->zName); + } + if( pItem->pTab ){ + sqlite3XPrintf(&x, 0, " tabname=%Q", pItem->pTab->zName); } if( pItem->zAlias ){ - sqlite3ExplainPrintf(pVdbe, " (AS %s)", pItem->zAlias); + sqlite3XPrintf(&x, 0, " (AS %s)", pItem->zAlias); } if( pItem->jointype & JT_LEFT ){ - sqlite3ExplainPrintf(pVdbe, " LEFT-JOIN"); + sqlite3XPrintf(&x, 0, " LEFT-JOIN"); } - sqlite3ExplainNL(pVdbe); + sqlite3StrAccumFinish(&x); + sqlite3TreeViewItem(pView, zLine, ipSrc->nSrc-1); + if( pItem->pSelect ){ + sqlite3TreeViewSelect(pView, pItem->pSelect, 0); + } + sqlite3TreeViewPop(pView); } - sqlite3ExplainPop(pVdbe); + sqlite3TreeViewPop(pView); } if( p->pWhere ){ - sqlite3ExplainPrintf(pVdbe, "WHERE "); - sqlite3ExplainExpr(pVdbe, p->pWhere); - sqlite3ExplainNL(pVdbe); + sqlite3TreeViewItem(pView, "WHERE", (n--)>0); + sqlite3TreeViewExpr(pView, p->pWhere, 0); + sqlite3TreeViewPop(pView); } if( p->pGroupBy ){ - sqlite3ExplainPrintf(pVdbe, "GROUPBY "); - sqlite3ExplainExprList(pVdbe, p->pGroupBy); - sqlite3ExplainNL(pVdbe); + sqlite3TreeViewExprList(pView, p->pGroupBy, (n--)>0, "GROUPBY"); } if( p->pHaving ){ - sqlite3ExplainPrintf(pVdbe, "HAVING "); - sqlite3ExplainExpr(pVdbe, p->pHaving); - sqlite3ExplainNL(pVdbe); + sqlite3TreeViewItem(pView, "HAVING", (n--)>0); + sqlite3TreeViewExpr(pView, p->pHaving, 0); + sqlite3TreeViewPop(pView); } if( p->pOrderBy ){ - sqlite3ExplainPrintf(pVdbe, "ORDERBY "); - sqlite3ExplainExprList(pVdbe, p->pOrderBy); - sqlite3ExplainNL(pVdbe); + sqlite3TreeViewExprList(pView, p->pOrderBy, (n--)>0, "ORDERBY"); } if( p->pLimit ){ - sqlite3ExplainPrintf(pVdbe, "LIMIT "); - sqlite3ExplainExpr(pVdbe, p->pLimit); - sqlite3ExplainNL(pVdbe); + sqlite3TreeViewItem(pView, "LIMIT", (n--)>0); + sqlite3TreeViewExpr(pView, p->pLimit, 0); + sqlite3TreeViewPop(pView); } if( p->pOffset ){ - sqlite3ExplainPrintf(pVdbe, "OFFSET "); - sqlite3ExplainExpr(pVdbe, p->pOffset); - sqlite3ExplainNL(pVdbe); + sqlite3TreeViewItem(pView, "OFFSET", (n--)>0); + sqlite3TreeViewExpr(pView, p->pOffset, 0); + sqlite3TreeViewPop(pView); } + if( p->pPrior ){ + const char *zOp = "UNION"; + switch( p->op ){ + case TK_ALL: zOp = "UNION ALL"; break; + case TK_INTERSECT: zOp = "INTERSECT"; break; + case TK_EXCEPT: zOp = "EXCEPT"; break; + } + sqlite3TreeViewItem(pView, zOp, (n--)>0); + sqlite3TreeViewSelect(pView, p->pPrior, 0); + sqlite3TreeViewPop(pView); + } + sqlite3TreeViewPop(pView); } -void sqlite3ExplainSelect(Vdbe *pVdbe, Select *p){ - if( p==0 ){ - sqlite3ExplainPrintf(pVdbe, "(null-select)"); - return; - } - sqlite3ExplainPush(pVdbe); - while( p ){ - explainOneSelect(pVdbe, p); - p = p->pNext; - if( p==0 ) break; - sqlite3ExplainNL(pVdbe); - sqlite3ExplainPrintf(pVdbe, "%s\n", selectOpName(p->op)); - } - sqlite3ExplainPrintf(pVdbe, "END"); - sqlite3ExplainPop(pVdbe); -} - -/* End of the structure debug printing code -*****************************************************************************/ -#endif /* defined(SQLITE_ENABLE_TREE_EXPLAIN) */ +#endif /* SQLITE_DEBUG */ diff --git a/src/shell.c b/src/shell.c index 00d4597545..94d20d3549 100644 --- a/src/shell.c +++ b/src/shell.c @@ -33,6 +33,9 @@ #include #include #include "sqlite3.h" +#if SQLITE_USER_AUTHENTICATION +# include "sqlite3userauth.h" +#endif #include #include @@ -64,6 +67,7 @@ #if defined(_WIN32) || defined(WIN32) # include +# include #define isatty(h) _isatty(h) #ifndef access # define access(f,m) _access((f),(m)) @@ -168,7 +172,8 @@ static HANDLE hProcess; static FILETIME ftKernelBegin; static FILETIME ftUserBegin; static sqlite3_int64 ftWallBegin; -typedef BOOL (WINAPI *GETPROCTIMES)(HANDLE, LPFILETIME, LPFILETIME, LPFILETIME, LPFILETIME); +typedef BOOL (WINAPI *GETPROCTIMES)(HANDLE, LPFILETIME, LPFILETIME, + LPFILETIME, LPFILETIME); static GETPROCTIMES getProcessTimesAddr = NULL; /* @@ -179,15 +184,16 @@ static int hasTimer(void){ if( getProcessTimesAddr ){ return 1; } else { - /* GetProcessTimes() isn't supported in WIN95 and some other Windows versions. - ** See if the version we are running on has it, and if it does, save off - ** a pointer to it and the current process handle. + /* GetProcessTimes() isn't supported in WIN95 and some other Windows + ** versions. See if the version we are running on has it, and if it + ** does, save off a pointer to it and the current process handle. */ hProcess = GetCurrentProcess(); if( hProcess ){ HINSTANCE hinstLib = LoadLibrary(TEXT("Kernel32.dll")); if( NULL != hinstLib ){ - getProcessTimesAddr = (GETPROCTIMES) GetProcAddress(hinstLib, "GetProcessTimes"); + getProcessTimesAddr = + (GETPROCTIMES) GetProcAddress(hinstLib, "GetProcessTimes"); if( NULL != getProcessTimesAddr ){ return 1; } @@ -204,7 +210,8 @@ static int hasTimer(void){ static void beginTimer(void){ if( enableTimer && getProcessTimesAddr ){ FILETIME ftCreation, ftExit; - getProcessTimesAddr(hProcess, &ftCreation, &ftExit, &ftKernelBegin, &ftUserBegin); + getProcessTimesAddr(hProcess,&ftCreation,&ftExit, + &ftKernelBegin,&ftUserBegin); ftWallBegin = timeOfDay(); } } @@ -223,7 +230,7 @@ static void endTimer(void){ if( enableTimer && getProcessTimesAddr){ FILETIME ftCreation, ftExit, ftKernelEnd, ftUserEnd; sqlite3_int64 ftWallEnd = timeOfDay(); - getProcessTimesAddr(hProcess, &ftCreation, &ftExit, &ftKernelEnd, &ftUserEnd); + getProcessTimesAddr(hProcess,&ftCreation,&ftExit,&ftKernelEnd,&ftUserEnd); printf("Run Time: real %.3f user %f sys %f\n", (ftWallEnd - ftWallBegin)*0.001, timeDiff(&ftUserBegin, &ftUserEnd), @@ -431,23 +438,29 @@ static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ return zResult; } -struct previous_mode_data { - int valid; /* Is there legit data in here? */ - int mode; - int showHeader; - int colWidth[100]; +/* +** Shell output mode information from before ".explain on", +** saved so that it can be restored by ".explain off" +*/ +typedef struct SavedModeInfo SavedModeInfo; +struct SavedModeInfo { + int valid; /* Is there legit data in here? */ + int mode; /* Mode prior to ".explain on" */ + int showHeader; /* The ".header" setting prior to ".explain on" */ + int colWidth[100]; /* Column widths prior to ".explain on" */ }; /* -** An pointer to an instance of this structure is passed from -** the main program to the callback. This is used to communicate -** state and mode information. +** State information about the database connection is contained in an +** instance of the following structure. */ -struct callback_data { +typedef struct ShellState ShellState; +struct ShellState { sqlite3 *db; /* The database */ int echoOn; /* True to echo input commands */ int autoEQP; /* Run EXPLAIN QUERY PLAN prior to seach SQL stmt */ int statsOn; /* True to display memory stats before each finalize */ + int scanstatsOn; /* True to display scan stats before each finalize */ int outCount; /* Revert to stdout when reaching zero */ int cnt; /* Number of records displayed so far */ FILE *out; /* Write results here */ @@ -456,15 +469,15 @@ struct callback_data { int mode; /* An output mode setting */ int writableSchema; /* True if PRAGMA writable_schema=ON */ int showHeader; /* True to show column names in List or Column mode */ + unsigned shellFlgs; /* Various flags */ char *zDestTable; /* Name of destination table when MODE_Insert */ char separator[20]; /* Separator character for MODE_List */ + char newline[20]; /* Record separator in MODE_Csv */ int colWidth[100]; /* Requested width of each column when in column mode*/ int actualWidth[100]; /* Actual width of each column */ char nullvalue[20]; /* The text to print when a NULL comes back from ** the database */ - struct previous_mode_data explainPrev; - /* Holds the mode information just before - ** .explain ON */ + SavedModeInfo normalMode;/* Holds the mode just before .explain ON */ char outfile[FILENAME_MAX]; /* Filename for *out */ const char *zDbFilename; /* name of the database file */ char *zFreeOnClose; /* Filename to free when closing */ @@ -476,6 +489,13 @@ struct callback_data { int iIndent; /* Index of current op in aiIndent[] */ }; +/* +** These are the allowed shellFlgs values +*/ +#define SHFLG_Scratch 0x00001 /* The --scratch option is used */ +#define SHFLG_Pagecache 0x00002 /* The --pagecache option is used */ +#define SHFLG_Lookaside 0x00004 /* Lookaside memory is used */ + /* ** These are the allowed modes. */ @@ -520,7 +540,7 @@ static int strlen30(const char *z){ ** A callback for the sqlite3_log() interface. */ static void shellLog(void *pArg, int iErrCode, const char *zMsg){ - struct callback_data *p = (struct callback_data*)pArg; + ShellState *p = (ShellState*)pArg; if( p->pLog==0 ) return; fprintf(p->pLog, "(%d) %s\n", iErrCode, zMsg); fflush(p->pLog); @@ -659,9 +679,10 @@ static const char needCsvQuote[] = { /* ** Output a single term of CSV. Actually, p->separator is used for ** the separator, which may or may not be a comma. p->nullvalue is -** the null value. Strings are quoted if necessary. +** the null value. Strings are quoted if necessary. The separator +** is only issued if bSep is true. */ -static void output_csv(struct callback_data *p, const char *z, int bSep){ +static void output_csv(ShellState *p, const char *z, int bSep){ FILE *out = p->out; if( z==0 ){ fprintf(out,"%s",p->nullvalue); @@ -708,9 +729,15 @@ static void interrupt_handler(int NotUsed){ ** This is the callback routine that the shell ** invokes for each row of a query result. */ -static int shell_callback(void *pArg, int nArg, char **azArg, char **azCol, int *aiType){ +static int shell_callback( + void *pArg, + int nArg, /* Number of result columns */ + char **azArg, /* Text of each result column */ + char **azCol, /* Column names */ + int *aiType /* Column types */ +){ int i; - struct callback_data *p = (struct callback_data*)pArg; + ShellState *p = (ShellState*)pArg; switch( p->mode ){ case MODE_Line: { @@ -855,17 +882,26 @@ static int shell_callback(void *pArg, int nArg, char **azArg, char **azCol, int break; } case MODE_Csv: { +#if defined(WIN32) || defined(_WIN32) + fflush(p->out); + _setmode(_fileno(p->out), _O_BINARY); +#endif if( p->cnt++==0 && p->showHeader ){ for(i=0; iout,"\n"); + fprintf(p->out,"%s",p->newline); } - if( azArg==0 ) break; - for(i=0; i0 ){ + for(i=0; iout,"%s",p->newline); } - fprintf(p->out,"\n"); +#if defined(WIN32) || defined(_WIN32) + fflush(p->out); + _setmode(_fileno(p->out), _O_TEXT); +#endif break; } case MODE_Insert: { @@ -911,11 +947,11 @@ static int callback(void *pArg, int nArg, char **azArg, char **azCol){ } /* -** Set the destination table field of the callback_data structure to +** Set the destination table field of the ShellState structure to ** the name of the table given. Escape any quote characters in the ** table name. */ -static void set_table_name(struct callback_data *p, const char *zName){ +static void set_table_name(ShellState *p, const char *zName){ int i, n; int needQuote; char *z; @@ -1005,7 +1041,7 @@ static char *appendText(char *zIn, char const *zAppend, char quote){ ** won't consume the semicolon terminator. */ static int run_table_dump_query( - struct callback_data *p, /* Query context */ + ShellState *p, /* Query context */ const char *zSelect, /* SELECT statement to extract content */ const char *zFirstRow /* Print before first row, if not NULL */ ){ @@ -1068,7 +1104,7 @@ static char *save_err_msg( */ static int display_stats( sqlite3 *db, /* Database to query */ - struct callback_data *pArg, /* Pointer to struct callback_data */ + ShellState *pArg, /* Pointer to ShellState */ int bReset /* True to reset the stats */ ){ int iCur; @@ -1078,57 +1114,77 @@ static int display_stats( iHiwtr = iCur = -1; sqlite3_status(SQLITE_STATUS_MEMORY_USED, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Memory Used: %d (max %d) bytes\n", iCur, iHiwtr); + fprintf(pArg->out, + "Memory Used: %d (max %d) bytes\n", + iCur, iHiwtr); iHiwtr = iCur = -1; sqlite3_status(SQLITE_STATUS_MALLOC_COUNT, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Number of Outstanding Allocations: %d (max %d)\n", iCur, iHiwtr); -/* -** Not currently used by the CLI. -** iHiwtr = iCur = -1; -** sqlite3_status(SQLITE_STATUS_PAGECACHE_USED, &iCur, &iHiwtr, bReset); -** fprintf(pArg->out, "Number of Pcache Pages Used: %d (max %d) pages\n", iCur, iHiwtr); -*/ + fprintf(pArg->out, "Number of Outstanding Allocations: %d (max %d)\n", + iCur, iHiwtr); + if( pArg->shellFlgs & SHFLG_Pagecache ){ + iHiwtr = iCur = -1; + sqlite3_status(SQLITE_STATUS_PAGECACHE_USED, &iCur, &iHiwtr, bReset); + fprintf(pArg->out, + "Number of Pcache Pages Used: %d (max %d) pages\n", + iCur, iHiwtr); + } iHiwtr = iCur = -1; sqlite3_status(SQLITE_STATUS_PAGECACHE_OVERFLOW, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Number of Pcache Overflow Bytes: %d (max %d) bytes\n", iCur, iHiwtr); -/* -** Not currently used by the CLI. -** iHiwtr = iCur = -1; -** sqlite3_status(SQLITE_STATUS_SCRATCH_USED, &iCur, &iHiwtr, bReset); -** fprintf(pArg->out, "Number of Scratch Allocations Used: %d (max %d)\n", iCur, iHiwtr); -*/ + fprintf(pArg->out, + "Number of Pcache Overflow Bytes: %d (max %d) bytes\n", + iCur, iHiwtr); + if( pArg->shellFlgs & SHFLG_Scratch ){ + iHiwtr = iCur = -1; + sqlite3_status(SQLITE_STATUS_SCRATCH_USED, &iCur, &iHiwtr, bReset); + fprintf(pArg->out, "Number of Scratch Allocations Used: %d (max %d)\n", + iCur, iHiwtr); + } iHiwtr = iCur = -1; sqlite3_status(SQLITE_STATUS_SCRATCH_OVERFLOW, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Number of Scratch Overflow Bytes: %d (max %d) bytes\n", iCur, iHiwtr); + fprintf(pArg->out, + "Number of Scratch Overflow Bytes: %d (max %d) bytes\n", + iCur, iHiwtr); iHiwtr = iCur = -1; sqlite3_status(SQLITE_STATUS_MALLOC_SIZE, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Largest Allocation: %d bytes\n", iHiwtr); + fprintf(pArg->out, "Largest Allocation: %d bytes\n", + iHiwtr); iHiwtr = iCur = -1; sqlite3_status(SQLITE_STATUS_PAGECACHE_SIZE, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Largest Pcache Allocation: %d bytes\n", iHiwtr); + fprintf(pArg->out, "Largest Pcache Allocation: %d bytes\n", + iHiwtr); iHiwtr = iCur = -1; sqlite3_status(SQLITE_STATUS_SCRATCH_SIZE, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Largest Scratch Allocation: %d bytes\n", iHiwtr); + fprintf(pArg->out, "Largest Scratch Allocation: %d bytes\n", + iHiwtr); #ifdef YYTRACKMAXSTACKDEPTH iHiwtr = iCur = -1; sqlite3_status(SQLITE_STATUS_PARSER_STACK, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Deepest Parser Stack: %d (max %d)\n", iCur, iHiwtr); + fprintf(pArg->out, "Deepest Parser Stack: %d (max %d)\n", + iCur, iHiwtr); #endif } if( pArg && pArg->out && db ){ - iHiwtr = iCur = -1; - sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_USED, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Lookaside Slots Used: %d (max %d)\n", iCur, iHiwtr); - sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_HIT, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Successful lookaside attempts: %d\n", iHiwtr); - sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Lookaside failures due to size: %d\n", iHiwtr); - sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Lookaside failures due to OOM: %d\n", iHiwtr); + if( pArg->shellFlgs & SHFLG_Lookaside ){ + iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_USED, + &iCur, &iHiwtr, bReset); + fprintf(pArg->out, "Lookaside Slots Used: %d (max %d)\n", + iCur, iHiwtr); + sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_HIT, + &iCur, &iHiwtr, bReset); + fprintf(pArg->out, "Successful lookaside attempts: %d\n", iHiwtr); + sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE, + &iCur, &iHiwtr, bReset); + fprintf(pArg->out, "Lookaside failures due to size: %d\n", iHiwtr); + sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL, + &iCur, &iHiwtr, bReset); + fprintf(pArg->out, "Lookaside failures due to OOM: %d\n", iHiwtr); + } iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_USED, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Pager Heap Usage: %d bytes\n", iCur); iHiwtr = iCur = -1; + fprintf(pArg->out, "Pager Heap Usage: %d bytes\n",iCur); + iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_HIT, &iCur, &iHiwtr, 1); fprintf(pArg->out, "Page cache hits: %d\n", iCur); iHiwtr = iCur = -1; @@ -1139,18 +1195,19 @@ static int display_stats( fprintf(pArg->out, "Page cache writes: %d\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_SCHEMA_USED, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Schema Heap Usage: %d bytes\n", iCur); + fprintf(pArg->out, "Schema Heap Usage: %d bytes\n",iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_STMT_USED, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Statement Heap/Lookaside Usage: %d bytes\n", iCur); + fprintf(pArg->out, "Statement Heap/Lookaside Usage: %d bytes\n",iCur); } if( pArg && pArg->out && db && pArg->pStmt ){ - iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FULLSCAN_STEP, bReset); + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FULLSCAN_STEP, + bReset); fprintf(pArg->out, "Fullscan Steps: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_SORT, bReset); fprintf(pArg->out, "Sort Operations: %d\n", iCur); - iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_AUTOINDEX, bReset); + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_AUTOINDEX,bReset); fprintf(pArg->out, "Autoindex Inserts: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_VM_STEP, bReset); fprintf(pArg->out, "Virtual Machine Steps: %d\n", iCur); @@ -1159,6 +1216,51 @@ static int display_stats( return 0; } +/* +** Display scan stats. +*/ +static void display_scanstats( + sqlite3 *db, /* Database to query */ + ShellState *pArg /* Pointer to ShellState */ +){ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int i, k, n, mx; + fprintf(pArg->out, "-------- scanstats --------\n"); + mx = 0; + for(k=0; k<=mx; k++){ + double rEstLoop = 1.0; + for(i=n=0; 1; i++){ + sqlite3_stmt *p = pArg->pStmt; + sqlite3_int64 nLoop, nVisit; + double rEst; + int iSid; + const char *zExplain; + if( sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_NLOOP, (void*)&nLoop) ){ + break; + } + sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_SELECTID, (void*)&iSid); + if( iSid>mx ) mx = iSid; + if( iSid!=k ) continue; + if( n==0 ){ + rEstLoop = (double)nLoop; + if( k>0 ) fprintf(pArg->out, "-------- subquery %d -------\n", k); + } + n++; + sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_NVISIT, (void*)&nVisit); + sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EST, (void*)&rEst); + sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EXPLAIN, (void*)&zExplain); + fprintf(pArg->out, "Loop %2d: %s\n", n, zExplain); + rEstLoop *= rEst; + fprintf(pArg->out, + " nLoop=%-8lld nRow=%-8lld estRow=%-8lld estRow/Loop=%-8g\n", + nLoop, nVisit, (sqlite3_int64)(rEstLoop+0.5), rEst + ); + } + } + fprintf(pArg->out, "---------------------------\n"); +#endif +} + /* ** Parameter azArray points to a zero-terminated array of strings. zStr ** points to a single nul-terminated string. Return non-zero if zStr @@ -1175,7 +1277,7 @@ static int str_in_array(const char *zStr, const char **azArray){ /* ** If compiled statement pSql appears to be an EXPLAIN statement, allocate -** and populate the callback_data.aiIndent[] array with the number of +** and populate the ShellState.aiIndent[] array with the number of ** spaces each opcode should be indented before it is output. ** ** The indenting rules are: @@ -1191,7 +1293,7 @@ static int str_in_array(const char *zStr, const char **azArray){ ** then indent all opcodes between the earlier instruction ** and "Goto" by 2 spaces. */ -static void explain_data_prepare(struct callback_data *p, sqlite3_stmt *pSql){ +static void explain_data_prepare(ShellState *p, sqlite3_stmt *pSql){ const char *zSql; /* The text of the SQL statement */ const char *z; /* Used to check if this is an EXPLAIN */ int *abYield = 0; /* True if op is an OP_Yield */ @@ -1200,7 +1302,8 @@ static void explain_data_prepare(struct callback_data *p, sqlite3_stmt *pSql){ const char *azNext[] = { "Next", "Prev", "VPrev", "VNext", "SorterNext", "NextIfOpen", "PrevIfOpen", 0 }; - const char *azYield[] = { "Yield", "SeekLT", "SeekGT", "RowSetRead", "Rewind", 0 }; + const char *azYield[] = { "Yield", "SeekLT", "SeekGT", "RowSetRead", + "Rewind", 0 }; const char *azGoto[] = { "Goto", 0 }; /* Try to figure out if this is really an EXPLAIN statement. If this @@ -1251,7 +1354,7 @@ static void explain_data_prepare(struct callback_data *p, sqlite3_stmt *pSql){ /* ** Free the array allocated by explain_data_prepare(). */ -static void explain_data_delete(struct callback_data *p){ +static void explain_data_delete(ShellState *p){ sqlite3_free(p->aiIndent); p->aiIndent = 0; p->nIndent = 0; @@ -1268,12 +1371,12 @@ static void explain_data_delete(struct callback_data *p){ ** and callback data argument. */ static int shell_exec( - sqlite3 *db, /* An open database */ - const char *zSql, /* SQL to be evaluated */ + sqlite3 *db, /* An open database */ + const char *zSql, /* SQL to be evaluated */ int (*xCallback)(void*,int,char**,char**,int*), /* Callback function */ - /* (not the same as sqlite3_exec) */ - struct callback_data *pArg, /* Pointer to struct callback_data */ - char **pzErrMsg /* Error msg written here */ + /* (not the same as sqlite3_exec) */ + ShellState *pArg, /* Pointer to ShellState */ + char **pzErrMsg /* Error msg written here */ ){ sqlite3_stmt *pStmt = NULL; /* Statement to execute. */ int rc = SQLITE_OK; /* Return Code */ @@ -1313,7 +1416,8 @@ static int shell_exec( /* Show the EXPLAIN QUERY PLAN if .eqp is on */ if( pArg && pArg->autoEQP ){ sqlite3_stmt *pExplain; - char *zEQP = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", sqlite3_sql(pStmt)); + char *zEQP = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", + sqlite3_sql(pStmt)); rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0); if( rc==SQLITE_OK ){ while( sqlite3_step(pExplain)==SQLITE_ROW ){ @@ -1327,15 +1431,6 @@ static int shell_exec( sqlite3_free(zEQP); } - /* Output TESTCTRL_EXPLAIN text of requested */ - if( pArg && pArg->mode==MODE_Explain ){ - const char *zExplain = 0; - sqlite3_test_control(SQLITE_TESTCTRL_EXPLAIN_STMT, pStmt, &zExplain); - if( zExplain && zExplain[0] ){ - fprintf(pArg->out, "%s", zExplain); - } - } - /* If the shell is currently in ".explain" mode, gather the extra ** data required to add indents to the output.*/ if( pArg && pArg->mode==MODE_Explain ){ @@ -1406,6 +1501,11 @@ static int shell_exec( display_stats(db, pArg, 0); } + /* print loop-counters if required */ + if( pArg && pArg->scanstatsOn ){ + display_scanstats(db, pArg); + } + /* Finalize the statement just executed. If this fails, save a ** copy of the error message. Otherwise, set zSql to point to the ** next statement to execute. */ @@ -1441,7 +1541,7 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ const char *zType; const char *zSql; const char *zPrepStmt = 0; - struct callback_data *p = (struct callback_data *)pArg; + ShellState *p = (ShellState *)pArg; UNUSED_PARAMETER(azCol); if( nArg!=3 ) return 1; @@ -1537,7 +1637,7 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ ** "ORDER BY rowid DESC" to the end. */ static int run_schema_dump_query( - struct callback_data *p, + ShellState *p, const char *zQuery ){ int rc; @@ -1583,6 +1683,7 @@ static char zHelp[] = ".exit Exit this program\n" ".explain ?on|off? Turn output mode suitable for EXPLAIN on or off.\n" " With no args, it turns EXPLAIN on.\n" + ".fullschema Show schema and the content of sqlite_stat tables\n" ".headers on|off Turn display of headers on or off\n" ".help Show this message\n" ".import FILE TABLE Import data from FILE into TABLE\n" @@ -1615,10 +1716,12 @@ static char zHelp[] = ".read FILENAME Execute SQL in FILENAME\n" ".restore ?DB? FILE Restore content of DB (default \"main\") from FILE\n" ".save FILE Write in-memory database into FILE\n" + ".scanstats on|off Turn sqlite3_stmt_scanstatus() metrics on or off\n" ".schema ?TABLE? Show the CREATE statements\n" " If TABLE specified, only show tables matching\n" " LIKE pattern TABLE.\n" - ".separator STRING Change separator used by output mode and .import\n" + ".separator STRING ?NL? Change separator used by output mode and .import\n" + " NL is the end-of-line mark for CSV\n" ".shell CMD ARGS... Run CMD ARGS... in a system shell\n" ".show Show the current values for various settings\n" ".stats on|off Turn stats on or off\n" @@ -1635,13 +1738,73 @@ static char zHelp[] = ; /* Forward reference */ -static int process_input(struct callback_data *p, FILE *in); +static int process_input(ShellState *p, FILE *in); +/* +** Implementation of the "readfile(X)" SQL function. The entire content +** of the file named X is read and returned as a BLOB. NULL is returned +** if the file does not exist or is unreadable. +*/ +static void readfileFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zName; + FILE *in; + long nIn; + void *pBuf; + + zName = (const char*)sqlite3_value_text(argv[0]); + if( zName==0 ) return; + in = fopen(zName, "rb"); + if( in==0 ) return; + fseek(in, 0, SEEK_END); + nIn = ftell(in); + rewind(in); + pBuf = sqlite3_malloc( nIn ); + if( pBuf && 1==fread(pBuf, nIn, 1, in) ){ + sqlite3_result_blob(context, pBuf, nIn, sqlite3_free); + }else{ + sqlite3_free(pBuf); + } + fclose(in); +} + +/* +** Implementation of the "writefile(X,Y)" SQL function. The argument Y +** is written into file X. The number of bytes written is returned. Or +** NULL is returned if something goes wrong, such as being unable to open +** file X for writing. +*/ +static void writefileFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + FILE *out; + const char *z; + sqlite3_int64 rc; + const char *zFile; + + zFile = (const char*)sqlite3_value_text(argv[0]); + if( zFile==0 ) return; + out = fopen(zFile, "wb"); + if( out==0 ) return; + z = (const char*)sqlite3_value_blob(argv[1]); + if( z==0 ){ + rc = 0; + }else{ + rc = fwrite(z, 1, sqlite3_value_bytes(argv[1]), out); + } + fclose(out); + sqlite3_result_int64(context, rc); +} /* ** Make sure the database is open. If it is not, then open it. If ** the database fails to open, print an error message and exit. */ -static void open_db(struct callback_data *p, int keepAlive){ +static void open_db(ShellState *p, int keepAlive){ if( p->db==0 ){ sqlite3_initialize(); sqlite3_open(p->zDbFilename, &p->db); @@ -1659,6 +1822,10 @@ static void open_db(struct callback_data *p, int keepAlive){ #ifndef SQLITE_OMIT_LOAD_EXTENSION sqlite3_enable_load_extension(p->db, 1); #endif + sqlite3_create_function(db, "readfile", 1, SQLITE_UTF8, 0, + readfileFunc, 0, 0); + sqlite3_create_function(db, "writefile", 2, SQLITE_UTF8, 0, + writefileFunc, 0, 0); } } @@ -1818,7 +1985,11 @@ static FILE *output_file_open(const char *zFile){ */ static void sql_trace_callback(void *pArg, const char *z){ FILE *f = (FILE*)pArg; - if( f ) fprintf(f, "%s\n", z); + if( f ){ + int i = (int)strlen(z); + while( i>0 && z[i-1]==';' ){ i--; } + fprintf(f, "%.*s;\n", i, z); + } } /* @@ -1936,7 +2107,7 @@ static char *csv_read_one_field(CSVReader *p){ ** work for WITHOUT ROWID tables. */ static void tryToCloneData( - struct callback_data *p, + ShellState *p, sqlite3 *newDb, const char *zTable ){ @@ -2049,10 +2220,10 @@ end_data_xfer: ** sqlite_master table, try again moving backwards. */ static void tryToCloneSchema( - struct callback_data *p, + ShellState *p, sqlite3 *newDb, const char *zWhere, - void (*xForEach)(struct callback_data*,sqlite3*,const char*) + void (*xForEach)(ShellState*,sqlite3*,const char*) ){ sqlite3_stmt *pQuery = 0; char *zQuery = 0; @@ -2123,7 +2294,7 @@ end_schema_xfer: ** as possible out of the main database (which might be corrupt) and write it ** into zNewDb. */ -static void tryToClone(struct callback_data *p, const char *zNewDb){ +static void tryToClone(ShellState *p, const char *zNewDb){ int rc; sqlite3 *newDb = 0; if( access(zNewDb,0)==0 ){ @@ -2148,7 +2319,7 @@ static void tryToClone(struct callback_data *p, const char *zNewDb){ /* ** Change the output file back to stdout */ -static void output_reset(struct callback_data *p){ +static void output_reset(ShellState *p){ if( p->outfile[0]=='|' ){ pclose(p->out); }else{ @@ -2164,7 +2335,7 @@ static void output_reset(struct callback_data *p){ ** ** Return 1 on error, 2 to exit, and 0 otherwise. */ -static int do_meta_command(char *zLine, struct callback_data *p){ +static int do_meta_command(char *zLine, ShellState *p){ int i = 1; int nArg = 0; int n, c; @@ -2282,7 +2453,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ }else if( c=='d' && n>1 && strncmp(azArg[0], "databases", n)==0 ){ - struct callback_data data; + ShellState data; char *zErrMsg = 0; open_db(p, 0); memcpy(&data, p, sizeof(data)); @@ -2380,11 +2551,11 @@ static int do_meta_command(char *zLine, struct callback_data *p){ if( c=='e' && strncmp(azArg[0], "explain", n)==0 ){ int val = nArg>=2 ? booleanValue(azArg[1]) : 1; if(val == 1) { - if(!p->explainPrev.valid) { - p->explainPrev.valid = 1; - p->explainPrev.mode = p->mode; - p->explainPrev.showHeader = p->showHeader; - memcpy(p->explainPrev.colWidth,p->colWidth,sizeof(p->colWidth)); + if(!p->normalMode.valid) { + p->normalMode.valid = 1; + p->normalMode.mode = p->mode; + p->normalMode.showHeader = p->showHeader; + memcpy(p->normalMode.colWidth,p->colWidth,sizeof(p->colWidth)); } /* We could put this code under the !p->explainValid ** condition so that it does not execute if we are already in @@ -2404,11 +2575,62 @@ static int do_meta_command(char *zLine, struct callback_data *p){ p->colWidth[5] = 13; /* P4 */ p->colWidth[6] = 2; /* P5 */ p->colWidth[7] = 13; /* Comment */ - }else if (p->explainPrev.valid) { - p->explainPrev.valid = 0; - p->mode = p->explainPrev.mode; - p->showHeader = p->explainPrev.showHeader; - memcpy(p->colWidth,p->explainPrev.colWidth,sizeof(p->colWidth)); + }else if (p->normalMode.valid) { + p->normalMode.valid = 0; + p->mode = p->normalMode.mode; + p->showHeader = p->normalMode.showHeader; + memcpy(p->colWidth,p->normalMode.colWidth,sizeof(p->colWidth)); + } + }else + + if( c=='f' && strncmp(azArg[0], "fullschema", n)==0 ){ + ShellState data; + char *zErrMsg = 0; + int doStats = 0; + if( nArg!=1 ){ + fprintf(stderr, "Usage: .fullschema\n"); + rc = 1; + goto meta_command_exit; + } + open_db(p, 0); + memcpy(&data, p, sizeof(data)); + data.showHeader = 0; + data.mode = MODE_Semi; + rc = sqlite3_exec(p->db, + "SELECT sql FROM" + " (SELECT sql sql, type type, tbl_name tbl_name, name name, rowid x" + " FROM sqlite_master UNION ALL" + " SELECT sql, type, tbl_name, name, rowid FROM sqlite_temp_master) " + "WHERE type!='meta' AND sql NOTNULL AND name NOT LIKE 'sqlite_%' " + "ORDER BY rowid", + callback, &data, &zErrMsg + ); + if( rc==SQLITE_OK ){ + sqlite3_stmt *pStmt; + rc = sqlite3_prepare_v2(p->db, + "SELECT rowid FROM sqlite_master" + " WHERE name GLOB 'sqlite_stat[134]'", + -1, &pStmt, 0); + doStats = sqlite3_step(pStmt)==SQLITE_ROW; + sqlite3_finalize(pStmt); + } + if( doStats==0 ){ + fprintf(p->out, "/* No STAT tables available */\n"); + }else{ + fprintf(p->out, "ANALYZE sqlite_master;\n"); + sqlite3_exec(p->db, "SELECT 'ANALYZE sqlite_master'", + callback, &data, &zErrMsg); + data.mode = MODE_Insert; + data.zDestTable = "sqlite_stat1"; + shell_exec(p->db, "SELECT * FROM sqlite_stat1", + shell_callback, &data,&zErrMsg); + data.zDestTable = "sqlite_stat3"; + shell_exec(p->db, "SELECT * FROM sqlite_stat3", + shell_callback, &data,&zErrMsg); + data.zDestTable = "sqlite_stat4"; + shell_exec(p->db, "SELECT * FROM sqlite_stat4", + shell_callback, &data, &zErrMsg); + fprintf(p->out, "ANALYZE sqlite_master;\n"); } }else @@ -2583,7 +2805,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ }else if( c=='i' && strncmp(azArg[0], "indices", n)==0 ){ - struct callback_data data; + ShellState data; char *zErrMsg = 0; open_db(p, 0); memcpy(&data, p, sizeof(data)); @@ -2699,6 +2921,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ }else if( c2=='c' && strncmp(azArg[1],"csv",n2)==0 ){ p->mode = MODE_Csv; sqlite3_snprintf(sizeof(p->separator), p->separator, ","); + sqlite3_snprintf(sizeof(p->newline), p->newline, "\r\n"); }else if( c2=='t' && strncmp(azArg[1],"tabs",n2)==0 ){ p->mode = MODE_List; sqlite3_snprintf(sizeof(p->separator), p->separator, "\t"); @@ -2875,8 +3098,21 @@ static int do_meta_command(char *zLine, struct callback_data *p){ sqlite3_close(pSrc); }else + + if( c=='s' && strncmp(azArg[0], "scanstats", n)==0 ){ + if( nArg==2 ){ + p->scanstatsOn = booleanValue(azArg[1]); +#ifndef SQLITE_ENABLE_STMT_SCANSTATUS + fprintf(stderr, "Warning: .scanstats not available in this build.\n"); +#endif + }else{ + fprintf(stderr, "Usage: .scanstats on|off\n"); + rc = 1; + } + }else + if( c=='s' && strncmp(azArg[0], "schema", n)==0 ){ - struct callback_data data; + ShellState data; char *zErrMsg = 0; open_db(p, 0); memcpy(&data, p, sizeof(data)); @@ -2932,7 +3168,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ " (SELECT sql sql, type type, tbl_name tbl_name, name name, rowid x" " FROM sqlite_master UNION ALL" " SELECT sql, type, tbl_name, name, rowid FROM sqlite_temp_master) " - "WHERE type!='meta' AND sql NOTNULL AND name NOT LIKE 'sqlite_%'" + "WHERE type!='meta' AND sql NOTNULL AND name NOT LIKE 'sqlite_%' " "ORDER BY rowid", callback, &data, &zErrMsg ); @@ -2953,6 +3189,15 @@ static int do_meta_command(char *zLine, struct callback_data *p){ } }else + +#if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_SELECTTRACE) + if( c=='s' && n==11 && strncmp(azArg[0], "selecttrace", n)==0 ){ + extern int sqlite3SelectTrace; + sqlite3SelectTrace = nArg>=2 ? booleanValue(azArg[1]) : 0xff; + }else +#endif + + #ifdef SQLITE_DEBUG /* Undocumented commands for internal testing. Subject to change ** without notice. */ @@ -2977,20 +3222,23 @@ static int do_meta_command(char *zLine, struct callback_data *p){ #endif if( c=='s' && strncmp(azArg[0], "separator", n)==0 ){ - if( nArg==2 ){ - sqlite3_snprintf(sizeof(p->separator), p->separator, - "%.*s", (int)sizeof(p->separator)-1, azArg[1]); - }else{ - fprintf(stderr, "Usage: .separator STRING\n"); + if( nArg<2 || nArg>3 ){ + fprintf(stderr, "Usage: .separator SEPARATOR ?NEWLINE?\n"); rc = 1; } + if( nArg>=2 ){ + sqlite3_snprintf(sizeof(p->separator), p->separator, azArg[1]); + } + if( nArg>=3 ){ + sqlite3_snprintf(sizeof(p->newline), p->newline, azArg[2]); + } }else if( c=='s' && (strncmp(azArg[0], "shell", n)==0 || strncmp(azArg[0],"system",n)==0) ){ char *zCmd; - int i; + int i, x; if( nArg<2 ){ fprintf(stderr, "Usage: .system COMMAND\n"); rc = 1; @@ -3001,8 +3249,9 @@ static int do_meta_command(char *zLine, struct callback_data *p){ zCmd = sqlite3_mprintf(strchr(azArg[i],' ')==0?"%z %s":"%z \"%s\"", zCmd, azArg[i]); } - (void)system(zCmd); + x = system(zCmd); sqlite3_free(zCmd); + if( x ) fprintf(stderr, "System command returns %d\n", x); }else if( c=='s' && strncmp(azArg[0], "show", n)==0 ){ @@ -3014,7 +3263,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ } fprintf(p->out,"%9.9s: %s\n","echo", p->echoOn ? "on" : "off"); fprintf(p->out,"%9.9s: %s\n","eqp", p->autoEQP ? "on" : "off"); - fprintf(p->out,"%9.9s: %s\n","explain", p->explainPrev.valid ? "on" :"off"); + fprintf(p->out,"%9.9s: %s\n","explain", p->normalMode.valid ? "on" :"off"); fprintf(p->out,"%9.9s: %s\n","headers", p->showHeader ? "on" : "off"); fprintf(p->out,"%9.9s: %s\n","mode", modeDescr[p->mode]); fprintf(p->out,"%9.9s: ", "nullvalue"); @@ -3024,6 +3273,8 @@ static int do_meta_command(char *zLine, struct callback_data *p){ strlen30(p->outfile) ? p->outfile : "stdout"); fprintf(p->out,"%9.9s: ", "separator"); output_c_string(p->out, p->separator); + fprintf(p->out," "); + output_c_string(p->out, p->newline); fprintf(p->out, "\n"); fprintf(p->out,"%9.9s: %s\n","stats", p->statsOn ? "on" : "off"); fprintf(p->out,"%9.9s: ","width"); @@ -3117,7 +3368,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ for(i=0; iout, "%s%-*s", zSp, maxlen, azResult[j] ? azResult[j] : ""); + fprintf(p->out, "%s%-*s", zSp, maxlen, azResult[j] ? azResult[j]:""); } fprintf(p->out, "\n"); } @@ -3284,6 +3535,71 @@ static int do_meta_command(char *zLine, struct callback_data *p){ #endif }else +#if SQLITE_USER_AUTHENTICATION + if( c=='u' && strncmp(azArg[0], "user", n)==0 ){ + if( nArg<2 ){ + fprintf(stderr, "Usage: .user SUBCOMMAND ...\n"); + rc = 1; + goto meta_command_exit; + } + open_db(p, 0); + if( strcmp(azArg[1],"login")==0 ){ + if( nArg!=4 ){ + fprintf(stderr, "Usage: .user login USER PASSWORD\n"); + rc = 1; + goto meta_command_exit; + } + rc = sqlite3_user_authenticate(p->db, azArg[2], azArg[3], + (int)strlen(azArg[3])); + if( rc ){ + fprintf(stderr, "Authentication failed for user %s\n", azArg[2]); + rc = 1; + } + }else if( strcmp(azArg[1],"add")==0 ){ + if( nArg!=5 ){ + fprintf(stderr, "Usage: .user add USER PASSWORD ISADMIN\n"); + rc = 1; + goto meta_command_exit; + } + rc = sqlite3_user_add(p->db, azArg[2], + azArg[3], (int)strlen(azArg[3]), + booleanValue(azArg[4])); + if( rc ){ + fprintf(stderr, "User-Add failed: %d\n", rc); + rc = 1; + } + }else if( strcmp(azArg[1],"edit")==0 ){ + if( nArg!=5 ){ + fprintf(stderr, "Usage: .user edit USER PASSWORD ISADMIN\n"); + rc = 1; + goto meta_command_exit; + } + rc = sqlite3_user_change(p->db, azArg[2], + azArg[3], (int)strlen(azArg[3]), + booleanValue(azArg[4])); + if( rc ){ + fprintf(stderr, "User-Edit failed: %d\n", rc); + rc = 1; + } + }else if( strcmp(azArg[1],"delete")==0 ){ + if( nArg!=3 ){ + fprintf(stderr, "Usage: .user delete USER\n"); + rc = 1; + goto meta_command_exit; + } + rc = sqlite3_user_delete(p->db, azArg[2]); + if( rc ){ + fprintf(stderr, "User-Delete failed: %d\n", rc); + rc = 1; + } + }else{ + fprintf(stderr, "Usage: .user login|add|edit|delete ...\n"); + rc = 1; + goto meta_command_exit; + } + }else +#endif /* SQLITE_USER_AUTHENTICATION */ + if( c=='v' && strncmp(azArg[0], "version", n)==0 ){ fprintf(p->out, "SQLite %s %s\n" /*extra-version-info*/, sqlite3_libversion(), sqlite3_sourceid()); @@ -3404,7 +3720,7 @@ static int line_is_complete(char *zSql, int nSql){ ** ** Return the number of errors. */ -static int process_input(struct callback_data *p, FILE *in){ +static int process_input(ShellState *p, FILE *in){ char *zLine = 0; /* A single input line */ char *zSql = 0; /* Accumulated SQL text */ int nLine; /* Length of current line */ @@ -3506,6 +3822,7 @@ static int process_input(struct callback_data *p, FILE *in){ if( nSql ){ if( !_all_whitespace(zSql) ){ fprintf(stderr, "Error: incomplete SQL: %s\n", zSql); + errCnt++; } free(zSql); } @@ -3521,7 +3838,8 @@ static char *find_home_dir(void){ static char *home_dir = NULL; if( home_dir ) return home_dir; -#if !defined(_WIN32) && !defined(WIN32) && !defined(_WIN32_WCE) && !defined(__RTP__) && !defined(_WRS_KERNEL) +#if !defined(_WIN32) && !defined(WIN32) && !defined(_WIN32_WCE) \ + && !defined(__RTP__) && !defined(_WRS_KERNEL) { struct passwd *pwent; uid_t uid = getuid(); @@ -3583,7 +3901,7 @@ static char *find_home_dir(void){ ** Returns the number of errors. */ static int process_sqliterc( - struct callback_data *p, /* Configuration data */ + ShellState *p, /* Configuration data */ const char *sqliterc_override /* Name of config file. NULL to use default */ ){ char *home_dir = NULL; @@ -3636,11 +3954,15 @@ static const char zOptions[] = " -interactive force interactive I/O\n" " -line set output mode to 'line'\n" " -list set output mode to 'list'\n" + " -lookaside SIZE N use N entries of SZ bytes for lookaside memory\n" " -mmap N default mmap size set to N\n" #ifdef SQLITE_ENABLE_MULTIPLEX " -multiplex enable the multiplexor VFS\n" #endif + " -newline SEP set newline character(s) for CSV\n" " -nullvalue TEXT set text string for NULL values. Default ''\n" + " -pagecache SIZE N use N slots of SZ bytes each for page cache memory\n" + " -scratch SIZE N use N slots of SZ bytes each for scratch memory\n" " -separator SEP set output field separator. Default: '|'\n" " -stats print memory stats before each finalize\n" " -version show SQLite version\n" @@ -3665,16 +3987,18 @@ static void usage(int showDetail){ /* ** Initialize the state information in data */ -static void main_init(struct callback_data *data) { +static void main_init(ShellState *data) { memset(data, 0, sizeof(*data)); data->mode = MODE_List; memcpy(data->separator,"|", 2); + memcpy(data->newline,"\r\n", 3); data->showHeader = 0; + data->shellFlgs = SHFLG_Lookaside; sqlite3_config(SQLITE_CONFIG_URI, 1); sqlite3_config(SQLITE_CONFIG_LOG, shellLog, data); + sqlite3_config(SQLITE_CONFIG_MULTITHREAD); sqlite3_snprintf(sizeof(mainPrompt), mainPrompt,"sqlite> "); sqlite3_snprintf(sizeof(continuePrompt), continuePrompt," ...> "); - sqlite3_config(SQLITE_CONFIG_SINGLETHREAD); } /* @@ -3712,12 +4036,14 @@ static char *cmdline_option_value(int argc, char **argv, int i){ int main(int argc, char **argv){ char *zErrMsg = 0; - struct callback_data data; + ShellState data; const char *zInitFile = 0; - char *zFirstCmd = 0; int i; int rc = 0; int warnInmemoryDb = 0; + int readStdin = 1; + int nCmd = 0; + char **azCmd = 0; #if USE_SYSTEM_SQLITE+0!=1 if( strcmp(sqlite3_sourceid(),SQLITE_SOURCE_ID)!=0 ){ @@ -3737,6 +4063,18 @@ int main(int argc, char **argv){ signal(SIGINT, interrupt_handler); #endif +#ifdef SQLITE_SHELL_DBNAME_PROC + { + /* If the SQLITE_SHELL_DBNAME_PROC macro is defined, then it is the name + ** of a C-function that will provide the name of the database file. Use + ** this compile-time option to embed this shell program in larger + ** applications. */ + extern void SQLITE_SHELL_DBNAME_PROC(const char**); + SQLITE_SHELL_DBNAME_PROC(&data.zDbFilename); + warnInmemoryDb = 0; + } +#endif + /* Do an initial pass through the command-line argument to locate ** the name of the database file, the name of the initialization file, ** the size of the alternative malloc heap, @@ -3748,19 +4086,23 @@ int main(int argc, char **argv){ if( z[0]!='-' ){ if( data.zDbFilename==0 ){ data.zDbFilename = z; - continue; + }else{ + /* Excesss arguments are interpreted as SQL (or dot-commands) and + ** mean that nothing is read from stdin */ + readStdin = 0; + nCmd++; + azCmd = realloc(azCmd, sizeof(azCmd[0])*nCmd); + if( azCmd==0 ){ + fprintf(stderr, "out of memory\n"); + exit(1); + } + azCmd[nCmd-1] = z; } - if( zFirstCmd==0 ){ - zFirstCmd = z; - continue; - } - fprintf(stderr,"%s: Error: too many options: \"%s\"\n", Argv0, argv[i]); - fprintf(stderr,"Use -help for a list of options.\n"); - return 1; } if( z[1]=='-' ) z++; if( strcmp(z,"-separator")==0 || strcmp(z,"-nullvalue")==0 + || strcmp(z,"-newline")==0 || strcmp(z,"-cmd")==0 ){ (void)cmdline_option_value(argc, argv, ++i); @@ -3782,6 +4124,33 @@ int main(int argc, char **argv){ if( szHeap>0x7fff0000 ) szHeap = 0x7fff0000; sqlite3_config(SQLITE_CONFIG_HEAP, malloc((int)szHeap), (int)szHeap, 64); #endif + }else if( strcmp(z,"-scratch")==0 ){ + int n, sz; + sz = (int)integerValue(cmdline_option_value(argc,argv,++i)); + if( sz>400000 ) sz = 400000; + if( sz<2500 ) sz = 2500; + n = (int)integerValue(cmdline_option_value(argc,argv,++i)); + if( n>10 ) n = 10; + if( n<1 ) n = 1; + sqlite3_config(SQLITE_CONFIG_SCRATCH, malloc(n*sz+1), sz, n); + data.shellFlgs |= SHFLG_Scratch; + }else if( strcmp(z,"-pagecache")==0 ){ + int n, sz; + sz = (int)integerValue(cmdline_option_value(argc,argv,++i)); + if( sz>70000 ) sz = 70000; + if( sz<800 ) sz = 800; + n = (int)integerValue(cmdline_option_value(argc,argv,++i)); + if( n<10 ) n = 10; + sqlite3_config(SQLITE_CONFIG_PAGECACHE, malloc(n*sz+1), sz, n); + data.shellFlgs |= SHFLG_Pagecache; + }else if( strcmp(z,"-lookaside")==0 ){ + int n, sz; + sz = (int)integerValue(cmdline_option_value(argc,argv,++i)); + if( sz<0 ) sz = 0; + n = (int)integerValue(cmdline_option_value(argc,argv,++i)); + if( n<0 ) n = 0; + sqlite3_config(SQLITE_CONFIG_LOOKASIDE, sz, n); + if( sz*n==0 ) data.shellFlgs &= ~SHFLG_Lookaside; #ifdef SQLITE_ENABLE_VFSTRACE }else if( strcmp(z,"-vfstrace")==0 ){ extern int vfstrace_register( @@ -3818,11 +4187,6 @@ int main(int argc, char **argv){ #else fprintf(stderr,"%s: Error: no database filename specified\n", Argv0); return 1; -#endif -#ifdef SQLITE_SHELL_DBNAME_PROC - { extern void SQLITE_SHELL_DBNAME_PROC(const char**); - SQLITE_SHELL_DBNAME_PROC(&data.zDbFilename); - warnInmemoryDb = 0; } #endif } data.out = stdout; @@ -3870,6 +4234,9 @@ int main(int argc, char **argv){ }else if( strcmp(z,"-separator")==0 ){ sqlite3_snprintf(sizeof(data.separator), data.separator, "%s",cmdline_option_value(argc,argv,++i)); + }else if( strcmp(z,"-newline")==0 ){ + sqlite3_snprintf(sizeof(data.newline), data.newline, + "%s",cmdline_option_value(argc,argv,++i)); }else if( strcmp(z,"-nullvalue")==0 ){ sqlite3_snprintf(sizeof(data.nullvalue), data.nullvalue, "%s",cmdline_option_value(argc,argv,++i)); @@ -3883,6 +4250,8 @@ int main(int argc, char **argv){ data.autoEQP = 1; }else if( strcmp(z,"-stats")==0 ){ data.statsOn = 1; + }else if( strcmp(z,"-scanstats")==0 ){ + data.scanstatsOn = 1; }else if( strcmp(z,"-bail")==0 ){ bail_on_error = 1; }else if( strcmp(z,"-version")==0 ){ @@ -3894,6 +4263,12 @@ int main(int argc, char **argv){ stdin_is_interactive = 0; }else if( strcmp(z,"-heap")==0 ){ i++; + }else if( strcmp(z,"-scratch")==0 ){ + i+=2; + }else if( strcmp(z,"-pagecache")==0 ){ + i+=2; + }else if( strcmp(z,"-lookaside")==0 ){ + i+=2; }else if( strcmp(z,"-mmap")==0 ){ i++; }else if( strcmp(z,"-vfs")==0 ){ @@ -3909,6 +4284,10 @@ int main(int argc, char **argv){ }else if( strcmp(z,"-help")==0 ){ usage(1); }else if( strcmp(z,"-cmd")==0 ){ + /* Run commands that follow -cmd first and separately from commands + ** that simply appear on the command-line. This seems goofy. It would + ** be better if all commands ran in the order that they appear. But + ** we retain the goofy behavior for historical compatibility. */ if( i==argc-1 ) break; z = cmdline_option_value(argc,argv,++i); if( z[0]=='.' ){ @@ -3932,23 +4311,28 @@ int main(int argc, char **argv){ } } - if( zFirstCmd ){ - /* Run just the command that follows the database name + if( !readStdin ){ + /* Run all arguments that do not begin with '-' as if they were separate + ** command-line inputs, except for the argToSkip argument which contains + ** the database filename. */ - if( zFirstCmd[0]=='.' ){ - rc = do_meta_command(zFirstCmd, &data); - if( rc==2 ) rc = 0; - }else{ - open_db(&data, 0); - rc = shell_exec(data.db, zFirstCmd, shell_callback, &data, &zErrMsg); - if( zErrMsg!=0 ){ - fprintf(stderr,"Error: %s\n", zErrMsg); - return rc!=0 ? rc : 1; - }else if( rc!=0 ){ - fprintf(stderr,"Error: unable to process SQL \"%s\"\n", zFirstCmd); - return rc; + for(i=0; i ** ** When unlocking, the same SHARED or EXCLUSIVE flag must be supplied as -** was given no the corresponding lock. +** was given on the corresponding lock. ** ** The xShmLock method can transition between unlocked and SHARED or ** between unlocked and EXCLUSIVE. It cannot transition between SHARED @@ -1506,26 +1499,28 @@ struct sqlite3_mem_methods { ** SQLITE_CONFIG_SERIALIZED configuration option. ** ** [[SQLITE_CONFIG_MALLOC]]
    SQLITE_CONFIG_MALLOC
    -**
    ^(This option takes a single argument which is a pointer to an -** instance of the [sqlite3_mem_methods] structure. The argument specifies +**
    ^(The SQLITE_CONFIG_MALLOC option takes a single argument which is +** a pointer to an instance of the [sqlite3_mem_methods] structure. +** The argument specifies ** alternative low-level memory allocation routines to be used in place of ** the memory allocation routines built into SQLite.)^ ^SQLite makes ** its own private copy of the content of the [sqlite3_mem_methods] structure ** before the [sqlite3_config()] call returns.
    ** ** [[SQLITE_CONFIG_GETMALLOC]]
    SQLITE_CONFIG_GETMALLOC
    -**
    ^(This option takes a single argument which is a pointer to an -** instance of the [sqlite3_mem_methods] structure. The [sqlite3_mem_methods] +**
    ^(The SQLITE_CONFIG_GETMALLOC option takes a single argument which +** is a pointer to an instance of the [sqlite3_mem_methods] structure. +** The [sqlite3_mem_methods] ** structure is filled with the currently defined memory allocation routines.)^ ** This option can be used to overload the default memory allocation ** routines with a wrapper that simulations memory allocation failure or ** tracks memory usage, for example.
    ** ** [[SQLITE_CONFIG_MEMSTATUS]]
    SQLITE_CONFIG_MEMSTATUS
    -**
    ^This option takes single argument of type int, interpreted as a -** boolean, which enables or disables the collection of memory allocation -** statistics. ^(When memory allocation statistics are disabled, the -** following SQLite interfaces become non-operational: +**
    ^The SQLITE_CONFIG_MEMSTATUS option takes single argument of type int, +** interpreted as a boolean, which enables or disables the collection of +** memory allocation statistics. ^(When memory allocation statistics are +** disabled, the following SQLite interfaces become non-operational: **
      **
    • [sqlite3_memory_used()] **
    • [sqlite3_memory_highwater()] @@ -1538,53 +1533,67 @@ struct sqlite3_mem_methods { **
    ** ** [[SQLITE_CONFIG_SCRATCH]]
    SQLITE_CONFIG_SCRATCH
    -**
    ^This option specifies a static memory buffer that SQLite can use for -** scratch memory. There are three arguments: A pointer an 8-byte +**
    ^The SQLITE_CONFIG_SCRATCH option specifies a static memory buffer +** that SQLite can use for scratch memory. ^(There are three arguments +** to SQLITE_CONFIG_SCRATCH: A pointer an 8-byte ** aligned memory buffer from which the scratch allocations will be ** drawn, the size of each scratch allocation (sz), -** and the maximum number of scratch allocations (N). The sz -** argument must be a multiple of 16. +** and the maximum number of scratch allocations (N).)^ ** The first argument must be a pointer to an 8-byte aligned buffer ** of at least sz*N bytes of memory. -** ^SQLite will use no more than two scratch buffers per thread. So -** N should be set to twice the expected maximum number of threads. -** ^SQLite will never require a scratch buffer that is more than 6 -** times the database page size. ^If SQLite needs needs additional +** ^SQLite will not use more than one scratch buffers per thread. +** ^SQLite will never request a scratch buffer that is more than 6 +** times the database page size. +** ^If SQLite needs needs additional ** scratch memory beyond what is provided by this configuration option, then -** [sqlite3_malloc()] will be used to obtain the memory needed.
    +** [sqlite3_malloc()] will be used to obtain the memory needed.

    +** ^When the application provides any amount of scratch memory using +** SQLITE_CONFIG_SCRATCH, SQLite avoids unnecessary large +** [sqlite3_malloc|heap allocations]. +** This can help [Robson proof|prevent memory allocation failures] due to heap +** fragmentation in low-memory embedded systems. +** ** ** [[SQLITE_CONFIG_PAGECACHE]]

    SQLITE_CONFIG_PAGECACHE
    -**
    ^This option specifies a static memory buffer that SQLite can use for -** the database page cache with the default page cache implementation. +**
    ^The SQLITE_CONFIG_PAGECACHE option specifies a static memory buffer +** that SQLite can use for the database page cache with the default page +** cache implementation. ** This configuration should not be used if an application-define page -** cache implementation is loaded using the SQLITE_CONFIG_PCACHE2 option. -** There are three arguments to this option: A pointer to 8-byte aligned +** cache implementation is loaded using the [SQLITE_CONFIG_PCACHE2] +** configuration option. +** ^There are three arguments to SQLITE_CONFIG_PAGECACHE: A pointer to +** 8-byte aligned ** memory, the size of each page buffer (sz), and the number of pages (N). ** The sz argument should be the size of the largest database page -** (a power of two between 512 and 32768) plus a little extra for each -** page header. ^The page header size is 20 to 40 bytes depending on -** the host architecture. ^It is harmless, apart from the wasted memory, -** to make sz a little too large. The first -** argument should point to an allocation of at least sz*N bytes of memory. +** (a power of two between 512 and 65536) plus some extra bytes for each +** page header. ^The number of extra bytes needed by the page header +** can be determined using the [SQLITE_CONFIG_PCACHE_HDRSZ] option +** to [sqlite3_config()]. +** ^It is harmless, apart from the wasted memory, +** for the sz parameter to be larger than necessary. The first +** argument should pointer to an 8-byte aligned block of memory that +** is at least sz*N bytes of memory, otherwise subsequent behavior is +** undefined. ** ^SQLite will use the memory provided by the first argument to satisfy its ** memory needs for the first N pages that it adds to cache. ^If additional ** page cache memory is needed beyond what is provided by this option, then -** SQLite goes to [sqlite3_malloc()] for the additional storage space. -** The pointer in the first argument must -** be aligned to an 8-byte boundary or subsequent behavior of SQLite -** will be undefined.
    +** SQLite goes to [sqlite3_malloc()] for the additional storage space. ** ** [[SQLITE_CONFIG_HEAP]]
    SQLITE_CONFIG_HEAP
    -**
    ^This option specifies a static memory buffer that SQLite will use -** for all of its dynamic memory allocation needs beyond those provided -** for by [SQLITE_CONFIG_SCRATCH] and [SQLITE_CONFIG_PAGECACHE]. -** There are three arguments: An 8-byte aligned pointer to the memory, +**
    ^The SQLITE_CONFIG_HEAP option specifies a static memory buffer +** that SQLite will use for all of its dynamic memory allocation needs +** beyond those provided for by [SQLITE_CONFIG_SCRATCH] and +** [SQLITE_CONFIG_PAGECACHE]. +** ^The SQLITE_CONFIG_HEAP option is only available if SQLite is compiled +** with either [SQLITE_ENABLE_MEMSYS3] or [SQLITE_ENABLE_MEMSYS5] and returns +** [SQLITE_ERROR] if invoked otherwise. +** ^There are three arguments to SQLITE_CONFIG_HEAP: +** An 8-byte aligned pointer to the memory, ** the number of bytes in the memory buffer, and the minimum allocation size. ** ^If the first pointer (the memory pointer) is NULL, then SQLite reverts ** to using its default memory allocator (the system malloc() implementation), ** undoing any prior invocation of [SQLITE_CONFIG_MALLOC]. ^If the -** memory pointer is not NULL and either [SQLITE_ENABLE_MEMSYS3] or -** [SQLITE_ENABLE_MEMSYS5] are defined, then the alternative memory +** memory pointer is not NULL then the alternative memory ** allocator is engaged to handle all of SQLites memory allocation needs. ** The first pointer (the memory pointer) must be aligned to an 8-byte ** boundary or subsequent behavior of SQLite will be undefined. @@ -1592,11 +1601,11 @@ struct sqlite3_mem_methods { ** for the minimum allocation size are 2**5 through 2**8.
    ** ** [[SQLITE_CONFIG_MUTEX]]
    SQLITE_CONFIG_MUTEX
    -**
    ^(This option takes a single argument which is a pointer to an -** instance of the [sqlite3_mutex_methods] structure. The argument specifies -** alternative low-level mutex routines to be used in place -** the mutex routines built into SQLite.)^ ^SQLite makes a copy of the -** content of the [sqlite3_mutex_methods] structure before the call to +**
    ^(The SQLITE_CONFIG_MUTEX option takes a single argument which is a +** pointer to an instance of the [sqlite3_mutex_methods] structure. +** The argument specifies alternative low-level mutex routines to be used +** in place the mutex routines built into SQLite.)^ ^SQLite makes a copy of +** the content of the [sqlite3_mutex_methods] structure before the call to ** [sqlite3_config()] returns. ^If SQLite is compiled with ** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then ** the entire mutexing subsystem is omitted from the build and hence calls to @@ -1604,8 +1613,8 @@ struct sqlite3_mem_methods { ** return [SQLITE_ERROR].
    ** ** [[SQLITE_CONFIG_GETMUTEX]]
    SQLITE_CONFIG_GETMUTEX
    -**
    ^(This option takes a single argument which is a pointer to an -** instance of the [sqlite3_mutex_methods] structure. The +**
    ^(The SQLITE_CONFIG_GETMUTEX option takes a single argument which +** is a pointer to an instance of the [sqlite3_mutex_methods] structure. The ** [sqlite3_mutex_methods] ** structure is filled with the currently defined mutex routines.)^ ** This option can be used to overload the default mutex allocation @@ -1617,25 +1626,25 @@ struct sqlite3_mem_methods { ** return [SQLITE_ERROR].
    ** ** [[SQLITE_CONFIG_LOOKASIDE]]
    SQLITE_CONFIG_LOOKASIDE
    -**
    ^(This option takes two arguments that determine the default -** memory allocation for the lookaside memory allocator on each -** [database connection]. The first argument is the +**
    ^(The SQLITE_CONFIG_LOOKASIDE option takes two arguments that determine +** the default size of lookaside memory on each [database connection]. +** The first argument is the ** size of each lookaside buffer slot and the second is the number of -** slots allocated to each database connection.)^ ^(This option sets the -** default lookaside size. The [SQLITE_DBCONFIG_LOOKASIDE] -** verb to [sqlite3_db_config()] can be used to change the lookaside +** slots allocated to each database connection.)^ ^(SQLITE_CONFIG_LOOKASIDE +** sets the default lookaside size. The [SQLITE_DBCONFIG_LOOKASIDE] +** option to [sqlite3_db_config()] can be used to change the lookaside ** configuration on individual connections.)^
    ** ** [[SQLITE_CONFIG_PCACHE2]]
    SQLITE_CONFIG_PCACHE2
    -**
    ^(This option takes a single argument which is a pointer to -** an [sqlite3_pcache_methods2] object. This object specifies the interface -** to a custom page cache implementation.)^ ^SQLite makes a copy of the -** object and uses it for page cache memory allocations.
    +**
    ^(The SQLITE_CONFIG_PCACHE2 option takes a single argument which is +** a pointer to an [sqlite3_pcache_methods2] object. This object specifies +** the interface to a custom page cache implementation.)^ +** ^SQLite makes a copy of the [sqlite3_pcache_methods2] object.
    ** ** [[SQLITE_CONFIG_GETPCACHE2]]
    SQLITE_CONFIG_GETPCACHE2
    -**
    ^(This option takes a single argument which is a pointer to an -** [sqlite3_pcache_methods2] object. SQLite copies of the current -** page cache implementation into that object.)^
    +**
    ^(The SQLITE_CONFIG_GETPCACHE2 option takes a single argument which +** is a pointer to an [sqlite3_pcache_methods2] object. SQLite copies of +** the current page cache implementation into that object.)^
    ** ** [[SQLITE_CONFIG_LOG]]
    SQLITE_CONFIG_LOG
    **
    The SQLITE_CONFIG_LOG option is used to configure the SQLite @@ -1658,10 +1667,11 @@ struct sqlite3_mem_methods { ** function must be threadsafe.
    ** ** [[SQLITE_CONFIG_URI]]
    SQLITE_CONFIG_URI -**
    ^(This option takes a single argument of type int. If non-zero, then -** URI handling is globally enabled. If the parameter is zero, then URI handling -** is globally disabled.)^ ^If URI handling is globally enabled, all filenames -** passed to [sqlite3_open()], [sqlite3_open_v2()], [sqlite3_open16()] or +**
    ^(The SQLITE_CONFIG_URI option takes a single argument of type int. +** If non-zero, then URI handling is globally enabled. If the parameter is zero, +** then URI handling is globally disabled.)^ ^If URI handling is globally +** enabled, all filenames passed to [sqlite3_open()], [sqlite3_open_v2()], +** [sqlite3_open16()] or ** specified as part of [ATTACH] commands are interpreted as URIs, regardless ** of whether or not the [SQLITE_OPEN_URI] flag is set when the database ** connection is opened. ^If it is globally disabled, filenames are @@ -1671,9 +1681,10 @@ struct sqlite3_mem_methods { ** [SQLITE_USE_URI] symbol defined.)^ ** ** [[SQLITE_CONFIG_COVERING_INDEX_SCAN]]
    SQLITE_CONFIG_COVERING_INDEX_SCAN -**
    ^This option takes a single integer argument which is interpreted as -** a boolean in order to enable or disable the use of covering indices for -** full table scans in the query optimizer. ^The default setting is determined +**
    ^The SQLITE_CONFIG_COVERING_INDEX_SCAN option takes a single integer +** argument which is interpreted as a boolean in order to enable or disable +** the use of covering indices for full table scans in the query optimizer. +** ^The default setting is determined ** by the [SQLITE_ALLOW_COVERING_INDEX_SCAN] compile-time option, or is "on" ** if that compile-time option is omitted. ** The ability to disable the use of covering indices for full table scans @@ -1713,19 +1724,28 @@ struct sqlite3_mem_methods { ** ^The default setting can be overridden by each database connection using ** either the [PRAGMA mmap_size] command, or by using the ** [SQLITE_FCNTL_MMAP_SIZE] file control. ^(The maximum allowed mmap size -** cannot be changed at run-time. Nor may the maximum allowed mmap size -** exceed the compile-time maximum mmap size set by the +** will be silently truncated if necessary so that it does not exceed the +** compile-time maximum mmap size set by the ** [SQLITE_MAX_MMAP_SIZE] compile-time option.)^ ** ^If either argument to this option is negative, then that argument is ** changed to its compile-time default. ** ** [[SQLITE_CONFIG_WIN32_HEAPSIZE]] **
    SQLITE_CONFIG_WIN32_HEAPSIZE -**
    ^This option is only available if SQLite is compiled for Windows -** with the [SQLITE_WIN32_MALLOC] pre-processor macro defined. -** SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit unsigned integer value +**
    ^The SQLITE_CONFIG_WIN32_HEAPSIZE option is only available if SQLite is +** compiled for Windows with the [SQLITE_WIN32_MALLOC] pre-processor macro +** defined. ^SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit unsigned integer value ** that specifies the maximum size of the created heap. ** +** +** [[SQLITE_CONFIG_PCACHE_HDRSZ]] +**
    SQLITE_CONFIG_PCACHE_HDRSZ +**
    ^The SQLITE_CONFIG_PCACHE_HDRSZ option takes a single parameter which +** is a pointer to an integer and writes into that integer the number of extra +** bytes per page required for each page in [SQLITE_CONFIG_PAGECACHE]. +** The amount of extra space required can change depending on the compiler, +** target platform, and SQLite version. +** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ #define SQLITE_CONFIG_MULTITHREAD 2 /* nil */ @@ -1750,6 +1770,7 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ #define SQLITE_CONFIG_MMAP_SIZE 22 /* sqlite3_int64, sqlite3_int64 */ #define SQLITE_CONFIG_WIN32_HEAPSIZE 23 /* int nByte */ +#define SQLITE_CONFIG_PCACHE_HDRSZ 24 /* int *psz */ /* ** CAPI3REF: Database Connection Configuration Options @@ -1877,47 +1898,45 @@ sqlite3_int64 sqlite3_last_insert_rowid(sqlite3*); /* ** CAPI3REF: Count The Number Of Rows Modified ** -** ^This function returns the number of database rows that were changed -** or inserted or deleted by the most recently completed SQL statement -** on the [database connection] specified by the first parameter. -** ^(Only changes that are directly specified by the [INSERT], [UPDATE], -** or [DELETE] statement are counted. Auxiliary changes caused by -** triggers or [foreign key actions] are not counted.)^ Use the -** [sqlite3_total_changes()] function to find the total number of changes -** including changes caused by triggers and foreign key actions. +** ^This function returns the number of rows modified, inserted or +** deleted by the most recently completed INSERT, UPDATE or DELETE +** statement on the database connection specified by the only parameter. +** ^Executing any other type of SQL statement does not modify the value +** returned by this function. ** -** ^Changes to a view that are simulated by an [INSTEAD OF trigger] -** are not counted. Only real table changes are counted. +** ^Only changes made directly by the INSERT, UPDATE or DELETE statement are +** considered - auxiliary changes caused by [CREATE TRIGGER | triggers], +** [foreign key actions] or [REPLACE] constraint resolution are not counted. +** +** Changes to a view that are intercepted by +** [INSTEAD OF trigger | INSTEAD OF triggers] are not counted. ^The value +** returned by sqlite3_changes() immediately after an INSERT, UPDATE or +** DELETE statement run on a view is always zero. Only changes made to real +** tables are counted. ** -** ^(A "row change" is a change to a single row of a single table -** caused by an INSERT, DELETE, or UPDATE statement. Rows that -** are changed as side effects of [REPLACE] constraint resolution, -** rollback, ABORT processing, [DROP TABLE], or by any other -** mechanisms do not count as direct row changes.)^ -** -** A "trigger context" is a scope of execution that begins and -** ends with the script of a [CREATE TRIGGER | trigger]. -** Most SQL statements are -** evaluated outside of any trigger. This is the "top level" -** trigger context. If a trigger fires from the top level, a -** new trigger context is entered for the duration of that one -** trigger. Subtriggers create subcontexts for their duration. -** -** ^Calling [sqlite3_exec()] or [sqlite3_step()] recursively does -** not create a new trigger context. -** -** ^This function returns the number of direct row changes in the -** most recent INSERT, UPDATE, or DELETE statement within the same -** trigger context. -** -** ^Thus, when called from the top level, this function returns the -** number of changes in the most recent INSERT, UPDATE, or DELETE -** that also occurred at the top level. ^(Within the body of a trigger, -** the sqlite3_changes() interface can be called to find the number of -** changes in the most recently completed INSERT, UPDATE, or DELETE -** statement within the body of the same trigger. -** However, the number returned does not include changes -** caused by subtriggers since those have their own context.)^ +** Things are more complicated if the sqlite3_changes() function is +** executed while a trigger program is running. This may happen if the +** program uses the [changes() SQL function], or if some other callback +** function invokes sqlite3_changes() directly. Essentially: +** +**
      +**
    • ^(Before entering a trigger program the value returned by +** sqlite3_changes() function is saved. After the trigger program +** has finished, the original value is restored.)^ +** +**
    • ^(Within a trigger program each INSERT, UPDATE and DELETE +** statement sets the value returned by sqlite3_changes() +** upon completion as normal. Of course, this value will not include +** any changes performed by sub-triggers, as the sqlite3_changes() +** value will be saved and restored after each sub-trigger has run.)^ +**
    +** +** ^This means that if the changes() SQL function (or similar) is used +** by the first INSERT, UPDATE or DELETE statement within a trigger, it +** returns the value as set when the calling statement began executing. +** ^If it is used by the second or subsequent such statement within a trigger +** program, the value returned reflects the number of rows modified by the +** previous INSERT, UPDATE or DELETE statement within the same trigger. ** ** See also the [sqlite3_total_changes()] interface, the ** [count_changes pragma], and the [changes() SQL function]. @@ -1931,20 +1950,17 @@ int sqlite3_changes(sqlite3*); /* ** CAPI3REF: Total Number Of Rows Modified ** -** ^This function returns the number of row changes caused by [INSERT], -** [UPDATE] or [DELETE] statements since the [database connection] was opened. -** ^(The count returned by sqlite3_total_changes() includes all changes -** from all [CREATE TRIGGER | trigger] contexts and changes made by -** [foreign key actions]. However, -** the count does not include changes used to implement [REPLACE] constraints, -** do rollbacks or ABORT processing, or [DROP TABLE] processing. The -** count does not include rows of views that fire an [INSTEAD OF trigger], -** though if the INSTEAD OF trigger makes changes of its own, those changes -** are counted.)^ -** ^The sqlite3_total_changes() function counts the changes as soon as -** the statement that makes them is completed (when the statement handle -** is passed to [sqlite3_reset()] or [sqlite3_finalize()]). -** +** ^This function returns the total number of rows inserted, modified or +** deleted by all [INSERT], [UPDATE] or [DELETE] statements completed +** since the database connection was opened, including those executed as +** part of trigger programs. ^Executing any other type of SQL statement +** does not affect the value returned by sqlite3_total_changes(). +** +** ^Changes made as part of [foreign key actions] are included in the +** count, but those made as part of REPLACE constraint resolution are +** not. ^Changes to a view that are intercepted by INSTEAD OF triggers +** are not counted. +** ** See also the [sqlite3_changes()] interface, the ** [count_changes pragma], and the [total_changes() SQL function]. ** @@ -2031,28 +2047,35 @@ int sqlite3_complete16(const void *sql); /* ** CAPI3REF: Register A Callback To Handle SQLITE_BUSY Errors +** KEYWORDS: {busy-handler callback} {busy handler} ** -** ^This routine sets a callback function that might be invoked whenever -** an attempt is made to open a database table that another thread -** or process has locked. +** ^The sqlite3_busy_handler(D,X,P) routine sets a callback function X +** that might be invoked with argument P whenever +** an attempt is made to access a database table associated with +** [database connection] D when another thread +** or process has the table locked. +** The sqlite3_busy_handler() interface is used to implement +** [sqlite3_busy_timeout()] and [PRAGMA busy_timeout]. ** -** ^If the busy callback is NULL, then [SQLITE_BUSY] or [SQLITE_IOERR_BLOCKED] +** ^If the busy callback is NULL, then [SQLITE_BUSY] ** is returned immediately upon encountering the lock. ^If the busy callback ** is not NULL, then the callback might be invoked with two arguments. ** ** ^The first argument to the busy handler is a copy of the void* pointer which ** is the third argument to sqlite3_busy_handler(). ^The second argument to ** the busy handler callback is the number of times that the busy handler has -** been invoked for this locking event. ^If the +** been invoked previously for the same locking event. ^If the ** busy callback returns 0, then no additional attempts are made to -** access the database and [SQLITE_BUSY] or [SQLITE_IOERR_BLOCKED] is returned. +** access the database and [SQLITE_BUSY] is returned +** to the application. ** ^If the callback returns non-zero, then another attempt -** is made to open the database for reading and the cycle repeats. +** is made to access the database and the cycle repeats. ** ** The presence of a busy handler does not guarantee that it will be invoked ** when there is lock contention. ^If SQLite determines that invoking the busy ** handler could result in a deadlock, it will go ahead and return [SQLITE_BUSY] -** or [SQLITE_IOERR_BLOCKED] instead of invoking the busy handler. +** to the application instead of invoking the +** busy handler. ** Consider a scenario where one process is holding a read lock that ** it is trying to promote to a reserved lock and ** a second process is holding a reserved lock that it is trying @@ -2066,28 +2089,15 @@ int sqlite3_complete16(const void *sql); ** ** ^The default busy callback is NULL. ** -** ^The [SQLITE_BUSY] error is converted to [SQLITE_IOERR_BLOCKED] -** when SQLite is in the middle of a large transaction where all the -** changes will not fit into the in-memory cache. SQLite will -** already hold a RESERVED lock on the database file, but it needs -** to promote this lock to EXCLUSIVE so that it can spill cache -** pages into the database file without harm to concurrent -** readers. ^If it is unable to promote the lock, then the in-memory -** cache will be left in an inconsistent state and so the error -** code is promoted from the relatively benign [SQLITE_BUSY] to -** the more severe [SQLITE_IOERR_BLOCKED]. ^This error code promotion -** forces an automatic rollback of the changes. See the -** -** CorruptionFollowingBusyError wiki page for a discussion of why -** this is important. -** ** ^(There can only be a single busy handler defined for each ** [database connection]. Setting a new busy handler clears any ** previously set handler.)^ ^Note that calling [sqlite3_busy_timeout()] -** will also set or clear the busy handler. +** or evaluating [PRAGMA busy_timeout=N] will change the +** busy handler and thus clear any previously set busy handler. ** ** The busy callback should not take any actions which modify the -** database connection that invoked the busy handler. Any such actions +** database connection that invoked the busy handler. In other words, +** the busy handler is not reentrant. Any such actions ** result in undefined behavior. ** ** A busy handler must not close the database connection @@ -2103,15 +2113,17 @@ int sqlite3_busy_handler(sqlite3*, int(*)(void*,int), void*); ** will sleep multiple times until at least "ms" milliseconds of sleeping ** have accumulated. ^After at least "ms" milliseconds of sleeping, ** the handler returns 0 which causes [sqlite3_step()] to return -** [SQLITE_BUSY] or [SQLITE_IOERR_BLOCKED]. +** [SQLITE_BUSY]. ** ** ^Calling this routine with an argument less than or equal to zero ** turns off all busy handlers. ** ** ^(There can only be a single busy handler for a particular -** [database connection] any any given moment. If another busy handler +** [database connection] at any given moment. If another busy handler ** was defined (using [sqlite3_busy_handler()]) prior to calling ** this routine, that other busy handler is cleared.)^ +** +** See also: [PRAGMA busy_timeout] */ int sqlite3_busy_timeout(sqlite3*, int ms); @@ -2311,6 +2323,10 @@ char *sqlite3_vsnprintf(int,char*,const char*, va_list); ** sqlite3_malloc() is zero or negative then sqlite3_malloc() returns ** a NULL pointer. ** +** ^The sqlite3_malloc64(N) routine works just like +** sqlite3_malloc(N) except that N is an unsigned 64-bit integer instead +** of a signed 32-bit integer. +** ** ^Calling sqlite3_free() with a pointer previously returned ** by sqlite3_malloc() or sqlite3_realloc() releases that memory so ** that it might be reused. ^The sqlite3_free() routine is @@ -2322,24 +2338,38 @@ char *sqlite3_vsnprintf(int,char*,const char*, va_list); ** might result if sqlite3_free() is called with a non-NULL pointer that ** was not obtained from sqlite3_malloc() or sqlite3_realloc(). ** -** ^(The sqlite3_realloc() interface attempts to resize a -** prior memory allocation to be at least N bytes, where N is the -** second parameter. The memory allocation to be resized is the first -** parameter.)^ ^ If the first parameter to sqlite3_realloc() +** ^The sqlite3_realloc(X,N) interface attempts to resize a +** prior memory allocation X to be at least N bytes. +** ^If the X parameter to sqlite3_realloc(X,N) ** is a NULL pointer then its behavior is identical to calling -** sqlite3_malloc(N) where N is the second parameter to sqlite3_realloc(). -** ^If the second parameter to sqlite3_realloc() is zero or +** sqlite3_malloc(N). +** ^If the N parameter to sqlite3_realloc(X,N) is zero or ** negative then the behavior is exactly the same as calling -** sqlite3_free(P) where P is the first parameter to sqlite3_realloc(). -** ^sqlite3_realloc() returns a pointer to a memory allocation -** of at least N bytes in size or NULL if sufficient memory is unavailable. +** sqlite3_free(X). +** ^sqlite3_realloc(X,N) returns a pointer to a memory allocation +** of at least N bytes in size or NULL if insufficient memory is available. ** ^If M is the size of the prior allocation, then min(N,M) bytes ** of the prior allocation are copied into the beginning of buffer returned -** by sqlite3_realloc() and the prior allocation is freed. -** ^If sqlite3_realloc() returns NULL, then the prior allocation -** is not freed. +** by sqlite3_realloc(X,N) and the prior allocation is freed. +** ^If sqlite3_realloc(X,N) returns NULL and N is positive, then the +** prior allocation is not freed. ** -** ^The memory returned by sqlite3_malloc() and sqlite3_realloc() +** ^The sqlite3_realloc64(X,N) interfaces works the same as +** sqlite3_realloc(X,N) except that N is a 64-bit unsigned integer instead +** of a 32-bit signed integer. +** +** ^If X is a memory allocation previously obtained from sqlite3_malloc(), +** sqlite3_malloc64(), sqlite3_realloc(), or sqlite3_realloc64(), then +** sqlite3_msize(X) returns the size of that memory allocation in bytes. +** ^The value returned by sqlite3_msize(X) might be larger than the number +** of bytes requested when X was allocated. ^If X is a NULL pointer then +** sqlite3_msize(X) returns zero. If X points to something that is not +** the beginning of memory allocation, or if it points to a formerly +** valid memory allocation that has now been freed, then the behavior +** of sqlite3_msize(X) is undefined and possibly harmful. +** +** ^The memory returned by sqlite3_malloc(), sqlite3_realloc(), +** sqlite3_malloc64(), and sqlite3_realloc64() ** is always aligned to at least an 8 byte boundary, or to a ** 4 byte boundary if the [SQLITE_4_BYTE_ALIGNED_MALLOC] compile-time ** option is used. @@ -2367,8 +2397,11 @@ char *sqlite3_vsnprintf(int,char*,const char*, va_list); ** [sqlite3_free()] or [sqlite3_realloc()]. */ void *sqlite3_malloc(int); +void *sqlite3_malloc64(sqlite3_uint64); void *sqlite3_realloc(void*, int); +void *sqlite3_realloc64(void*, sqlite3_uint64); void sqlite3_free(void*); +sqlite3_uint64 sqlite3_msize(void*); /* ** CAPI3REF: Memory Allocator Statistics @@ -2406,13 +2439,14 @@ sqlite3_int64 sqlite3_memory_highwater(int resetFlag); ** applications to access the same PRNG for other purposes. ** ** ^A call to this routine stores N bytes of randomness into buffer P. -** ^If N is less than one, then P can be a NULL pointer. +** ^The P parameter can be a NULL pointer. ** ** ^If this routine has not been previously called or if the previous -** call had N less than one, then the PRNG is seeded using randomness -** obtained from the xRandomness method of the default [sqlite3_vfs] object. -** ^If the previous call to this routine had an N of 1 or more then -** the pseudo-randomness is generated +** call had N less than one or a NULL pointer for P, then the PRNG is +** seeded using randomness obtained from the xRandomness method of +** the default [sqlite3_vfs] object. +** ^If the previous call to this routine had an N of 1 or more and a +** non-NULL P then the pseudo-randomness is generated ** internally and without recourse to the [sqlite3_vfs] xRandomness ** method. */ @@ -2513,8 +2547,8 @@ int sqlite3_set_authorizer( ** [sqlite3_set_authorizer | authorizer documentation] for additional ** information. ** -** Note that SQLITE_IGNORE is also used as a [SQLITE_ROLLBACK | return code] -** from the [sqlite3_vtab_on_conflict()] interface. +** Note that SQLITE_IGNORE is also used as a [conflict resolution mode] +** returned from the [sqlite3_vtab_on_conflict()] interface. */ #define SQLITE_DENY 1 /* Abort the SQL statement with an error */ #define SQLITE_IGNORE 2 /* Don't allow access, but don't generate an error */ @@ -2655,9 +2689,9 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** an English language description of the error following a failure of any ** of the sqlite3_open() routines. ** -** ^The default encoding for the database will be UTF-8 if -** sqlite3_open() or sqlite3_open_v2() is called and -** UTF-16 in the native byte order if sqlite3_open16() is used. +** ^The default encoding will be UTF-8 for databases created using +** sqlite3_open() or sqlite3_open_v2(). ^The default encoding for databases +** created using sqlite3_open16() will be UTF-16 in the native byte order. ** ** Whether or not an error occurs when it is opened, resources ** associated with the [database connection] handle should be released by @@ -2745,13 +2779,14 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** then it is interpreted as an absolute path. ^If the path does not begin ** with a '/' (meaning that the authority section is omitted from the URI) ** then the path is interpreted as a relative path. -** ^On windows, the first component of an absolute path -** is a drive specification (e.g. "C:"). +** ^(On windows, the first component of an absolute path +** is a drive specification (e.g. "C:").)^ ** ** [[core URI query parameters]] ** The query component of a URI may contain parameters that are interpreted ** either by SQLite itself, or by a [VFS | custom VFS implementation]. -** SQLite interprets the following three query parameters: +** SQLite and its built-in [VFSes] interpret the +** following query parameters: ** **
      **
    • vfs: ^The "vfs" parameter may be used to specify the name of @@ -2786,11 +2821,9 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** a URI filename, its value overrides any behavior requested by setting ** SQLITE_OPEN_PRIVATECACHE or SQLITE_OPEN_SHAREDCACHE flag. ** -**
    • psow: ^The psow parameter may be "true" (or "on" or "yes" or -** "1") or "false" (or "off" or "no" or "0") to indicate that the +**
    • psow: ^The psow parameter indicates whether or not the ** [powersafe overwrite] property does or does not apply to the -** storage media on which the database file resides. ^The psow query -** parameter only works for the built-in unix and Windows VFSes. +** storage media on which the database file resides. ** **
    • nolock: ^The nolock parameter is a boolean query parameter ** which if set disables file locking in rollback journal modes. This @@ -3086,6 +3119,10 @@ int sqlite3_limit(sqlite3*, int id, int newVal); ** ** [[SQLITE_LIMIT_TRIGGER_DEPTH]] ^(
      SQLITE_LIMIT_TRIGGER_DEPTH
      **
      The maximum depth of recursion for triggers.
      )^ +** +** [[SQLITE_LIMIT_WORKER_THREADS]] ^(
      SQLITE_LIMIT_WORKER_THREADS
      +**
      The maximum number of auxiliary worker threads that a single +** [prepared statement] may start.
      )^ ** */ #define SQLITE_LIMIT_LENGTH 0 @@ -3099,6 +3136,7 @@ int sqlite3_limit(sqlite3*, int id, int newVal); #define SQLITE_LIMIT_LIKE_PATTERN_LENGTH 8 #define SQLITE_LIMIT_VARIABLE_NUMBER 9 #define SQLITE_LIMIT_TRIGGER_DEPTH 10 +#define SQLITE_LIMIT_WORKER_THREADS 11 /* ** CAPI3REF: Compiling An SQL Statement @@ -3372,18 +3410,18 @@ typedef struct sqlite3_context sqlite3_context; ** If the fourth parameter to sqlite3_bind_blob() is negative, then ** the behavior is undefined. ** If a non-negative fourth parameter is provided to sqlite3_bind_text() -** or sqlite3_bind_text16() then that parameter must be the byte offset +** or sqlite3_bind_text16() or sqlite3_bind_text64() then +** that parameter must be the byte offset ** where the NUL terminator would occur assuming the string were NUL ** terminated. If any NUL characters occur at byte offsets less than ** the value of the fourth parameter then the resulting string value will ** contain embedded NULs. The result of expressions involving strings ** with embedded NULs is undefined. ** -** ^The fifth argument to sqlite3_bind_blob(), sqlite3_bind_text(), and -** sqlite3_bind_text16() is a destructor used to dispose of the BLOB or +** ^The fifth argument to the BLOB and string binding interfaces +** is a destructor used to dispose of the BLOB or ** string after SQLite has finished with it. ^The destructor is called -** to dispose of the BLOB or string even if the call to sqlite3_bind_blob(), -** sqlite3_bind_text(), or sqlite3_bind_text16() fails. +** to dispose of the BLOB or string even if the call to bind API fails. ** ^If the fifth argument is ** the special value [SQLITE_STATIC], then SQLite assumes that the ** information is in static, unmanaged space and does not need to be freed. @@ -3391,6 +3429,14 @@ typedef struct sqlite3_context sqlite3_context; ** SQLite makes its own private copy of the data immediately, before ** the sqlite3_bind_*() routine returns. ** +** ^The sixth argument to sqlite3_bind_text64() must be one of +** [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE] +** to specify the encoding of the text in the third parameter. If +** the sixth argument to sqlite3_bind_text64() is not one of the +** allowed values shown above, or if the text encoding is different +** from the encoding specified by the sixth parameter, then the behavior +** is undefined. +** ** ^The sqlite3_bind_zeroblob() routine binds a BLOB of length N that ** is filled with zeroes. ^A zeroblob uses a fixed amount of memory ** (just an integer to hold its size) while it is being processed. @@ -3411,6 +3457,9 @@ typedef struct sqlite3_context sqlite3_context; ** ** ^The sqlite3_bind_* routines return [SQLITE_OK] on success or an ** [error code] if anything goes wrong. +** ^[SQLITE_TOOBIG] might be returned if the size of a string or BLOB +** exceeds limits imposed by [sqlite3_limit]([SQLITE_LIMIT_LENGTH]) or +** [SQLITE_MAX_LENGTH]. ** ^[SQLITE_RANGE] is returned if the parameter ** index is out of range. ^[SQLITE_NOMEM] is returned if malloc() fails. ** @@ -3418,12 +3467,16 @@ typedef struct sqlite3_context sqlite3_context; ** [sqlite3_bind_parameter_name()], and [sqlite3_bind_parameter_index()]. */ int sqlite3_bind_blob(sqlite3_stmt*, int, const void*, int n, void(*)(void*)); +int sqlite3_bind_blob64(sqlite3_stmt*, int, const void*, sqlite3_uint64, + void(*)(void*)); int sqlite3_bind_double(sqlite3_stmt*, int, double); int sqlite3_bind_int(sqlite3_stmt*, int, int); int sqlite3_bind_int64(sqlite3_stmt*, int, sqlite3_int64); int sqlite3_bind_null(sqlite3_stmt*, int); -int sqlite3_bind_text(sqlite3_stmt*, int, const char*, int n, void(*)(void*)); +int sqlite3_bind_text(sqlite3_stmt*,int,const char*,int,void(*)(void*)); int sqlite3_bind_text16(sqlite3_stmt*, int, const void*, int, void(*)(void*)); +int sqlite3_bind_text64(sqlite3_stmt*, int, const char*, sqlite3_uint64, + void(*)(void*), unsigned char encoding); int sqlite3_bind_value(sqlite3_stmt*, int, const sqlite3_value*); int sqlite3_bind_zeroblob(sqlite3_stmt*, int, int n); @@ -4115,9 +4168,9 @@ int sqlite3_create_function_v2( ** These constant define integer codes that represent the various ** text encodings supported by SQLite. */ -#define SQLITE_UTF8 1 -#define SQLITE_UTF16LE 2 -#define SQLITE_UTF16BE 3 +#define SQLITE_UTF8 1 /* IMP: R-37514-35566 */ +#define SQLITE_UTF16LE 2 /* IMP: R-03371-37637 */ +#define SQLITE_UTF16BE 3 /* IMP: R-51971-34154 */ #define SQLITE_UTF16 4 /* Use native byte order */ #define SQLITE_ANY 5 /* Deprecated */ #define SQLITE_UTF16_ALIGNED 8 /* sqlite3_create_collation only */ @@ -4172,7 +4225,7 @@ SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int), ** object results in undefined behavior. ** ** ^These routines work just like the corresponding [column access functions] -** except that these routines take a single [protected sqlite3_value] object +** except that these routines take a single [protected sqlite3_value] object ** pointer instead of a [sqlite3_stmt*] pointer and an integer column number. ** ** ^The sqlite3_value_text16() interface extracts a UTF-16 string @@ -4419,6 +4472,10 @@ typedef void (*sqlite3_destructor_type)(void*); ** set the return value of the application-defined function to be ** a text string which is represented as UTF-8, UTF-16 native byte order, ** UTF-16 little endian, or UTF-16 big endian, respectively. +** ^The sqlite3_result_text64() interface sets the return value of an +** application-defined function to be a text string in an encoding +** specified by the fifth (and last) parameter, which must be one +** of [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE]. ** ^SQLite takes the text result from the application from ** the 2nd parameter of the sqlite3_result_text* interfaces. ** ^If the 3rd parameter to the sqlite3_result_text* interfaces @@ -4462,6 +4519,8 @@ typedef void (*sqlite3_destructor_type)(void*); ** the [sqlite3_context] pointer, the results are undefined. */ void sqlite3_result_blob(sqlite3_context*, const void*, int, void(*)(void*)); +void sqlite3_result_blob64(sqlite3_context*,const void*, + sqlite3_uint64,void(*)(void*)); void sqlite3_result_double(sqlite3_context*, double); void sqlite3_result_error(sqlite3_context*, const char*, int); void sqlite3_result_error16(sqlite3_context*, const void*, int); @@ -4472,6 +4531,8 @@ void sqlite3_result_int(sqlite3_context*, int); void sqlite3_result_int64(sqlite3_context*, sqlite3_int64); void sqlite3_result_null(sqlite3_context*); void sqlite3_result_text(sqlite3_context*, const char*, int, void(*)(void*)); +void sqlite3_result_text64(sqlite3_context*, const char*,sqlite3_uint64, + void(*)(void*), unsigned char encoding); void sqlite3_result_text16(sqlite3_context*, const void*, int, void(*)(void*)); void sqlite3_result_text16le(sqlite3_context*, const void*, int,void(*)(void*)); void sqlite3_result_text16be(sqlite3_context*, const void*, int,void(*)(void*)); @@ -4701,6 +4762,13 @@ int sqlite3_sleep(int); ** is a NULL pointer, then SQLite performs a search for an appropriate ** temporary file directory. ** +** Applications are strongly discouraged from using this global variable. +** It is required to set a temporary folder on Windows Runtime (WinRT). +** But for all other platforms, it is highly recommended that applications +** neither read nor write this variable. This global variable is a relic +** that exists for backwards compatibility of legacy applications and should +** be avoided in new projects. +** ** It is not safe to read or modify this variable in more than one ** thread at a time. It is not safe to read or modify this variable ** if a [database connection] is being used at the same time in a separate @@ -4719,6 +4787,11 @@ int sqlite3_sleep(int); ** Hence, if this variable is modified directly, either it should be ** made NULL or made to point to memory obtained from [sqlite3_malloc] ** or else the use of the [temp_store_directory pragma] should be avoided. +** Except when requested by the [temp_store_directory pragma], SQLite +** does not free the memory that sqlite3_temp_directory points to. If +** the application wants that memory to be freed, it must do +** so itself, taking care to only do so after all [database connection] +** objects have been destroyed. ** ** Note to Windows Runtime users: The temporary directory must be set ** prior to calling [sqlite3_open] or [sqlite3_open_v2]. Otherwise, various @@ -5079,20 +5152,27 @@ SQLITE_DEPRECATED void sqlite3_soft_heap_limit(int N); /* ** CAPI3REF: Extract Metadata About A Column Of A Table ** -** ^This routine returns metadata about a specific column of a specific -** database table accessible using the [database connection] handle -** passed as the first function argument. +** ^(The sqlite3_table_column_metadata(X,D,T,C,....) routine returns +** information about column C of table T in database D +** on [database connection] X.)^ ^The sqlite3_table_column_metadata() +** interface returns SQLITE_OK and fills in the non-NULL pointers in +** the final five arguments with appropriate values if the specified +** column exists. ^The sqlite3_table_column_metadata() interface returns +** SQLITE_ERROR and if the specified column does not exist. +** ^If the column-name parameter to sqlite3_table_column_metadata() is a +** NULL pointer, then this routine simply checks for the existance of the +** table and returns SQLITE_OK if the table exists and SQLITE_ERROR if it +** does not. ** ** ^The column is identified by the second, third and fourth parameters to -** this function. ^The second parameter is either the name of the database +** this function. ^(The second parameter is either the name of the database ** (i.e. "main", "temp", or an attached database) containing the specified -** table or NULL. ^If it is NULL, then all attached databases are searched +** table or NULL.)^ ^If it is NULL, then all attached databases are searched ** for the table using the same algorithm used by the database engine to ** resolve unqualified table references. ** ** ^The third and fourth parameters to this function are the table and column -** name of the desired column, respectively. Neither of these parameters -** may be NULL. +** name of the desired column, respectively. ** ** ^Metadata is returned by writing to the memory locations passed as the 5th ** and subsequent parameters to this function. ^Any of these arguments may be @@ -5111,16 +5191,17 @@ SQLITE_DEPRECATED void sqlite3_soft_heap_limit(int N); ** )^ ** ** ^The memory pointed to by the character pointers returned for the -** declaration type and collation sequence is valid only until the next +** declaration type and collation sequence is valid until the next ** call to any SQLite API function. ** ** ^If the specified table is actually a view, an [error code] is returned. ** -** ^If the specified column is "rowid", "oid" or "_rowid_" and an +** ^If the specified column is "rowid", "oid" or "_rowid_" and the table +** is not a [WITHOUT ROWID] table and an ** [INTEGER PRIMARY KEY] column has been explicitly declared, then the output ** parameters are set for the explicitly declared column. ^(If there is no -** explicitly declared [INTEGER PRIMARY KEY] column, then the output -** parameters are set as follows: +** [INTEGER PRIMARY KEY] column, then the outputs +** for the [rowid] are set as follows: ** ** 
       **     data type: "INTEGER"
@@ -5130,13 +5211,9 @@ SQLITE_DEPRECATED void sqlite3_soft_heap_limit(int N);
 **     auto increment: 0
 **
      )^ ** -** ^(This function may load one or more schemas from database files. If an -** error occurs during this process, or if the requested table or column -** cannot be found, an [error code] is returned and an error message left -** in the [database connection] (to be retrieved using sqlite3_errmsg()).)^ -** -** ^This API is only available if the library was compiled with the -** [SQLITE_ENABLE_COLUMN_METADATA] C-preprocessor symbol defined. +** ^This function causes all database schemas to be read from disk and +** parsed, if that has not already been done, and returns an error if +** any errors are encountered while loading the schema. */ int sqlite3_table_column_metadata( sqlite3 *db, /* Connection handle */ @@ -5589,26 +5666,42 @@ typedef struct sqlite3_blob sqlite3_blob; ** SELECT zColumn FROM zDb.zTable WHERE [rowid] = iRow; ** )^ ** +** ^(Parameter zDb is not the filename that contains the database, but +** rather the symbolic name of the database. For attached databases, this is +** the name that appears after the AS keyword in the [ATTACH] statement. +** For the main database file, the database name is "main". For TEMP +** tables, the database name is "temp".)^ +** ** ^If the flags parameter is non-zero, then the BLOB is opened for read -** and write access. ^If it is zero, the BLOB is opened for read access. -** ^It is not possible to open a column that is part of an index or primary -** key for writing. ^If [foreign key constraints] are enabled, it is -** not possible to open a column that is part of a [child key] for writing. +** and write access. ^If the flags parameter is zero, the BLOB is opened for +** read-only access. ** -** ^Note that the database name is not the filename that contains -** the database but rather the symbolic name of the database that -** appears after the AS keyword when the database is connected using [ATTACH]. -** ^For the main database file, the database name is "main". -** ^For TEMP tables, the database name is "temp". +** ^(On success, [SQLITE_OK] is returned and the new [BLOB handle] is stored +** in *ppBlob. Otherwise an [error code] is returned and, unless the error +** code is SQLITE_MISUSE, *ppBlob is set to NULL.)^ ^This means that, provided +** the API is not misused, it is always safe to call [sqlite3_blob_close()] +** on *ppBlob after this function it returns. +** +** This function fails with SQLITE_ERROR if any of the following are true: +**
        +**
      • ^(Database zDb does not exist)^, +**
      • ^(Table zTable does not exist within database zDb)^, +**
      • ^(Table zTable is a WITHOUT ROWID table)^, +**
      • ^(Column zColumn does not exist)^, +**
      • ^(Row iRow is not present in the table)^, +**
      • ^(The specified column of row iRow contains a value that is not +** a TEXT or BLOB value)^, +**
      • ^(Column zColumn is part of an index, PRIMARY KEY or UNIQUE +** constraint and the blob is being opened for read/write access)^, +**
      • ^([foreign key constraints | Foreign key constraints] are enabled, +** column zColumn is part of a [child key] definition and the blob is +** being opened for read/write access)^. +**
      +** +** ^Unless it returns SQLITE_MISUSE, this function sets the +** [database connection] error code and message accessible via +** [sqlite3_errcode()] and [sqlite3_errmsg()] and related functions. ** -** ^(On success, [SQLITE_OK] is returned and the new [BLOB handle] is written -** to *ppBlob. Otherwise an [error code] is returned and *ppBlob is set -** to be a null pointer.)^ -** ^This function sets the [database connection] error code and message -** accessible via [sqlite3_errcode()] and [sqlite3_errmsg()] and related -** functions. ^Note that the *ppBlob variable is always initialized in a -** way that makes it safe to invoke [sqlite3_blob_close()] on *ppBlob -** regardless of the success or failure of this routine. ** ** ^(If the row that a BLOB handle points to is modified by an ** [UPDATE], [DELETE], or by [ON CONFLICT] side-effects @@ -5626,13 +5719,9 @@ typedef struct sqlite3_blob sqlite3_blob; ** interface. Use the [UPDATE] SQL command to change the size of a ** blob. ** -** ^The [sqlite3_blob_open()] interface will fail for a [WITHOUT ROWID] -** table. Incremental BLOB I/O is not possible on [WITHOUT ROWID] tables. -** ** ^The [sqlite3_bind_zeroblob()] and [sqlite3_result_zeroblob()] interfaces -** and the built-in [zeroblob] SQL function can be used, if desired, -** to create an empty, zero-filled blob in which to read or write using -** this interface. +** and the built-in [zeroblob] SQL function may be used to create a +** zero-filled blob to read or write using the incremental-blob interface. ** ** To avoid a resource leak, every open [BLOB handle] should eventually ** be released by a call to [sqlite3_blob_close()]. @@ -5674,24 +5763,22 @@ SQLITE_EXPERIMENTAL int sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64); /* ** CAPI3REF: Close A BLOB Handle ** -** ^Closes an open [BLOB handle]. +** ^This function closes an open [BLOB handle]. ^(The BLOB handle is closed +** unconditionally. Even if this routine returns an error code, the +** handle is still closed.)^ ** -** ^Closing a BLOB shall cause the current transaction to commit -** if there are no other BLOBs, no pending prepared statements, and the -** database connection is in [autocommit mode]. -** ^If any writes were made to the BLOB, they might be held in cache -** until the close operation if they will fit. +** ^If the blob handle being closed was opened for read-write access, and if +** the database is in auto-commit mode and there are no other open read-write +** blob handles or active write statements, the current transaction is +** committed. ^If an error occurs while committing the transaction, an error +** code is returned and the transaction rolled back. ** -** ^(Closing the BLOB often forces the changes -** out to disk and so if any I/O errors occur, they will likely occur -** at the time when the BLOB is closed. Any errors that occur during -** closing are reported as a non-zero return value.)^ -** -** ^(The BLOB is closed unconditionally. Even if this routine returns -** an error code, the BLOB is still closed.)^ -** -** ^Calling this routine with a null pointer (such as would be returned -** by a failed call to [sqlite3_blob_open()]) is a harmless no-op. +** Calling this function with an argument that is not a NULL pointer or an +** open blob handle results in undefined behaviour. ^Calling this routine +** with a null pointer (such as would be returned by a failed call to +** [sqlite3_blob_open()]) is a harmless no-op. ^Otherwise, if this function +** is passed a valid open blob handle, the values returned by the +** sqlite3_errcode() and sqlite3_errmsg() functions are set before returning. */ int sqlite3_blob_close(sqlite3_blob *); @@ -5741,21 +5828,27 @@ int sqlite3_blob_read(sqlite3_blob *, void *Z, int N, int iOffset); /* ** CAPI3REF: Write Data Into A BLOB Incrementally ** -** ^This function is used to write data into an open [BLOB handle] from a -** caller-supplied buffer. ^N bytes of data are copied from the buffer Z -** into the open BLOB, starting at offset iOffset. +** ^(This function is used to write data into an open [BLOB handle] from a +** caller-supplied buffer. N bytes of data are copied from the buffer Z +** into the open BLOB, starting at offset iOffset.)^ +** +** ^(On success, sqlite3_blob_write() returns SQLITE_OK. +** Otherwise, an [error code] or an [extended error code] is returned.)^ +** ^Unless SQLITE_MISUSE is returned, this function sets the +** [database connection] error code and message accessible via +** [sqlite3_errcode()] and [sqlite3_errmsg()] and related functions. ** ** ^If the [BLOB handle] passed as the first argument was not opened for ** writing (the flags parameter to [sqlite3_blob_open()] was zero), ** this function returns [SQLITE_READONLY]. ** -** ^This function may only modify the contents of the BLOB; it is +** This function may only modify the contents of the BLOB; it is ** not possible to increase the size of a BLOB using this API. ** ^If offset iOffset is less than N bytes from the end of the BLOB, -** [SQLITE_ERROR] is returned and no data is written. ^If N is -** less than zero [SQLITE_ERROR] is returned and no data is written. -** The size of the BLOB (and hence the maximum value of N+iOffset) -** can be determined using the [sqlite3_blob_bytes()] interface. +** [SQLITE_ERROR] is returned and no data is written. The size of the +** BLOB (and hence the maximum value of N+iOffset) can be determined +** using the [sqlite3_blob_bytes()] interface. ^If N or iOffset are less +** than zero [SQLITE_ERROR] is returned and no data is written. ** ** ^An attempt to write to an expired [BLOB handle] fails with an ** error code of [SQLITE_ABORT]. ^Writes to the BLOB that occurred @@ -5764,9 +5857,6 @@ int sqlite3_blob_read(sqlite3_blob *, void *Z, int N, int iOffset); ** have been overwritten by the statement that expired the BLOB handle ** or by other independent statements. ** -** ^(On success, sqlite3_blob_write() returns SQLITE_OK. -** Otherwise, an [error code] or an [extended error code] is returned.)^ -** ** This routine only works on a [BLOB handle] which has been created ** by a prior successful call to [sqlite3_blob_open()] and which has not ** been closed by [sqlite3_blob_close()]. Passing any other pointer in @@ -5819,45 +5909,48 @@ int sqlite3_vfs_unregister(sqlite3_vfs*); ** ** The SQLite source code contains multiple implementations ** of these mutex routines. An appropriate implementation -** is selected automatically at compile-time. ^(The following +** is selected automatically at compile-time. The following ** implementations are available in the SQLite core: ** **
        **
      • SQLITE_MUTEX_PTHREADS **
      • SQLITE_MUTEX_W32 **
      • SQLITE_MUTEX_NOOP -**
      )^ +**
    ** -** ^The SQLITE_MUTEX_NOOP implementation is a set of routines +** The SQLITE_MUTEX_NOOP implementation is a set of routines ** that does no real locking and is appropriate for use in -** a single-threaded application. ^The SQLITE_MUTEX_PTHREADS and +** a single-threaded application. The SQLITE_MUTEX_PTHREADS and ** SQLITE_MUTEX_W32 implementations are appropriate for use on Unix ** and Windows. ** -** ^(If SQLite is compiled with the SQLITE_MUTEX_APPDEF preprocessor +** If SQLite is compiled with the SQLITE_MUTEX_APPDEF preprocessor ** macro defined (with "-DSQLITE_MUTEX_APPDEF=1"), then no mutex ** implementation is included with the library. In this case the ** application must supply a custom mutex implementation using the ** [SQLITE_CONFIG_MUTEX] option of the sqlite3_config() function ** before calling sqlite3_initialize() or any other public sqlite3_ -** function that calls sqlite3_initialize().)^ +** function that calls sqlite3_initialize(). ** ** ^The sqlite3_mutex_alloc() routine allocates a new -** mutex and returns a pointer to it. ^If it returns NULL -** that means that a mutex could not be allocated. ^SQLite -** will unwind its stack and return an error. ^(The argument -** to sqlite3_mutex_alloc() is one of these integer constants: +** mutex and returns a pointer to it. ^The sqlite3_mutex_alloc() +** routine returns NULL if it is unable to allocate the requested +** mutex. The argument to sqlite3_mutex_alloc() must one of these +** integer constants: ** **
      **
    • SQLITE_MUTEX_FAST **
    • SQLITE_MUTEX_RECURSIVE **
    • SQLITE_MUTEX_STATIC_MASTER **
    • SQLITE_MUTEX_STATIC_MEM -**
    • SQLITE_MUTEX_STATIC_MEM2 +**
    • SQLITE_MUTEX_STATIC_OPEN **
    • SQLITE_MUTEX_STATIC_PRNG **
    • SQLITE_MUTEX_STATIC_LRU -**
    • SQLITE_MUTEX_STATIC_LRU2 -**
    )^ +**
  • SQLITE_MUTEX_STATIC_PMEM +**
  • SQLITE_MUTEX_STATIC_APP1 +**
  • SQLITE_MUTEX_STATIC_APP2 +**
  • SQLITE_MUTEX_STATIC_APP3 +** ** ** ^The first two constants (SQLITE_MUTEX_FAST and SQLITE_MUTEX_RECURSIVE) ** cause sqlite3_mutex_alloc() to create @@ -5865,14 +5958,14 @@ int sqlite3_vfs_unregister(sqlite3_vfs*); ** is used but not necessarily so when SQLITE_MUTEX_FAST is used. ** The mutex implementation does not need to make a distinction ** between SQLITE_MUTEX_RECURSIVE and SQLITE_MUTEX_FAST if it does -** not want to. ^SQLite will only request a recursive mutex in -** cases where it really needs one. ^If a faster non-recursive mutex +** not want to. SQLite will only request a recursive mutex in +** cases where it really needs one. If a faster non-recursive mutex ** implementation is available on the host platform, the mutex subsystem ** might return such a mutex in response to SQLITE_MUTEX_FAST. ** ** ^The other allowed parameters to sqlite3_mutex_alloc() (anything other ** than SQLITE_MUTEX_FAST and SQLITE_MUTEX_RECURSIVE) each return -** a pointer to a static preexisting mutex. ^Six static mutexes are +** a pointer to a static preexisting mutex. ^Nine static mutexes are ** used by the current version of SQLite. Future versions of SQLite ** may add additional static mutexes. Static mutexes are for internal ** use by SQLite only. Applications that use SQLite mutexes should @@ -5881,16 +5974,13 @@ int sqlite3_vfs_unregister(sqlite3_vfs*); ** ** ^Note that if one of the dynamic mutex parameters (SQLITE_MUTEX_FAST ** or SQLITE_MUTEX_RECURSIVE) is used then sqlite3_mutex_alloc() -** returns a different mutex on every call. ^But for the static +** returns a different mutex on every call. ^For the static ** mutex types, the same mutex is returned on every call that has ** the same type number. ** ** ^The sqlite3_mutex_free() routine deallocates a previously -** allocated dynamic mutex. ^SQLite is careful to deallocate every -** dynamic mutex that it allocates. The dynamic mutexes must not be in -** use when they are deallocated. Attempting to deallocate a static -** mutex results in undefined behavior. ^SQLite never deallocates -** a static mutex. +** allocated dynamic mutex. Attempting to deallocate a static +** mutex results in undefined behavior. ** ** ^The sqlite3_mutex_enter() and sqlite3_mutex_try() routines attempt ** to enter a mutex. ^If another thread is already within the mutex, @@ -5898,23 +5988,21 @@ int sqlite3_vfs_unregister(sqlite3_vfs*); ** SQLITE_BUSY. ^The sqlite3_mutex_try() interface returns [SQLITE_OK] ** upon successful entry. ^(Mutexes created using ** SQLITE_MUTEX_RECURSIVE can be entered multiple times by the same thread. -** In such cases the, +** In such cases, the ** mutex must be exited an equal number of times before another thread -** can enter.)^ ^(If the same thread tries to enter any other -** kind of mutex more than once, the behavior is undefined. -** SQLite will never exhibit -** such behavior in its own use of mutexes.)^ +** can enter.)^ If the same thread tries to enter any mutex other +** than an SQLITE_MUTEX_RECURSIVE more than once, the behavior is undefined. ** ** ^(Some systems (for example, Windows 95) do not support the operation ** implemented by sqlite3_mutex_try(). On those systems, sqlite3_mutex_try() -** will always return SQLITE_BUSY. The SQLite core only ever uses -** sqlite3_mutex_try() as an optimization so this is acceptable behavior.)^ +** will always return SQLITE_BUSY. The SQLite core only ever uses +** sqlite3_mutex_try() as an optimization so this is acceptable +** behavior.)^ ** ** ^The sqlite3_mutex_leave() routine exits a mutex that was -** previously entered by the same thread. ^(The behavior +** previously entered by the same thread. The behavior ** is undefined if the mutex is not currently entered by the -** calling thread or is not currently allocated. SQLite will -** never do either.)^ +** calling thread or is not currently allocated. ** ** ^If the argument to sqlite3_mutex_enter(), sqlite3_mutex_try(), or ** sqlite3_mutex_leave() is a NULL pointer, then all three routines @@ -5935,9 +6023,9 @@ void sqlite3_mutex_leave(sqlite3_mutex*); ** used to allocate and use mutexes. ** ** Usually, the default mutex implementations provided by SQLite are -** sufficient, however the user has the option of substituting a custom +** sufficient, however the application has the option of substituting a custom ** implementation for specialized deployments or systems for which SQLite -** does not provide a suitable implementation. In this case, the user +** does not provide a suitable implementation. In this case, the application ** creates and populates an instance of this structure to pass ** to sqlite3_config() along with the [SQLITE_CONFIG_MUTEX] option. ** Additionally, an instance of this structure can be used as an @@ -5978,13 +6066,13 @@ void sqlite3_mutex_leave(sqlite3_mutex*); ** (i.e. it is acceptable to provide an implementation that segfaults if ** it is passed a NULL pointer). ** -** The xMutexInit() method must be threadsafe. ^It must be harmless to +** The xMutexInit() method must be threadsafe. It must be harmless to ** invoke xMutexInit() multiple times within the same process and without ** intervening calls to xMutexEnd(). Second and subsequent calls to ** xMutexInit() must be no-ops. ** -** ^xMutexInit() must not use SQLite memory allocation ([sqlite3_malloc()] -** and its associates). ^Similarly, xMutexAlloc() must not use SQLite memory +** xMutexInit() must not use SQLite memory allocation ([sqlite3_malloc()] +** and its associates). Similarly, xMutexAlloc() must not use SQLite memory ** allocation for a static mutex. ^However xMutexAlloc() may use SQLite ** memory allocation for a fast or recursive mutex. ** @@ -6010,29 +6098,29 @@ struct sqlite3_mutex_methods { ** CAPI3REF: Mutex Verification Routines ** ** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routines -** are intended for use inside assert() statements. ^The SQLite core +** are intended for use inside assert() statements. The SQLite core ** never uses these routines except inside an assert() and applications -** are advised to follow the lead of the core. ^The SQLite core only +** are advised to follow the lead of the core. The SQLite core only ** provides implementations for these routines when it is compiled -** with the SQLITE_DEBUG flag. ^External mutex implementations +** with the SQLITE_DEBUG flag. External mutex implementations ** are only required to provide these routines if SQLITE_DEBUG is ** defined and if NDEBUG is not defined. ** -** ^These routines should return true if the mutex in their argument +** These routines should return true if the mutex in their argument ** is held or not held, respectively, by the calling thread. ** -** ^The implementation is not required to provide versions of these +** The implementation is not required to provide versions of these ** routines that actually work. If the implementation does not provide working ** versions of these routines, it should at least provide stubs that always ** return true so that one does not get spurious assertion failures. ** -** ^If the argument to sqlite3_mutex_held() is a NULL pointer then +** If the argument to sqlite3_mutex_held() is a NULL pointer then ** the routine should return 1. This seems counter-intuitive since ** clearly the mutex cannot be held if it does not exist. But ** the reason the mutex does not exist is because the build is not ** using mutexes. And we do not want the assert() containing the ** call to sqlite3_mutex_held() to fail, so a non-zero return is -** the appropriate thing to do. ^The sqlite3_mutex_notheld() +** the appropriate thing to do. The sqlite3_mutex_notheld() ** interface should also return 1 when given a NULL pointer. */ #ifndef NDEBUG @@ -6060,6 +6148,9 @@ int sqlite3_mutex_notheld(sqlite3_mutex*); #define SQLITE_MUTEX_STATIC_LRU 6 /* lru page list */ #define SQLITE_MUTEX_STATIC_LRU2 7 /* NOT USED */ #define SQLITE_MUTEX_STATIC_PMEM 7 /* sqlite3PageMalloc() */ +#define SQLITE_MUTEX_STATIC_APP1 8 /* For use by application */ +#define SQLITE_MUTEX_STATIC_APP2 9 /* For use by application */ +#define SQLITE_MUTEX_STATIC_APP3 10 /* For use by application */ /* ** CAPI3REF: Retrieve the mutex for a database connection @@ -6151,11 +6242,13 @@ int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_ISKEYWORD 16 #define SQLITE_TESTCTRL_SCRATCHMALLOC 17 #define SQLITE_TESTCTRL_LOCALTIME_FAULT 18 -#define SQLITE_TESTCTRL_EXPLAIN_STMT 19 +#define SQLITE_TESTCTRL_EXPLAIN_STMT 19 /* NOT USED */ #define SQLITE_TESTCTRL_NEVER_CORRUPT 20 #define SQLITE_TESTCTRL_VDBE_COVERAGE 21 #define SQLITE_TESTCTRL_BYTEORDER 22 -#define SQLITE_TESTCTRL_LAST 22 +#define SQLITE_TESTCTRL_ISINIT 23 +#define SQLITE_TESTCTRL_SORTER_MMAP 24 +#define SQLITE_TESTCTRL_LAST 24 /* ** CAPI3REF: SQLite Runtime Status @@ -6346,12 +6439,12 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); ** the current value is always zero.)^ ** ** [[SQLITE_DBSTATUS_CACHE_USED]] ^(
    SQLITE_DBSTATUS_CACHE_USED
    -**
    This parameter returns the approximate number of of bytes of heap +**
    This parameter returns the approximate number of bytes of heap ** memory used by all pager caches associated with the database connection.)^ ** ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_USED is always 0. ** ** [[SQLITE_DBSTATUS_SCHEMA_USED]] ^(
    SQLITE_DBSTATUS_SCHEMA_USED
    -**
    This parameter returns the approximate number of of bytes of heap +**
    This parameter returns the approximate number of bytes of heap ** memory used to store the schema for all databases associated ** with the connection - main, temp, and any [ATTACH]-ed databases.)^ ** ^The full amount of memory used by the schemas is reported, even if the @@ -6360,7 +6453,7 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); ** ^The highwater mark associated with SQLITE_DBSTATUS_SCHEMA_USED is always 0. ** ** [[SQLITE_DBSTATUS_STMT_USED]] ^(
    SQLITE_DBSTATUS_STMT_USED
    -**
    This parameter returns the approximate number of of bytes of heap +**
    This parameter returns the approximate number of bytes of heap ** and lookaside memory used by all prepared statements associated with ** the database connection.)^ ** ^The highwater mark associated with SQLITE_DBSTATUS_STMT_USED is always 0. @@ -6760,6 +6853,10 @@ typedef struct sqlite3_backup sqlite3_backup; ** must be different or else sqlite3_backup_init(D,N,S,M) will fail with ** an error. ** +** ^A call to sqlite3_backup_init() will fail, returning SQLITE_ERROR, if +** there is already a read or read-write transaction open on the +** destination database. +** ** ^If an error occurs within sqlite3_backup_init(D,N,S,M), then NULL is ** returned and an error code and error message are stored in the ** destination [database connection] D. @@ -7083,12 +7180,10 @@ void sqlite3_log(int iErrCode, const char *zFormat, ...); ** CAPI3REF: Write-Ahead Log Commit Hook ** ** ^The [sqlite3_wal_hook()] function is used to register a callback that -** will be invoked each time a database connection commits data to a -** [write-ahead log] (i.e. whenever a transaction is committed in -** [journal_mode | journal_mode=WAL mode]). +** is invoked each time data is committed to a database in wal mode. ** -** ^The callback is invoked by SQLite after the commit has taken place and -** the associated write-lock on the database released, so the implementation +** ^(The callback is invoked by SQLite after the commit has taken place and +** the associated write-lock on the database released)^, so the implementation ** may read, write or [checkpoint] the database as required. ** ** ^The first parameter passed to the callback function when it is invoked @@ -7139,6 +7234,9 @@ void *sqlite3_wal_hook( ** ^The [wal_autocheckpoint pragma] can be used to invoke this interface ** from SQL. ** +** ^Checkpoints initiated by this mechanism are +** [sqlite3_wal_checkpoint_v2|PASSIVE]. +** ** ^Every new [database connection] defaults to having the auto-checkpoint ** enabled with a threshold of 1000 or [SQLITE_DEFAULT_WAL_AUTOCHECKPOINT] ** pages. The use of this interface @@ -7150,90 +7248,114 @@ int sqlite3_wal_autocheckpoint(sqlite3 *db, int N); /* ** CAPI3REF: Checkpoint a database ** -** ^The [sqlite3_wal_checkpoint(D,X)] interface causes database named X -** on [database connection] D to be [checkpointed]. ^If X is NULL or an -** empty string, then a checkpoint is run on all databases of -** connection D. ^If the database connection D is not in -** [WAL | write-ahead log mode] then this interface is a harmless no-op. +** ^(The sqlite3_wal_checkpoint(D,X) is equivalent to +** [sqlite3_wal_checkpoint_v2](D,X,[SQLITE_CHECKPOINT_PASSIVE],0,0).)^ ** -** ^The [wal_checkpoint pragma] can be used to invoke this interface -** from SQL. ^The [sqlite3_wal_autocheckpoint()] interface and the -** [wal_autocheckpoint pragma] can be used to cause this interface to be -** run whenever the WAL reaches a certain size threshold. +** In brief, sqlite3_wal_checkpoint(D,X) causes the content in the +** [write-ahead log] for database X on [database connection] D to be +** transferred into the database file and for the write-ahead log to +** be reset. See the [checkpointing] documentation for addition +** information. ** -** See also: [sqlite3_wal_checkpoint_v2()] +** This interface used to be the only way to cause a checkpoint to +** occur. But then the newer and more powerful [sqlite3_wal_checkpoint_v2()] +** interface was added. This interface is retained for backwards +** compatibility and as a convenience for applications that need to manually +** start a callback but which do not need the full power (and corresponding +** complication) of [sqlite3_wal_checkpoint_v2()]. */ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); /* ** CAPI3REF: Checkpoint a database ** -** Run a checkpoint operation on WAL database zDb attached to database -** handle db. The specific operation is determined by the value of the -** eMode parameter: +** ^(The sqlite3_wal_checkpoint_v2(D,X,M,L,C) interface runs a checkpoint +** operation on database X of [database connection] D in mode M. Status +** information is written back into integers pointed to by L and C.)^ +** ^(The M parameter must be a valid [checkpoint mode]:)^ ** **
    **
    SQLITE_CHECKPOINT_PASSIVE
    -** Checkpoint as many frames as possible without waiting for any database -** readers or writers to finish. Sync the db file if all frames in the log -** are checkpointed. This mode is the same as calling -** sqlite3_wal_checkpoint(). The busy-handler callback is never invoked. +** ^Checkpoint as many frames as possible without waiting for any database +** readers or writers to finish, then sync the database file if all frames +** in the log were checkpointed. ^The [busy-handler callback] +** is never invoked in the SQLITE_CHECKPOINT_PASSIVE mode. +** ^On the other hand, passive mode might leave the checkpoint unfinished +** if there are concurrent readers or writers. ** **
    SQLITE_CHECKPOINT_FULL
    -** This mode blocks (calls the busy-handler callback) until there is no +** ^This mode blocks (it invokes the +** [sqlite3_busy_handler|busy-handler callback]) until there is no ** database writer and all readers are reading from the most recent database -** snapshot. It then checkpoints all frames in the log file and syncs the -** database file. This call blocks database writers while it is running, -** but not database readers. +** snapshot. ^It then checkpoints all frames in the log file and syncs the +** database file. ^This mode blocks new database writers while it is pending, +** but new database readers are allowed to continue unimpeded. ** **
    SQLITE_CHECKPOINT_RESTART
    -** This mode works the same way as SQLITE_CHECKPOINT_FULL, except after -** checkpointing the log file it blocks (calls the busy-handler callback) -** until all readers are reading from the database file only. This ensures -** that the next client to write to the database file restarts the log file -** from the beginning. This call blocks database writers while it is running, -** but not database readers. +** ^This mode works the same way as SQLITE_CHECKPOINT_FULL with the addition +** that after checkpointing the log file it blocks (calls the +** [busy-handler callback]) +** until all readers are reading from the database file only. ^This ensures +** that the next writer will restart the log file from the beginning. +** ^Like SQLITE_CHECKPOINT_FULL, this mode blocks new +** database writer attempts while it is pending, but does not impede readers. +** +**
    SQLITE_CHECKPOINT_TRUNCATE
    +** ^This mode works the same way as SQLITE_CHECKPOINT_RESTART with the +** addition that it also truncates the log file to zero bytes just prior +** to a successful return. **
    ** -** If pnLog is not NULL, then *pnLog is set to the total number of frames in -** the log file before returning. If pnCkpt is not NULL, then *pnCkpt is set to -** the total number of checkpointed frames (including any that were already -** checkpointed when this function is called). *pnLog and *pnCkpt may be -** populated even if sqlite3_wal_checkpoint_v2() returns other than SQLITE_OK. -** If no values are available because of an error, they are both set to -1 -** before returning to communicate this to the caller. +** ^If pnLog is not NULL, then *pnLog is set to the total number of frames in +** the log file or to -1 if the checkpoint could not run because +** of an error or because the database is not in [WAL mode]. ^If pnCkpt is not +** NULL,then *pnCkpt is set to the total number of checkpointed frames in the +** log file (including any that were already checkpointed before the function +** was called) or to -1 if the checkpoint could not run due to an error or +** because the database is not in WAL mode. ^Note that upon successful +** completion of an SQLITE_CHECKPOINT_TRUNCATE, the log file will have been +** truncated to zero bytes and so both *pnLog and *pnCkpt will be set to zero. ** -** All calls obtain an exclusive "checkpoint" lock on the database file. If +** ^All calls obtain an exclusive "checkpoint" lock on the database file. ^If ** any other process is running a checkpoint operation at the same time, the -** lock cannot be obtained and SQLITE_BUSY is returned. Even if there is a +** lock cannot be obtained and SQLITE_BUSY is returned. ^Even if there is a ** busy-handler configured, it will not be invoked in this case. ** -** The SQLITE_CHECKPOINT_FULL and RESTART modes also obtain the exclusive -** "writer" lock on the database file. If the writer lock cannot be obtained -** immediately, and a busy-handler is configured, it is invoked and the writer -** lock retried until either the busy-handler returns 0 or the lock is -** successfully obtained. The busy-handler is also invoked while waiting for -** database readers as described above. If the busy-handler returns 0 before +** ^The SQLITE_CHECKPOINT_FULL, RESTART and TRUNCATE modes also obtain the +** exclusive "writer" lock on the database file. ^If the writer lock cannot be +** obtained immediately, and a busy-handler is configured, it is invoked and +** the writer lock retried until either the busy-handler returns 0 or the lock +** is successfully obtained. ^The busy-handler is also invoked while waiting for +** database readers as described above. ^If the busy-handler returns 0 before ** the writer lock is obtained or while waiting for database readers, the ** checkpoint operation proceeds from that point in the same way as ** SQLITE_CHECKPOINT_PASSIVE - checkpointing as many frames as possible -** without blocking any further. SQLITE_BUSY is returned in this case. +** without blocking any further. ^SQLITE_BUSY is returned in this case. ** -** If parameter zDb is NULL or points to a zero length string, then the -** specified operation is attempted on all WAL databases. In this case the -** values written to output parameters *pnLog and *pnCkpt are undefined. If +** ^If parameter zDb is NULL or points to a zero length string, then the +** specified operation is attempted on all WAL databases [attached] to +** [database connection] db. In this case the +** values written to output parameters *pnLog and *pnCkpt are undefined. ^If ** an SQLITE_BUSY error is encountered when processing one or more of the ** attached WAL databases, the operation is still attempted on any remaining -** attached databases and SQLITE_BUSY is returned to the caller. If any other +** attached databases and SQLITE_BUSY is returned at the end. ^If any other ** error occurs while processing an attached database, processing is abandoned -** and the error code returned to the caller immediately. If no error +** and the error code is returned to the caller immediately. ^If no error ** (SQLITE_BUSY or otherwise) is encountered while processing the attached ** databases, SQLITE_OK is returned. ** -** If database zDb is the name of an attached database that is not in WAL -** mode, SQLITE_OK is returned and both *pnLog and *pnCkpt set to -1. If +** ^If database zDb is the name of an attached database that is not in WAL +** mode, SQLITE_OK is returned and both *pnLog and *pnCkpt set to -1. ^If ** zDb is not NULL (or a zero length string) and is not the name of any ** attached database, SQLITE_ERROR is returned to the caller. +** +** ^Unless it returns SQLITE_MISUSE, +** the sqlite3_wal_checkpoint_v2() interface +** sets the error information that is queried by +** [sqlite3_errcode()] and [sqlite3_errmsg()]. +** +** ^The [PRAGMA wal_checkpoint] command can be used to invoke this interface +** from SQL. */ int sqlite3_wal_checkpoint_v2( sqlite3 *db, /* Database handle */ @@ -7244,16 +7366,18 @@ int sqlite3_wal_checkpoint_v2( ); /* -** CAPI3REF: Checkpoint operation parameters +** CAPI3REF: Checkpoint Mode Values +** KEYWORDS: {checkpoint mode} ** -** These constants can be used as the 3rd parameter to -** [sqlite3_wal_checkpoint_v2()]. See the [sqlite3_wal_checkpoint_v2()] -** documentation for additional information about the meaning and use of -** each of these values. +** These constants define all valid values for the "checkpoint mode" passed +** as the third parameter to the [sqlite3_wal_checkpoint_v2()] interface. +** See the [sqlite3_wal_checkpoint_v2()] documentation for details on the +** meaning of each of these checkpoint modes. */ -#define SQLITE_CHECKPOINT_PASSIVE 0 -#define SQLITE_CHECKPOINT_FULL 1 -#define SQLITE_CHECKPOINT_RESTART 2 +#define SQLITE_CHECKPOINT_PASSIVE 0 /* Do as much as possible w/o blocking */ +#define SQLITE_CHECKPOINT_FULL 1 /* Wait for writers, then checkpoint */ +#define SQLITE_CHECKPOINT_RESTART 2 /* Like FULL but wait for for readers */ +#define SQLITE_CHECKPOINT_TRUNCATE 3 /* Like RESTART but also truncate WAL */ /* ** CAPI3REF: Virtual Table Interface Configuration @@ -7326,6 +7450,7 @@ int sqlite3_vtab_on_conflict(sqlite3 *); /* ** CAPI3REF: Conflict resolution modes +** KEYWORDS: {conflict resolution mode} ** ** These constants are returned by [sqlite3_vtab_on_conflict()] to ** inform a [virtual table] implementation what the [ON CONFLICT] mode @@ -7341,6 +7466,98 @@ int sqlite3_vtab_on_conflict(sqlite3 *); /* #define SQLITE_ABORT 4 // Also an error code */ #define SQLITE_REPLACE 5 +/* +** CAPI3REF: Prepared Statement Scan Status Opcodes +** KEYWORDS: {scanstatus options} +** +** The following constants can be used for the T parameter to the +** [sqlite3_stmt_scanstatus(S,X,T,V)] interface. Each constant designates a +** different metric for sqlite3_stmt_scanstatus() to return. +** +**
    +** [[SQLITE_SCANSTAT_NLOOP]]
    SQLITE_SCANSTAT_NLOOP
    +**
    ^The [sqlite3_int64] variable pointed to by the T parameter will be +** set to the total number of times that the X-th loop has run.
    +** +** [[SQLITE_SCANSTAT_NVISIT]]
    SQLITE_SCANSTAT_NVISIT
    +**
    ^The [sqlite3_int64] variable pointed to by the T parameter will be set +** to the total number of rows examined by all iterations of the X-th loop.
    +** +** [[SQLITE_SCANSTAT_EST]]
    SQLITE_SCANSTAT_EST
    +**
    ^The "double" variable pointed to by the T parameter will be set to the +** query planner's estimate for the average number of rows output from each +** iteration of the X-th loop. If the query planner's estimates was accurate, +** then this value will approximate the quotient NVISIT/NLOOP and the +** product of this value for all prior loops with the same SELECTID will +** be the NLOOP value for the current loop. +** +** [[SQLITE_SCANSTAT_NAME]]
    SQLITE_SCANSTAT_NAME
    +**
    ^The "const char *" variable pointed to by the T parameter will be set +** to a zero-terminated UTF-8 string containing the name of the index or table +** used for the X-th loop. +** +** [[SQLITE_SCANSTAT_EXPLAIN]]
    SQLITE_SCANSTAT_EXPLAIN
    +**
    ^The "const char *" variable pointed to by the T parameter will be set +** to a zero-terminated UTF-8 string containing the [EXPLAIN QUERY PLAN] +** description for the X-th loop. +** +** [[SQLITE_SCANSTAT_SELECTID]]
    SQLITE_SCANSTAT_SELECT
    +**
    ^The "int" variable pointed to by the T parameter will be set to the +** "select-id" for the X-th loop. The select-id identifies which query or +** subquery the loop is part of. The main query has a select-id of zero. +** The select-id is the same value as is output in the first column +** of an [EXPLAIN QUERY PLAN] query. +**
    +*/ +#define SQLITE_SCANSTAT_NLOOP 0 +#define SQLITE_SCANSTAT_NVISIT 1 +#define SQLITE_SCANSTAT_EST 2 +#define SQLITE_SCANSTAT_NAME 3 +#define SQLITE_SCANSTAT_EXPLAIN 4 +#define SQLITE_SCANSTAT_SELECTID 5 + +/* +** CAPI3REF: Prepared Statement Scan Status +** +** Return status data for a single loop within query pStmt. +** +** The "iScanStatusOp" parameter determines which status information to return. +** The "iScanStatusOp" must be one of the [scanstatus options] or the behavior +** of this interface is undefined. +** ^The requested measurement is written into a variable pointed to by +** the "pOut" parameter. +** Parameter "idx" identifies the specific loop to retrieve statistics for. +** Loops are numbered starting from zero. ^If idx is out of range - less than +** zero or greater than or equal to the total number of loops used to implement +** the statement - a non-zero value is returned and the variable that pOut +** points to is unchanged. +** +** ^Statistics might not be available for all loops in all statements. ^In cases +** where there exist loops with no available statistics, this function behaves +** as if the loop did not exist - it returns non-zero and leave the variable +** that pOut points to unchanged. +** +** This API is only available if the library is built with pre-processor +** symbol [SQLITE_ENABLE_STMT_SCANSTATUS] defined. +** +** See also: [sqlite3_stmt_scanstatus_reset()] +*/ +SQLITE_EXPERIMENTAL int sqlite3_stmt_scanstatus( + sqlite3_stmt *pStmt, /* Prepared statement for which info desired */ + int idx, /* Index of loop to report on */ + int iScanStatusOp, /* Information desired. SQLITE_SCANSTAT_* */ + void *pOut /* Result written here */ +); + +/* +** CAPI3REF: Zero Scan-Status Counters +** +** ^Zero all [sqlite3_stmt_scanstatus()] related event counters. +** +** This API is only available if the library is built with pre-processor +** symbol [SQLITE_ENABLE_STMT_SCANSTATUS] defined. +*/ +SQLITE_EXPERIMENTAL void sqlite3_stmt_scanstatus_reset(sqlite3_stmt*); /* diff --git a/src/sqlite3.rc b/src/sqlite3.rc index aedbb63ebd..04dd086488 100644 --- a/src/sqlite3.rc +++ b/src/sqlite3.rc @@ -35,6 +35,14 @@ LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US #pragma code_page(1252) #endif /* defined(_WIN32) */ +/* + * Icon + */ + +#define IDI_SQLITE 101 + +IDI_SQLITE ICON "..\\art\\sqlite370.ico" + /* * Version */ diff --git a/src/sqlite3ext.h b/src/sqlite3ext.h index ecf93f62f6..f9a066592d 100644 --- a/src/sqlite3ext.h +++ b/src/sqlite3ext.h @@ -28,7 +28,7 @@ typedef struct sqlite3_api_routines sqlite3_api_routines; ** WARNING: In order to maintain backwards compatibility, add new ** interfaces to the end of this structure only. If you insert new ** interfaces in the middle of this structure, then older different -** versions of SQLite will not be able to load each others' shared +** versions of SQLite will not be able to load each other's shared ** libraries! */ struct sqlite3_api_routines { @@ -250,11 +250,28 @@ struct sqlite3_api_routines { const char *(*uri_parameter)(const char*,const char*); char *(*vsnprintf)(int,char*,const char*,va_list); int (*wal_checkpoint_v2)(sqlite3*,const char*,int,int*,int*); + /* Version 3.8.7 and later */ + int (*auto_extension)(void(*)(void)); + int (*bind_blob64)(sqlite3_stmt*,int,const void*,sqlite3_uint64, + void(*)(void*)); + int (*bind_text64)(sqlite3_stmt*,int,const char*,sqlite3_uint64, + void(*)(void*),unsigned char); + int (*cancel_auto_extension)(void(*)(void)); + int (*load_extension)(sqlite3*,const char*,const char*,char**); + void *(*malloc64)(sqlite3_uint64); + sqlite3_uint64 (*msize)(void*); + void *(*realloc64)(void*,sqlite3_uint64); + void (*reset_auto_extension)(void); + void (*result_blob64)(sqlite3_context*,const void*,sqlite3_uint64, + void(*)(void*)); + void (*result_text64)(sqlite3_context*,const char*,sqlite3_uint64, + void(*)(void*), unsigned char); + int (*strglob)(const char*,const char*); }; /* ** The following macros redefine the API routines so that they are -** redirected throught the global sqlite3_api structure. +** redirected through the global sqlite3_api structure. ** ** This header file is also used by the loadext.c source file ** (part of the main SQLite library - not an extension) so that @@ -467,6 +484,19 @@ struct sqlite3_api_routines { #define sqlite3_uri_parameter sqlite3_api->uri_parameter #define sqlite3_uri_vsnprintf sqlite3_api->vsnprintf #define sqlite3_wal_checkpoint_v2 sqlite3_api->wal_checkpoint_v2 +/* Version 3.8.7 and later */ +#define sqlite3_auto_extension sqlite3_api->auto_extension +#define sqlite3_bind_blob64 sqlite3_api->bind_blob64 +#define sqlite3_bind_text64 sqlite3_api->bind_text64 +#define sqlite3_cancel_auto_extension sqlite3_api->cancel_auto_extension +#define sqlite3_load_extension sqlite3_api->load_extension +#define sqlite3_malloc64 sqlite3_api->malloc64 +#define sqlite3_msize sqlite3_api->msize +#define sqlite3_realloc64 sqlite3_api->realloc64 +#define sqlite3_reset_auto_extension sqlite3_api->reset_auto_extension +#define sqlite3_result_blob64 sqlite3_api->result_blob64 +#define sqlite3_result_text64 sqlite3_api->result_text64 +#define sqlite3_strglob sqlite3_api->strglob #endif /* SQLITE_CORE */ #ifndef SQLITE_CORE diff --git a/src/sqliteInt.h b/src/sqliteInt.h index cc3a030510..d90089ba0e 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -47,6 +47,15 @@ # define _LARGEFILE_SOURCE 1 #endif +/* Needed for various definitions... */ +#if defined(__GNUC__) && !defined(_GNU_SOURCE) +# define _GNU_SOURCE +#endif + +#if defined(__OpenBSD__) && !defined(_BSD_SOURCE) +# define _BSD_SOURCE +#endif + /* ** For MinGW, check to see if we can include the header file containing its ** version information, among other things. Normally, this internal MinGW @@ -104,15 +113,6 @@ #pragma warn -spa /* Suspicious pointer arithmetic */ #endif -/* Needed for various definitions... */ -#ifndef _GNU_SOURCE -# define _GNU_SOURCE -#endif - -#if defined(__OpenBSD__) && !defined(_BSD_SOURCE) -# define _BSD_SOURCE -#endif - /* ** Include standard header files as necessary */ @@ -153,6 +153,18 @@ # define SQLITE_PTR_TO_INT(X) ((int)(X)) #endif +/* +** A macro to hint to the compiler that a function should not be +** inlined. +*/ +#if defined(__GNUC__) +# define SQLITE_NOINLINE __attribute__((noinline)) +#elif defined(_MSC_VER) && _MSC_VER>=1310 +# define SQLITE_NOINLINE __declspec(noinline) +#else +# define SQLITE_NOINLINE +#endif + /* ** The SQLITE_THREADSAFE macro must be defined as 0, 1, or 2. ** 0 means mutexes are permanently disable and the library is never @@ -181,10 +193,9 @@ #endif /* -** The SQLITE_DEFAULT_MEMSTATUS macro must be defined as either 0 or 1. -** It determines whether or not the features related to -** SQLITE_CONFIG_MEMSTATUS are available by default or not. This value can -** be overridden at runtime using the sqlite3_config() API. +** EVIDENCE-OF: R-25715-37072 Memory allocation statistics are enabled by +** default unless SQLite is compiled with SQLITE_DEFAULT_MEMSTATUS=0 in +** which case memory allocation statistics are disabled by default. */ #if !defined(SQLITE_DEFAULT_MEMSTATUS) # define SQLITE_DEFAULT_MEMSTATUS 1 @@ -339,7 +350,7 @@ #endif /* -** Return true (non-zero) if the input is a integer that is too large +** Return true (non-zero) if the input is an integer that is too large ** to fit in 32-bits. This macro is used inside of various testcase() ** macros to verify that we have tested SQLite for large-file support. */ @@ -422,6 +433,27 @@ # define SQLITE_TEMP_STORE_xc 1 /* Exclude from ctime.c */ #endif +/* +** If no value has been provided for SQLITE_MAX_WORKER_THREADS, or if +** SQLITE_TEMP_STORE is set to 3 (never use temporary files), set it +** to zero. +*/ +#if SQLITE_TEMP_STORE==3 || SQLITE_THREADSAFE==0 +# undef SQLITE_MAX_WORKER_THREADS +# define SQLITE_MAX_WORKER_THREADS 0 +#endif +#ifndef SQLITE_MAX_WORKER_THREADS +# define SQLITE_MAX_WORKER_THREADS 8 +#endif +#ifndef SQLITE_DEFAULT_WORKER_THREADS +# define SQLITE_DEFAULT_WORKER_THREADS 0 +#endif +#if SQLITE_DEFAULT_WORKER_THREADS>SQLITE_MAX_WORKER_THREADS +# undef SQLITE_MAX_WORKER_THREADS +# define SQLITE_MAX_WORKER_THREADS SQLITE_DEFAULT_WORKER_THREADS +#endif + + /* ** GCC does not define the offsetof() macro so we'll have to do it ** ourselves. @@ -436,6 +468,11 @@ #define MIN(A,B) ((A)<(B)?(A):(B)) #define MAX(A,B) ((A)>(B)?(A):(B)) +/* +** Swap two objects of type TYPE. +*/ +#define SWAP(TYPE,A,B) {TYPE t=A; A=B; B=t;} + /* ** Check to see if this machine uses EBCDIC. (Yes, believe it or ** not, there are still machines out there that use EBCDIC.) @@ -525,7 +562,7 @@ typedef INT8_TYPE i8; /* 1-byte signed integer */ ** gives a possible range of values of approximately 1.0e986 to 1e-986. ** But the allowed values are "grainy". Not every value is representable. ** For example, quantities 16 and 17 are both represented by a LogEst -** of 40. However, since LogEst quantaties are suppose to be estimates, +** of 40. However, since LogEst quantities are suppose to be estimates, ** not exact values, this imprecision is not a problem. ** ** "LogEst" is short for "Logarithmic Estimate". @@ -606,7 +643,7 @@ extern const int sqlite3one; ** all alignment restrictions correct. ** ** Except, if SQLITE_4_BYTE_ALIGNED_MALLOC is defined, then the -** underlying malloc() implemention might return us 4-byte aligned +** underlying malloc() implementation might return us 4-byte aligned ** pointers. In that case, only verify 4-byte alignment. */ #ifdef SQLITE_4_BYTE_ALIGNED_MALLOC @@ -673,6 +710,16 @@ extern const int sqlite3one; # undef SQLITE_ENABLE_STAT3_OR_STAT4 #endif +/* +** SELECTTRACE_ENABLED will be either 1 or 0 depending on whether or not +** the Select query generator tracing logic is turned on. +*/ +#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_SELECTTRACE) +# define SELECTTRACE_ENABLED 1 +#else +# define SELECTTRACE_ENABLED 0 +#endif + /* ** An instance of the following structure is used to store the busy-handler ** callback for a given sqlite handle. @@ -805,12 +852,14 @@ typedef struct PrintfArguments PrintfArguments; typedef struct RowSet RowSet; typedef struct Savepoint Savepoint; typedef struct Select Select; +typedef struct SQLiteThread SQLiteThread; typedef struct SelectDest SelectDest; typedef struct SrcList SrcList; typedef struct StrAccum StrAccum; typedef struct Table Table; typedef struct TableLock TableLock; typedef struct Token Token; +typedef struct TreeView TreeView; typedef struct Trigger Trigger; typedef struct TriggerPrg TriggerPrg; typedef struct TriggerStep TriggerStep; @@ -876,7 +925,7 @@ struct Schema { Table *pSeqTab; /* The sqlite_sequence table used by AUTOINCREMENT */ u8 file_format; /* Schema format version for this file */ u8 enc; /* Text encoding used by this database */ - u16 flags; /* Flags associated with this schema */ + u16 schemaFlags; /* Flags associated with this schema */ int cache_size; /* Number of pages to use in the cache */ }; @@ -884,10 +933,10 @@ struct Schema { ** These macros can be used to test, set, or clear bits in the ** Db.pSchema->flags field. */ -#define DbHasProperty(D,I,P) (((D)->aDb[I].pSchema->flags&(P))==(P)) -#define DbHasAnyProperty(D,I,P) (((D)->aDb[I].pSchema->flags&(P))!=0) -#define DbSetProperty(D,I,P) (D)->aDb[I].pSchema->flags|=(P) -#define DbClearProperty(D,I,P) (D)->aDb[I].pSchema->flags&=~(P) +#define DbHasProperty(D,I,P) (((D)->aDb[I].pSchema->schemaFlags&(P))==(P)) +#define DbHasAnyProperty(D,I,P) (((D)->aDb[I].pSchema->schemaFlags&(P))!=0) +#define DbSetProperty(D,I,P) (D)->aDb[I].pSchema->schemaFlags|=(P) +#define DbClearProperty(D,I,P) (D)->aDb[I].pSchema->schemaFlags&=~(P) /* ** Allowed values for the DB.pSchema->flags field. @@ -907,7 +956,7 @@ struct Schema { ** The number of different kinds of things that can be limited ** using the sqlite3_limit() interface. */ -#define SQLITE_N_LIMIT (SQLITE_LIMIT_TRIGGER_DEPTH+1) +#define SQLITE_N_LIMIT (SQLITE_LIMIT_WORKER_THREADS+1) /* ** Lookaside malloc is a set of fixed-size buffers that can be used @@ -954,6 +1003,45 @@ struct FuncDefHash { FuncDef *a[23]; /* Hash table for functions */ }; +#ifdef SQLITE_USER_AUTHENTICATION +/* +** Information held in the "sqlite3" database connection object and used +** to manage user authentication. +*/ +typedef struct sqlite3_userauth sqlite3_userauth; +struct sqlite3_userauth { + u8 authLevel; /* Current authentication level */ + int nAuthPW; /* Size of the zAuthPW in bytes */ + char *zAuthPW; /* Password used to authenticate */ + char *zAuthUser; /* User name used to authenticate */ +}; + +/* Allowed values for sqlite3_userauth.authLevel */ +#define UAUTH_Unknown 0 /* Authentication not yet checked */ +#define UAUTH_Fail 1 /* User authentication failed */ +#define UAUTH_User 2 /* Authenticated as a normal user */ +#define UAUTH_Admin 3 /* Authenticated as an administrator */ + +/* Functions used only by user authorization logic */ +int sqlite3UserAuthTable(const char*); +int sqlite3UserAuthCheckLogin(sqlite3*,const char*,u8*); +void sqlite3UserAuthInit(sqlite3*); +void sqlite3CryptFunc(sqlite3_context*,int,sqlite3_value**); + +#endif /* SQLITE_USER_AUTHENTICATION */ + +/* +** typedef for the authorization callback function. +*/ +#ifdef SQLITE_USER_AUTHENTICATION + typedef int (*sqlite3_xauth)(void*,int,const char*,const char*,const char*, + const char*, const char*); +#else + typedef int (*sqlite3_xauth)(void*,int,const char*,const char*,const char*, + const char*); +#endif + + /* ** Each database connection is an instance of the following structure. */ @@ -971,6 +1059,7 @@ struct sqlite3 { int errCode; /* Most recent error code (SQLITE_*) */ int errMask; /* & result codes with this before returning */ u16 dbOptFlags; /* Flags to enable/disable optimizations */ + u8 enc; /* Text encoding */ u8 autoCommit; /* The auto-commit flag. */ u8 temp_store; /* 1: file 2: memory 0: default */ u8 mallocFailed; /* True if we have seen a malloc failure */ @@ -984,6 +1073,7 @@ struct sqlite3 { int nChange; /* Value returned by sqlite3_changes() */ int nTotalChange; /* Value returned by sqlite3_total_changes() */ int aLimit[SQLITE_N_LIMIT]; /* Limits */ + int nMaxSorterMmap; /* Maximum size of regions mapped by sorter */ struct sqlite3InitInfo { /* Information used during initialization */ int newTnum; /* Rootpage of table being initialized */ u8 iDb; /* Which db file is being initialized */ @@ -1020,8 +1110,7 @@ struct sqlite3 { } u1; Lookaside lookaside; /* Lookaside malloc configuration */ #ifndef SQLITE_OMIT_AUTHORIZATION - int (*xAuth)(void*,int,const char*,const char*,const char*,const char*); - /* Access authorization function */ + sqlite3_xauth xAuth; /* Access authorization function */ void *pAuthArg; /* 1st argument to the access auth function */ #endif #ifndef SQLITE_OMIT_PROGRESS_CALLBACK @@ -1047,7 +1136,6 @@ struct sqlite3 { i64 nDeferredCons; /* Net deferred constraints this transaction. */ i64 nDeferredImmCons; /* Net deferred immediate constraints */ int *pnBytesFreed; /* If not NULL, increment this in DbFree() */ - #ifdef SQLITE_ENABLE_UNLOCK_NOTIFY /* The following variables are all protected by the STATIC_MASTER ** mutex, not by sqlite3.mutex. They are used by code in notify.c. @@ -1065,12 +1153,16 @@ struct sqlite3 { void (*xUnlockNotify)(void **, int); /* Unlock notify callback */ sqlite3 *pNextBlocked; /* Next in list of all blocked connections */ #endif +#ifdef SQLITE_USER_AUTHENTICATION + sqlite3_userauth auth; /* User authentication information */ +#endif }; /* ** A macro to discover the encoding of a database. */ -#define ENC(db) ((db)->aDb[0].pSchema->enc) +#define SCHEMA_ENC(db) ((db)->aDb[0].pSchema->enc) +#define ENC(db) ((db)->enc) /* ** Possible values for the sqlite3.flags. @@ -1123,8 +1215,7 @@ struct sqlite3 { #define SQLITE_SubqCoroutine 0x0100 /* Evaluate subqueries as coroutines */ #define SQLITE_Transitive 0x0200 /* Transitive constraints */ #define SQLITE_OmitNoopJoin 0x0400 /* Omit unused tables in joins */ -#define SQLITE_Stat3 0x0800 /* Use the SQLITE_STAT3 table */ -#define SQLITE_AdjustOutEst 0x1000 /* Adjust output estimates using WHERE */ +#define SQLITE_Stat34 0x0800 /* Use STAT3 or STAT4 data */ #define SQLITE_AllOpts 0xffff /* All optimizations */ /* @@ -1211,6 +1302,7 @@ struct FuncDestructor { #define SQLITE_FUNC_COALESCE 0x200 /* Built-in coalesce() or ifnull() */ #define SQLITE_FUNC_UNLIKELY 0x400 /* Built-in unlikely() function */ #define SQLITE_FUNC_CONSTANT 0x800 /* Constant inputs give a constant output */ +#define SQLITE_FUNC_MINMAX 0x1000 /* True for min() and max() aggregates */ /* ** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are @@ -1258,6 +1350,9 @@ struct FuncDestructor { #define AGGREGATE(zName, nArg, arg, nc, xStep, xFinal) \ {nArg, SQLITE_UTF8|(nc*SQLITE_FUNC_NEEDCOLL), \ SQLITE_INT_TO_PTR(arg), 0, 0, xStep,xFinal,#zName,0,0} +#define AGGREGATE2(zName, nArg, arg, nc, xStep, xFinal, extraFlags) \ + {nArg, SQLITE_UTF8|(nc*SQLITE_FUNC_NEEDCOLL)|extraFlags, \ + SQLITE_INT_TO_PTR(arg), 0, 0, xStep,xFinal,#zName,0,0} /* ** All current savepoints are stored in a linked list starting at @@ -1344,18 +1439,18 @@ struct CollSeq { ** 't' for SQLITE_AFF_TEXT. But we can save a little space and improve ** the speed a little by numbering the values consecutively. ** -** But rather than start with 0 or 1, we begin with 'a'. That way, +** But rather than start with 0 or 1, we begin with 'A'. That way, ** when multiple affinity types are concatenated into a string and ** used as the P4 operand, they will be more readable. ** ** Note also that the numeric types are grouped together so that testing -** for a numeric type is a single comparison. +** for a numeric type is a single comparison. And the NONE type is first. */ -#define SQLITE_AFF_TEXT 'a' -#define SQLITE_AFF_NONE 'b' -#define SQLITE_AFF_NUMERIC 'c' -#define SQLITE_AFF_INTEGER 'd' -#define SQLITE_AFF_REAL 'e' +#define SQLITE_AFF_NONE 'A' +#define SQLITE_AFF_TEXT 'B' +#define SQLITE_AFF_NUMERIC 'C' +#define SQLITE_AFF_INTEGER 'D' +#define SQLITE_AFF_REAL 'E' #define sqlite3IsNumericAffinity(X) ((X)>=SQLITE_AFF_NUMERIC) @@ -1363,7 +1458,7 @@ struct CollSeq { ** The SQLITE_AFF_MASK values masks off the significant bits of an ** affinity value. */ -#define SQLITE_AFF_MASK 0x67 +#define SQLITE_AFF_MASK 0x47 /* ** Additional bit values that can be ORed with an affinity without @@ -1374,10 +1469,10 @@ struct CollSeq { ** operator is NULL. It is added to certain comparison operators to ** prove that the operands are always NOT NULL. */ -#define SQLITE_JUMPIFNULL 0x08 /* jumps if either operand is NULL */ -#define SQLITE_STOREP2 0x10 /* Store result in reg[P2] rather than jump */ +#define SQLITE_JUMPIFNULL 0x10 /* jumps if either operand is NULL */ +#define SQLITE_STOREP2 0x20 /* Store result in reg[P2] rather than jump */ #define SQLITE_NULLEQ 0x80 /* NULL=NULL */ -#define SQLITE_NOTNULL 0x88 /* Assert that operands are never NULL */ +#define SQLITE_NOTNULL 0x90 /* Assert that operands are never NULL */ /* ** An object of this type is created for each virtual table present in @@ -1477,6 +1572,9 @@ struct Table { i16 nCol; /* Number of columns in this table */ u16 nRef; /* Number of pointers to this Table */ LogEst szTabRow; /* Estimated size of each table row in bytes */ +#ifdef SQLITE_ENABLE_COSTMULT + LogEst costMult; /* Cost multiplier for using this table */ +#endif u8 tabFlags; /* Mask of TF_* values */ u8 keyConf; /* What to do in case of uniqueness conflict on iPKey */ #ifndef SQLITE_OMIT_ALTERTABLE @@ -1644,7 +1742,7 @@ struct UnpackedRecord { KeyInfo *pKeyInfo; /* Collation and sort-order information */ u16 nField; /* Number of entries in apMem[] */ i8 default_rc; /* Comparison result if keys are equal */ - u8 isCorrupt; /* Corruption detected by xRecordCompare() */ + u8 errCode; /* Error detected by xRecordCompare (CORRUPT or NOMEM) */ Mem *aMem; /* Values */ int r1; /* Value to return if (lhs > rhs) */ int r2; /* Value to return if (rhs < lhs) */ @@ -1688,7 +1786,6 @@ struct Index { u8 *aSortOrder; /* for each column: True==DESC, False==ASC */ char **azColl; /* Array of collation sequence names for index */ Expr *pPartIdxWhere; /* WHERE clause for partial indices */ - KeyInfo *pKeyInfo; /* A KeyInfo object suitable for this index */ int tnum; /* DB Page containing root of this index */ LogEst szIdxRow; /* Estimated average row size in bytes */ u16 nKeyCol; /* Number of columns forming the key */ @@ -1699,11 +1796,14 @@ struct Index { unsigned uniqNotNull:1; /* True if UNIQUE and NOT NULL for all columns */ unsigned isResized:1; /* True if resizeIndexObject() has been called */ unsigned isCovering:1; /* True if this is a covering index */ + unsigned noSkipScan:1; /* Do not try to use skip-scan if true */ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 int nSample; /* Number of elements in aSample[] */ int nSampleCol; /* Size of IndexSample.anEq[] and so on */ tRowcnt *aAvgEq; /* Average nEq values for keys not in aSample */ IndexSample *aSample; /* Samples of the left-most key */ + tRowcnt *aiRowEst; /* Non-logarithmic stat1 data for this index */ + tRowcnt nRowEst0; /* Non-logarithmic number of rows in the index */ #endif }; @@ -1717,6 +1817,9 @@ struct Index { /* Return true if index X is a PRIMARY KEY index */ #define IsPrimaryKeyIndex(X) ((X)->idxType==SQLITE_IDXTYPE_PRIMARYKEY) +/* Return true if index X is a UNIQUE index */ +#define IsUniqueIndex(X) ((X)->onError!=OE_None) + /* ** Each sample stored in the sqlite_stat3 table is represented in memory ** using a structure of this type. See documentation at the top of the @@ -1898,7 +2001,7 @@ struct Expr { int iTable; /* TK_COLUMN: cursor number of table holding column ** TK_REGISTER: register number ** TK_TRIGGER: 1 -> new, 0 -> old - ** EP_Unlikely: 1000 times likelihood */ + ** EP_Unlikely: 134217728 times likelihood */ ynVar iColumn; /* TK_COLUMN: column index. -1 for rowid. ** TK_VARIABLE: variable number (always >= 1). */ i16 iAgg; /* Which entry in pAggInfo->aCol[] or ->aFunc[] */ @@ -1913,7 +2016,7 @@ struct Expr { /* ** The following are the meanings of bits in the Expr.flags field. */ -#define EP_FromJoin 0x000001 /* Originated in ON or USING clause of a join */ +#define EP_FromJoin 0x000001 /* Originates in ON/USING clause of outer join */ #define EP_Agg 0x000002 /* Contains one or more aggregate functions */ #define EP_Resolved 0x000004 /* IDs have been resolved to COLUMNs */ #define EP_Error 0x000008 /* Expression contains one or more errors */ @@ -1933,6 +2036,7 @@ struct Expr { #define EP_NoReduce 0x020000 /* Cannot EXPRDUP_REDUCE this Expr */ #define EP_Unlikely 0x040000 /* unlikely() or likelihood() function */ #define EP_Constant 0x080000 /* Node is a constant */ +#define EP_CanBeNull 0x100000 /* Can be null despite NOT NULL constraint */ /* ** These macros can be used to test, set, or clear bits in the @@ -2131,11 +2235,12 @@ struct SrcList { #define WHERE_OMIT_OPEN_CLOSE 0x0010 /* Table cursors are already open */ #define WHERE_FORCE_TABLE 0x0020 /* Do not use an index-only search */ #define WHERE_ONETABLE_ONLY 0x0040 /* Only code the 1st table in pTabList */ -#define WHERE_AND_ONLY 0x0080 /* Don't use indices for OR terms */ + /* 0x0080 // not currently used */ #define WHERE_GROUPBY 0x0100 /* pOrderBy is really a GROUP BY */ #define WHERE_DISTINCTBY 0x0200 /* pOrderby is really a DISTINCT clause */ #define WHERE_WANT_DISTINCT 0x0400 /* All output needs to be distinct */ #define WHERE_SORTBYGROUP 0x0800 /* Support sqlite3WhereIsSorted() */ +#define WHERE_REOPEN_IDX 0x1000 /* Try to use OP_ReopenIdx */ /* Allowed return values from sqlite3WhereIsDistinct() */ @@ -2173,17 +2278,22 @@ struct NameContext { NameContext *pNext; /* Next outer name context. NULL for outermost */ int nRef; /* Number of names resolved by this context */ int nErr; /* Number of errors encountered while resolving names */ - u8 ncFlags; /* Zero or more NC_* flags defined below */ + u16 ncFlags; /* Zero or more NC_* flags defined below */ }; /* ** Allowed values for the NameContext, ncFlags field. +** +** Note: NC_MinMaxAgg must have the same value as SF_MinMaxAgg and +** SQLITE_FUNC_MINMAX. +** */ -#define NC_AllowAgg 0x01 /* Aggregate functions are allowed here */ -#define NC_HasAgg 0x02 /* One or more aggregate functions seen */ -#define NC_IsCheck 0x04 /* True if resolving names in a CHECK constraint */ -#define NC_InAggFunc 0x08 /* True if analyzing arguments to an agg func */ -#define NC_PartIdx 0x10 /* True if resolving a partial index WHERE */ +#define NC_AllowAgg 0x0001 /* Aggregate functions are allowed here */ +#define NC_HasAgg 0x0002 /* One or more aggregate functions seen */ +#define NC_IsCheck 0x0004 /* True if resolving names in a CHECK constraint */ +#define NC_InAggFunc 0x0008 /* True if analyzing arguments to an agg func */ +#define NC_PartIdx 0x0010 /* True if resolving a partial index WHERE */ +#define NC_MinMaxAgg 0x1000 /* min/max aggregates seen. See note above */ /* ** An instance of the following structure contains all information @@ -2210,6 +2320,9 @@ struct Select { u8 op; /* One of: TK_UNION TK_ALL TK_INTERSECT TK_EXCEPT */ u16 selFlags; /* Various SF_* values */ int iLimit, iOffset; /* Memory registers holding LIMIT & OFFSET counters */ +#if SELECTTRACE_ENABLED + char zSelName[12]; /* Symbolic name of this SELECT use for debugging */ +#endif int addrOpenEphm[2]; /* OP_OpenEphem opcodes related to this select */ u64 nSelectRow; /* Estimated number of result rows */ SrcList *pSrc; /* The FROM clause */ @@ -2234,13 +2347,13 @@ struct Select { #define SF_UsesEphemeral 0x0008 /* Uses the OpenEphemeral opcode */ #define SF_Expanded 0x0010 /* sqlite3SelectExpand() called on this */ #define SF_HasTypeInfo 0x0020 /* FROM subqueries have Table metadata */ - /* 0x0040 NOT USED */ +#define SF_Compound 0x0040 /* Part of a compound query */ #define SF_Values 0x0080 /* Synthesized from VALUES clause */ /* 0x0100 NOT USED */ #define SF_NestedFrom 0x0200 /* Part of a parenthesized FROM clause */ #define SF_MaybeConvert 0x0400 /* Need convertCompoundSelectToSubquery() */ #define SF_Recursive 0x0800 /* The recursive part of a recursive CTE */ -#define SF_Compound 0x1000 /* Part of a compound query */ +#define SF_MinMaxAgg 0x1000 /* Aggregate containing min() or max() */ /* @@ -2392,9 +2505,19 @@ struct TriggerPrg { ** The yDbMask datatype for the bitmask of all attached databases. */ #if SQLITE_MAX_ATTACHED>30 - typedef sqlite3_uint64 yDbMask; + typedef unsigned char yDbMask[(SQLITE_MAX_ATTACHED+9)/8]; +# define DbMaskTest(M,I) (((M)[(I)/8]&(1<<((I)&7)))!=0) +# define DbMaskZero(M) memset((M),0,sizeof(M)) +# define DbMaskSet(M,I) (M)[(I)/8]|=(1<<((I)&7)) +# define DbMaskAllZero(M) sqlite3DbMaskAllZero(M) +# define DbMaskNonZero(M) (sqlite3DbMaskAllZero(M)==0) #else typedef unsigned int yDbMask; +# define DbMaskTest(M,I) (((M)&(((yDbMask)1)<<(I)))!=0) +# define DbMaskZero(M) (M)=0 +# define DbMaskSet(M,I) (M)|=(((yDbMask)1)<<(I)) +# define DbMaskAllZero(M) (M)==0 +# define DbMaskNonZero(M) (M)!=0 #endif /* @@ -2458,6 +2581,10 @@ struct Parse { int regRowid; /* Register holding rowid of CREATE TABLE entry */ int regRoot; /* Register holding root page number for new objects */ int nMaxArg; /* Max args passed to user function by sub-program */ +#if SELECTTRACE_ENABLED + int nSelect; /* Number of SELECT statements seen */ + int nSelectIndent; /* How far to indent SELECTTRACE() output */ +#endif #ifndef SQLITE_OMIT_SHARED_CACHE int nTableLock; /* Number of locks in aTableLock */ TableLock *aTableLock; /* Required table locks for shared-cache mode */ @@ -2537,11 +2664,11 @@ struct AuthContext { ** Bitfield flags for P5 value in various opcodes. */ #define OPFLAG_NCHANGE 0x01 /* Set to update db->nChange */ +#define OPFLAG_EPHEM 0x01 /* OP_Column: Ephemeral output is ok */ #define OPFLAG_LASTROWID 0x02 /* Set to update db->lastRowid */ #define OPFLAG_ISUPDATE 0x04 /* This OP_Insert is an sql UPDATE */ #define OPFLAG_APPEND 0x08 /* This is likely to be an append */ #define OPFLAG_USESEEKRESULT 0x10 /* Try to avoid a seek in BtreeInsert() */ -#define OPFLAG_CLEARCACHE 0x20 /* Clear pseudo-table cache in OP_Column */ #define OPFLAG_LENGTHARG 0x40 /* OP_Column only used for length() */ #define OPFLAG_TYPEOFARG 0x80 /* OP_Column only used for typeof() */ #define OPFLAG_BULKCSR 0x01 /* OP_Open** used to open bulk cursor */ @@ -2767,9 +2894,11 @@ struct Walker { void (*xSelectCallback2)(Walker*,Select*);/* Second callback for SELECTs */ Parse *pParse; /* Parser context. */ int walkerDepth; /* Number of subqueries */ + u8 eCode; /* A small processing code */ union { /* Extra data for callback */ NameContext *pNC; /* Naming context */ - int i; /* Integer value */ + int n; /* A counter */ + int iCur; /* A cursor number */ SrcList *pSrcList; /* FROM clause */ struct SrcCount *pSrcCount; /* Counting column references */ } u; @@ -2805,6 +2934,17 @@ struct With { } a[1]; }; +#ifdef SQLITE_DEBUG +/* +** An instance of the TreeView object is used for printing the content of +** data structures on sqlite3DebugPrintf() using a tree-like view. +*/ +struct TreeView { + int iLevel; /* Which level of the tree we are on */ + u8 bLine[100]; /* Draw vertical in column i if bLine[i] is true */ +}; +#endif /* SQLITE_DEBUG */ + /* ** Assuming zIn points to the first byte of a UTF-8 character, ** advance zIn to point to the first byte of the next UTF-8 character. @@ -2832,8 +2972,8 @@ int sqlite3CantopenError(int); /* ** FTS4 is really an extension for FTS3. It is enabled using the -** SQLITE_ENABLE_FTS3 macro. But to avoid confusion we also all -** the SQLITE_ENABLE_FTS4 macro to serve as an alisse for SQLITE_ENABLE_FTS3. +** SQLITE_ENABLE_FTS3 macro. But to avoid confusion we also call +** the SQLITE_ENABLE_FTS4 macro to serve as an alias for SQLITE_ENABLE_FTS3. */ #if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3) # define SQLITE_ENABLE_FTS3 @@ -2870,6 +3010,7 @@ int sqlite3CantopenError(int); # define sqlite3Isxdigit(x) isxdigit((unsigned char)(x)) # define sqlite3Tolower(x) tolower((unsigned char)(x)) #endif +int sqlite3IsIdChar(u8); /* ** Internal function prototypes @@ -2880,15 +3021,15 @@ int sqlite3Strlen30(const char*); int sqlite3MallocInit(void); void sqlite3MallocEnd(void); -void *sqlite3Malloc(int); -void *sqlite3MallocZero(int); -void *sqlite3DbMallocZero(sqlite3*, int); -void *sqlite3DbMallocRaw(sqlite3*, int); +void *sqlite3Malloc(u64); +void *sqlite3MallocZero(u64); +void *sqlite3DbMallocZero(sqlite3*, u64); +void *sqlite3DbMallocRaw(sqlite3*, u64); char *sqlite3DbStrDup(sqlite3*,const char*); -char *sqlite3DbStrNDup(sqlite3*,const char*, int); -void *sqlite3Realloc(void*, int); -void *sqlite3DbReallocOrFree(sqlite3 *, void *, int); -void *sqlite3DbRealloc(sqlite3 *, void *, int); +char *sqlite3DbStrNDup(sqlite3*,const char*, u64); +void *sqlite3Realloc(void*, u64); +void *sqlite3DbReallocOrFree(sqlite3 *, void *, u64); +void *sqlite3DbRealloc(sqlite3 *, void *, u64); void sqlite3DbFree(sqlite3*, void*); int sqlite3MallocSize(void*); int sqlite3DbMallocSize(sqlite3*, void*); @@ -2968,25 +3109,14 @@ char *sqlite3MAppendf(sqlite3*,char*,const char*,...); void *sqlite3TestTextToPtr(const char*); #endif -/* Output formatting for SQLITE_TESTCTRL_EXPLAIN */ -#if defined(SQLITE_ENABLE_TREE_EXPLAIN) - void sqlite3ExplainBegin(Vdbe*); - void sqlite3ExplainPrintf(Vdbe*, const char*, ...); - void sqlite3ExplainNL(Vdbe*); - void sqlite3ExplainPush(Vdbe*); - void sqlite3ExplainPop(Vdbe*); - void sqlite3ExplainFinish(Vdbe*); - void sqlite3ExplainSelect(Vdbe*, Select*); - void sqlite3ExplainExpr(Vdbe*, Expr*); - void sqlite3ExplainExprList(Vdbe*, ExprList*); - const char *sqlite3VdbeExplanation(Vdbe*); -#else -# define sqlite3ExplainBegin(X) -# define sqlite3ExplainSelect(A,B) -# define sqlite3ExplainExpr(A,B) -# define sqlite3ExplainExprList(A,B) -# define sqlite3ExplainFinish(X) -# define sqlite3VdbeExplanation(X) 0 +#if defined(SQLITE_DEBUG) + TreeView *sqlite3TreeViewPush(TreeView*,u8); + void sqlite3TreeViewPop(TreeView*); + void sqlite3TreeViewLine(TreeView*, const char*, ...); + void sqlite3TreeViewItem(TreeView*, const char*, u8); + void sqlite3TreeViewExpr(TreeView*, const Expr*, u8); + void sqlite3TreeViewExprList(TreeView*, const ExprList*, u8, const char*); + void sqlite3TreeViewSelect(TreeView*, const Select*, u8); #endif @@ -3067,6 +3197,9 @@ void sqlite3CreateView(Parse*,Token*,Token*,Token*,Select*,int,int); # define sqlite3ViewGetColumnNames(A,B) 0 #endif +#if SQLITE_MAX_ATTACHED>30 + int sqlite3DbMaskAllZero(yDbMask); +#endif void sqlite3DropTable(Parse*, SrcList*, int, int); void sqlite3CodeDropTable(Parse*, Table*, int, int); void sqlite3DeleteTable(sqlite3*, Table*); @@ -3165,7 +3298,8 @@ void sqlite3CloseSavepoints(sqlite3 *); void sqlite3LeaveMutexAndCloseZombie(sqlite3*); int sqlite3ExprIsConstant(Expr*); int sqlite3ExprIsConstantNotJoin(Expr*); -int sqlite3ExprIsConstantOrFunction(Expr*); +int sqlite3ExprIsConstantOrFunction(Expr*, u8); +int sqlite3ExprIsTableConstant(Expr*,int); int sqlite3ExprIsInteger(Expr*, int*); int sqlite3ExprCanBeNull(const Expr*); int sqlite3ExprNeedsNoAffinityChange(const Expr*, char); @@ -3189,6 +3323,11 @@ ExprList *sqlite3ExprListDup(sqlite3*,ExprList*,int); SrcList *sqlite3SrcListDup(sqlite3*,SrcList*,int); IdList *sqlite3IdListDup(sqlite3*,IdList*); Select *sqlite3SelectDup(sqlite3*,Select*,int); +#if SELECTTRACE_ENABLED +void sqlite3SelectSetName(Select*,const char*); +#else +# define sqlite3SelectSetName(A,B) +#endif void sqlite3FuncDefInsert(FuncDefHash*, FuncDef*); FuncDef *sqlite3FindFunction(sqlite3*,const char*,int,int,u8,u8); void sqlite3RegisterBuiltinFunctions(sqlite3*); @@ -3275,38 +3414,23 @@ u64 sqlite3LogEstToInt(LogEst); /* ** Routines to read and write variable-length integers. These used to ** be defined locally, but now we use the varint routines in the util.c -** file. Code should use the MACRO forms below, as the Varint32 versions -** are coded to assume the single byte case is already handled (which -** the MACRO form does). +** file. */ int sqlite3PutVarint(unsigned char*, u64); -int sqlite3PutVarint32(unsigned char*, u32); u8 sqlite3GetVarint(const unsigned char *, u64 *); u8 sqlite3GetVarint32(const unsigned char *, u32 *); int sqlite3VarintLen(u64 v); /* -** The header of a record consists of a sequence variable-length integers. -** These integers are almost always small and are encoded as a single byte. -** The following macros take advantage this fact to provide a fast encode -** and decode of the integers in a record header. It is faster for the common -** case where the integer is a single byte. It is a little slower when the -** integer is two or more bytes. But overall it is faster. -** -** The following expressions are equivalent: -** -** x = sqlite3GetVarint32( A, &B ); -** x = sqlite3PutVarint32( A, B ); -** -** x = getVarint32( A, B ); -** x = putVarint32( A, B ); -** +** The common case is for a varint to be a single byte. They following +** macros handle the common case without a procedure call, but then call +** the procedure for larger varints. */ #define getVarint32(A,B) \ (u8)((*(A)<(u8)0x80)?((B)=(u32)*(A)),1:sqlite3GetVarint32((A),(u32 *)&(B))) #define putVarint32(A,B) \ (u8)(((u32)(B)<(u32)0x80)?(*(A)=(unsigned char)(B)),1:\ - sqlite3PutVarint32((A),(B))) + sqlite3PutVarint((A),(B))) #define getVarint sqlite3GetVarint #define putVarint sqlite3PutVarint @@ -3317,7 +3441,9 @@ char sqlite3CompareAffinity(Expr *pExpr, char aff2); int sqlite3IndexAffinityOk(Expr *pExpr, char idx_affinity); char sqlite3ExprAffinity(Expr *pExpr); int sqlite3Atoi64(const char*, i64*, int, u8); -void sqlite3Error(sqlite3*, int, const char*,...); +int sqlite3DecOrHexToI64(const char*, i64*); +void sqlite3ErrorWithMsg(sqlite3*, int, const char*,...); +void sqlite3Error(sqlite3*,int); void *sqlite3HexToBlob(sqlite3*, const char *z, int n); u8 sqlite3HexToInt(int h); int sqlite3TwoPartName(Parse *, Token *, Token *, Token **); @@ -3346,7 +3472,7 @@ void sqlite3FileSuffix3(const char*, char*); #else # define sqlite3FileSuffix3(X,Y) #endif -u8 sqlite3GetBoolean(const char *z,int); +u8 sqlite3GetBoolean(const char *z,u8); const void *sqlite3ValueText(sqlite3_value*, u8); int sqlite3ValueBytes(sqlite3_value*, u8); @@ -3419,7 +3545,7 @@ int sqlite3OpenTempDatabase(Parse *); void sqlite3StrAccumInit(StrAccum*, char*, int, int); void sqlite3StrAccumAppend(StrAccum*,const char*,int); void sqlite3StrAccumAppendAll(StrAccum*,const char*); -void sqlite3AppendSpace(StrAccum*,int); +void sqlite3AppendChar(StrAccum*,int,char); char *sqlite3StrAccumFinish(StrAccum*); void sqlite3StrAccumReset(StrAccum*); void sqlite3SelectDestInit(SelectDest*,int,int); @@ -3431,13 +3557,15 @@ void sqlite3BackupUpdate(sqlite3_backup *, Pgno, const u8 *); #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 void sqlite3AnalyzeFunctions(void); int sqlite3Stat4ProbeSetValue(Parse*,Index*,UnpackedRecord**,Expr*,u8,int,int*); +int sqlite3Stat4ValueFromExpr(Parse*, Expr*, u8, sqlite3_value**); void sqlite3Stat4ProbeFree(UnpackedRecord*); +int sqlite3Stat4Column(sqlite3*, const void*, int, int, sqlite3_value**); #endif /* ** The interface to the LEMON-generated parser */ -void *sqlite3ParserAlloc(void*(*)(size_t)); +void *sqlite3ParserAlloc(void*(*)(u64)); void sqlite3ParserFree(void*, void(*)(void*)); void sqlite3Parser(void*, int, Token, Parse*); #ifdef YYTRACKMAXSTACKDEPTH @@ -3568,11 +3696,21 @@ const char *sqlite3JournalModename(int); #define sqlite3EndBenignMalloc() #endif -#define IN_INDEX_ROWID 1 -#define IN_INDEX_EPH 2 -#define IN_INDEX_INDEX_ASC 3 -#define IN_INDEX_INDEX_DESC 4 -int sqlite3FindInIndex(Parse *, Expr *, int*); +/* +** Allowed return values from sqlite3FindInIndex() +*/ +#define IN_INDEX_ROWID 1 /* Search the rowid of the table */ +#define IN_INDEX_EPH 2 /* Search an ephemeral b-tree */ +#define IN_INDEX_INDEX_ASC 3 /* Existing index ASCENDING */ +#define IN_INDEX_INDEX_DESC 4 /* Existing index DESCENDING */ +#define IN_INDEX_NOOP 5 /* No table available. Use comparisons */ +/* +** Allowed flags for the 3rd parameter to sqlite3FindInIndex(). +*/ +#define IN_INDEX_NOOP_OK 0x0001 /* OK to return IN_INDEX_NOOP */ +#define IN_INDEX_MEMBERSHIP 0x0002 /* IN operator used for membership test */ +#define IN_INDEX_LOOP 0x0004 /* IN operator used as a loop */ +int sqlite3FindInIndex(Parse *, Expr *, u32, int*); #ifdef SQLITE_ENABLE_ATOMIC_WRITE int sqlite3JournalOpen(sqlite3_vfs *, const char *, sqlite3_file *, int, int); @@ -3667,9 +3805,16 @@ SQLITE_EXTERN void (*sqlite3IoTrace)(const char*,...); # define sqlite3MemdebugNoType(X,Y) 1 #endif #define MEMTYPE_HEAP 0x01 /* General heap allocations */ -#define MEMTYPE_LOOKASIDE 0x02 /* Might have been lookaside memory */ +#define MEMTYPE_LOOKASIDE 0x02 /* Heap that might have been lookaside */ #define MEMTYPE_SCRATCH 0x04 /* Scratch allocations */ #define MEMTYPE_PCACHE 0x08 /* Page cache allocations */ -#define MEMTYPE_DB 0x10 /* Uses sqlite3DbMalloc, not sqlite_malloc */ + +/* +** Threading interface +*/ +#if SQLITE_MAX_WORKER_THREADS>0 +int sqlite3ThreadCreate(SQLiteThread**,void*(*)(void*),void*); +int sqlite3ThreadJoin(SQLiteThread*, void**); +#endif #endif /* _SQLITEINT_H_ */ diff --git a/src/status.c b/src/status.c index 5fcb68ddc3..4c2eabb661 100644 --- a/src/status.c +++ b/src/status.c @@ -86,6 +86,9 @@ int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag){ if( op<0 || op>=ArraySize(wsdStat.nowValue) ){ return SQLITE_MISUSE_BKPT; } +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCurrent==0 || pHighwater==0 ) return SQLITE_MISUSE_BKPT; +#endif *pCurrent = wsdStat.nowValue[op]; *pHighwater = wsdStat.mxValue[op]; if( resetFlag ){ @@ -105,6 +108,11 @@ int sqlite3_db_status( int resetFlag /* Reset high-water mark if true */ ){ int rc = SQLITE_OK; /* Return code */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || pCurrent==0|| pHighwater==0 ){ + return SQLITE_MISUSE_BKPT; + } +#endif sqlite3_mutex_enter(db->mutex); switch( op ){ case SQLITE_DBSTATUS_LOOKASIDE_USED: { @@ -213,7 +221,7 @@ int sqlite3_db_status( } db->pnBytesFreed = 0; - *pHighwater = 0; + *pHighwater = 0; /* IMP: R-64479-57858 */ *pCurrent = nByte; break; @@ -238,7 +246,9 @@ int sqlite3_db_status( sqlite3PagerCacheStat(pPager, op, resetFlag, &nRet); } } - *pHighwater = 0; + *pHighwater = 0; /* IMP: R-42420-56072 */ + /* IMP: R-54100-20147 */ + /* IMP: R-29431-39229 */ *pCurrent = nRet; break; } @@ -248,7 +258,7 @@ int sqlite3_db_status( ** have been satisfied. The *pHighwater is always set to zero. */ case SQLITE_DBSTATUS_DEFERRED_FKS: { - *pHighwater = 0; + *pHighwater = 0; /* IMP: R-11967-56545 */ *pCurrent = db->nDeferredImmCons>0 || db->nDeferredCons>0; break; } diff --git a/src/table.c b/src/table.c index 26bbfb4f45..235d8dd3df 100644 --- a/src/table.c +++ b/src/table.c @@ -29,10 +29,10 @@ typedef struct TabResult { char **azResult; /* Accumulated output */ char *zErrMsg; /* Error message text, if an error occurs */ - int nAlloc; /* Slots allocated for azResult[] */ - int nRow; /* Number of rows in the result */ - int nColumn; /* Number of columns in the result */ - int nData; /* Slots used in azResult[]. (nRow+1)*nColumn */ + u32 nAlloc; /* Slots allocated for azResult[] */ + u32 nRow; /* Number of rows in the result */ + u32 nColumn; /* Number of columns in the result */ + u32 nData; /* Slots used in azResult[]. (nRow+1)*nColumn */ int rc; /* Return code from sqlite3_exec() */ } TabResult; @@ -58,7 +58,7 @@ static int sqlite3_get_table_cb(void *pArg, int nCol, char **argv, char **colv){ if( p->nData + need > p->nAlloc ){ char **azNew; p->nAlloc = p->nAlloc*2 + need; - azNew = sqlite3_realloc( p->azResult, sizeof(char*)*p->nAlloc ); + azNew = sqlite3_realloc64( p->azResult, sizeof(char*)*p->nAlloc ); if( azNew==0 ) goto malloc_failed; p->azResult = azNew; } @@ -73,7 +73,7 @@ static int sqlite3_get_table_cb(void *pArg, int nCol, char **argv, char **colv){ if( z==0 ) goto malloc_failed; p->azResult[p->nData++] = z; } - }else if( p->nColumn!=nCol ){ + }else if( (int)p->nColumn!=nCol ){ sqlite3_free(p->zErrMsg); p->zErrMsg = sqlite3_mprintf( "sqlite3_get_table() called with two or more incompatible queries" @@ -126,6 +126,9 @@ int sqlite3_get_table( int rc; TabResult res; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || pazResult==0 ) return SQLITE_MISUSE_BKPT; +#endif *pazResult = 0; if( pnColumn ) *pnColumn = 0; if( pnRow ) *pnRow = 0; @@ -182,7 +185,7 @@ int sqlite3_get_table( ** This routine frees the space the sqlite3_get_table() malloced. */ void sqlite3_free_table( - char **azResult /* Result returned from from sqlite3_get_table() */ + char **azResult /* Result returned from sqlite3_get_table() */ ){ if( azResult ){ int i, n; diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 92a107cd77..94d71574c5 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -635,6 +635,7 @@ static int DbWalHandler( Tcl_Interp *interp = pDb->interp; assert(pDb->pWalHook); + assert( db==pDb->db ); p = Tcl_DuplicateObj(pDb->pWalHook); Tcl_IncrRefCount(p); Tcl_ListObjAppendElement(interp, p, Tcl_NewStringObj(zDb, -1)); @@ -760,7 +761,7 @@ static void tclSqlFunc(sqlite3_context *context, int argc, sqlite3_value**argv){ /* If there are arguments to the function, make a shallow copy of the ** script object, lappend the arguments, then evaluate the copy. ** - ** By "shallow" copy, we mean a only the outer list Tcl_Obj is duplicated. + ** By "shallow" copy, we mean only the outer list Tcl_Obj is duplicated. ** The new Tcl_Obj contains pointers to the original list elements. ** That way, when Tcl_EvalObjv() is run and shimmers the first element ** of the list to tclCmdNameType, that alternate representation will @@ -872,6 +873,9 @@ static int auth_callback( const char *zArg2, const char *zArg3, const char *zArg4 +#ifdef SQLITE_USER_AUTHENTICATION + ,const char *zArg5 +#endif ){ const char *zCode; Tcl_DString str; @@ -924,6 +928,9 @@ static int auth_callback( Tcl_DStringAppendElement(&str, zArg2 ? zArg2 : ""); Tcl_DStringAppendElement(&str, zArg3 ? zArg3 : ""); Tcl_DStringAppendElement(&str, zArg4 ? zArg4 : ""); +#ifdef SQLITE_USER_AUTHENTICATION + Tcl_DStringAppendElement(&str, zArg5 ? zArg5 : ""); +#endif rc = Tcl_GlobalEval(pDb->interp, Tcl_DStringValue(&str)); Tcl_DStringFree(&str); zReply = rc==TCL_OK ? Tcl_GetStringResult(pDb->interp) : "SQLITE_DENY"; @@ -1700,8 +1707,11 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){ pDb->zAuth = 0; } if( pDb->zAuth ){ + typedef int (*sqlite3_auth_cb)( + void*,int,const char*,const char*, + const char*,const char*); pDb->interp = interp; - sqlite3_set_authorizer(pDb->db, auth_callback, pDb); + sqlite3_set_authorizer(pDb->db,(sqlite3_auth_cb)auth_callback,pDb); }else{ sqlite3_set_authorizer(pDb->db, 0, 0); } @@ -2381,7 +2391,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){ if( rc==TCL_OK ){ rc = createIncrblobChannel( - interp, pDb, zDb, zTable, zColumn, iRow, isReadonly + interp, pDb, zDb, zTable, zColumn, (sqlite3_int64)iRow, isReadonly ); } #endif @@ -3632,6 +3642,45 @@ static int db_use_legacy_prepare_cmd( Tcl_ResetResult(interp); return TCL_OK; } + +/* +** Tclcmd: db_last_stmt_ptr DB +** +** If the statement cache associated with database DB is not empty, +** return the text representation of the most recently used statement +** handle. +*/ +static int db_last_stmt_ptr( + ClientData cd, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + extern int sqlite3TestMakePointerStr(Tcl_Interp*, char*, void*); + Tcl_CmdInfo cmdInfo; + SqliteDb *pDb; + sqlite3_stmt *pStmt = 0; + char zBuf[100]; + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB"); + return TCL_ERROR; + } + + if( !Tcl_GetCommandInfo(interp, Tcl_GetString(objv[1]), &cmdInfo) ){ + Tcl_AppendResult(interp, "no such db: ", Tcl_GetString(objv[1]), (char*)0); + return TCL_ERROR; + } + pDb = (SqliteDb*)cmdInfo.objClientData; + + if( pDb->stmtList ) pStmt = pDb->stmtList->pStmt; + if( sqlite3TestMakePointerStr(interp, zBuf, pStmt) ){ + return TCL_ERROR; + } + Tcl_SetResult(interp, zBuf, TCL_VOLATILE); + + return TCL_OK; +} #endif /* @@ -3677,6 +3726,7 @@ static void init_all(Tcl_Interp *interp){ extern int Sqlitetest9_Init(Tcl_Interp*); extern int Sqlitetestasync_Init(Tcl_Interp*); extern int Sqlitetest_autoext_Init(Tcl_Interp*); + extern int Sqlitetest_blob_Init(Tcl_Interp*); extern int Sqlitetest_demovfs_Init(Tcl_Interp *); extern int Sqlitetest_func_Init(Tcl_Interp*); extern int Sqlitetest_hexio_Init(Tcl_Interp*); @@ -3721,6 +3771,7 @@ static void init_all(Tcl_Interp *interp){ Sqlitetest9_Init(interp); Sqlitetestasync_Init(interp); Sqlitetest_autoext_Init(interp); + Sqlitetest_blob_Init(interp); Sqlitetest_demovfs_Init(interp); Sqlitetest_func_Init(interp); Sqlitetest_hexio_Init(interp); @@ -3753,6 +3804,9 @@ static void init_all(Tcl_Interp *interp){ Tcl_CreateObjCommand( interp, "db_use_legacy_prepare", db_use_legacy_prepare_cmd, 0, 0 ); + Tcl_CreateObjCommand( + interp, "db_last_stmt_ptr", db_last_stmt_ptr, 0, 0 + ); #ifdef SQLITE_SSE Sqlitetestsse_Init(interp); diff --git a/src/test1.c b/src/test1.c index 8d2249cbb8..032e3f338c 100644 --- a/src/test1.c +++ b/src/test1.c @@ -1569,7 +1569,6 @@ static int test_libversion_number( ** Usage: sqlite3_table_column_metadata DB dbname tblname colname ** */ -#ifdef SQLITE_ENABLE_COLUMN_METADATA static int test_table_column_metadata( ClientData clientData, /* Pointer to sqlite3_enable_XXX function */ Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ @@ -1589,14 +1588,14 @@ static int test_table_column_metadata( int primarykey; int autoincrement; - if( objc!=5 ){ + if( objc!=5 && objc!=4 ){ Tcl_WrongNumArgs(interp, 1, objv, "DB dbname tblname colname"); return TCL_ERROR; } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; zDb = Tcl_GetString(objv[2]); zTbl = Tcl_GetString(objv[3]); - zCol = Tcl_GetString(objv[4]); + zCol = objc==5 ? Tcl_GetString(objv[4]) : 0; if( strlen(zDb)==0 ) zDb = 0; @@ -1618,7 +1617,6 @@ static int test_table_column_metadata( return TCL_OK; } -#endif #ifndef SQLITE_OMIT_INCRBLOB @@ -1651,154 +1649,6 @@ static int blobHandleFromObj( return TCL_OK; } -/* -** sqlite3_blob_bytes CHANNEL -*/ -static int test_blob_bytes( - ClientData clientData, /* Not used */ - Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ - int objc, /* Number of arguments */ - Tcl_Obj *CONST objv[] /* Command arguments */ -){ - sqlite3_blob *pBlob; - int nByte; - - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 1, objv, "CHANNEL"); - return TCL_ERROR; - } - - if( blobHandleFromObj(interp, objv[1], &pBlob) ) return TCL_ERROR; - nByte = sqlite3_blob_bytes(pBlob); - Tcl_SetObjResult(interp, Tcl_NewIntObj(nByte)); - - return TCL_OK; -} - -/* -** sqlite3_blob_close CHANNEL -*/ -static int test_blob_close( - ClientData clientData, /* Not used */ - Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ - int objc, /* Number of arguments */ - Tcl_Obj *CONST objv[] /* Command arguments */ -){ - sqlite3_blob *pBlob; - - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 1, objv, "CHANNEL"); - return TCL_ERROR; - } - - if( blobHandleFromObj(interp, objv[1], &pBlob) ) return TCL_ERROR; - sqlite3_blob_close(pBlob); - - return TCL_OK; -} - -/* -** sqlite3_blob_read CHANNEL OFFSET N -** -** This command is used to test the sqlite3_blob_read() in ways that -** the Tcl channel interface does not. The first argument should -** be the name of a valid channel created by the [incrblob] method -** of a database handle. This function calls sqlite3_blob_read() -** to read N bytes from offset OFFSET from the underlying SQLite -** blob handle. -** -** On success, a byte-array object containing the read data is -** returned. On failure, the interpreter result is set to the -** text representation of the returned error code (i.e. "SQLITE_NOMEM") -** and a Tcl exception is thrown. -*/ -static int test_blob_read( - ClientData clientData, /* Not used */ - Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ - int objc, /* Number of arguments */ - Tcl_Obj *CONST objv[] /* Command arguments */ -){ - sqlite3_blob *pBlob; - int nByte; - int iOffset; - unsigned char *zBuf = 0; - int rc; - - if( objc!=4 ){ - Tcl_WrongNumArgs(interp, 1, objv, "CHANNEL OFFSET N"); - return TCL_ERROR; - } - - if( blobHandleFromObj(interp, objv[1], &pBlob) ) return TCL_ERROR; - if( TCL_OK!=Tcl_GetIntFromObj(interp, objv[2], &iOffset) - || TCL_OK!=Tcl_GetIntFromObj(interp, objv[3], &nByte) - ){ - return TCL_ERROR; - } - - if( nByte>0 ){ - zBuf = (unsigned char *)Tcl_Alloc(nByte); - } - rc = sqlite3_blob_read(pBlob, zBuf, nByte, iOffset); - if( rc==SQLITE_OK ){ - Tcl_SetObjResult(interp, Tcl_NewByteArrayObj(zBuf, nByte)); - }else{ - Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); - } - Tcl_Free((char *)zBuf); - - return (rc==SQLITE_OK ? TCL_OK : TCL_ERROR); -} - -/* -** sqlite3_blob_write CHANNEL OFFSET DATA ?NDATA? -** -** This command is used to test the sqlite3_blob_write() in ways that -** the Tcl channel interface does not. The first argument should -** be the name of a valid channel created by the [incrblob] method -** of a database handle. This function calls sqlite3_blob_write() -** to write the DATA byte-array to the underlying SQLite blob handle. -** at offset OFFSET. -** -** On success, an empty string is returned. On failure, the interpreter -** result is set to the text representation of the returned error code -** (i.e. "SQLITE_NOMEM") and a Tcl exception is thrown. -*/ -static int test_blob_write( - ClientData clientData, /* Not used */ - Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ - int objc, /* Number of arguments */ - Tcl_Obj *CONST objv[] /* Command arguments */ -){ - sqlite3_blob *pBlob; - int iOffset; - int rc; - - unsigned char *zBuf; - int nBuf; - - if( objc!=4 && objc!=5 ){ - Tcl_WrongNumArgs(interp, 1, objv, "CHANNEL OFFSET DATA ?NDATA?"); - return TCL_ERROR; - } - - if( blobHandleFromObj(interp, objv[1], &pBlob) ) return TCL_ERROR; - if( TCL_OK!=Tcl_GetIntFromObj(interp, objv[2], &iOffset) ){ - return TCL_ERROR; - } - - zBuf = Tcl_GetByteArrayFromObj(objv[3], &nBuf); - if( objc==5 && Tcl_GetIntFromObj(interp, objv[4], &nBuf) ){ - return TCL_ERROR; - } - rc = sqlite3_blob_write(pBlob, zBuf, nBuf, iOffset); - if( rc!=SQLITE_OK ){ - Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); - } - - return (rc==SQLITE_OK ? TCL_OK : TCL_ERROR); -} - static int test_blob_reopen( ClientData clientData, /* Not used */ Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ @@ -2301,6 +2151,77 @@ static int test_stmt_status( return TCL_OK; } +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +/* +** Usage: sqlite3_stmt_scanstatus STMT IDX +*/ +static int test_stmt_scanstatus( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3_stmt *pStmt; /* First argument */ + int idx; /* Second argument */ + + const char *zName; + const char *zExplain; + sqlite3_int64 nLoop; + sqlite3_int64 nVisit; + double rEst; + int res; + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "STMT IDX"); + return TCL_ERROR; + } + if( getStmtPointer(interp, Tcl_GetString(objv[1]), &pStmt) ) return TCL_ERROR; + if( Tcl_GetIntFromObj(interp, objv[2], &idx) ) return TCL_ERROR; + + res = sqlite3_stmt_scanstatus(pStmt, idx, SQLITE_SCANSTAT_NLOOP, (void*)&nLoop); + if( res==0 ){ + Tcl_Obj *pRet = Tcl_NewObj(); + Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj("nLoop", -1)); + Tcl_ListObjAppendElement(0, pRet, Tcl_NewWideIntObj(nLoop)); + sqlite3_stmt_scanstatus(pStmt, idx, SQLITE_SCANSTAT_NVISIT, (void*)&nVisit); + Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj("nVisit", -1)); + Tcl_ListObjAppendElement(0, pRet, Tcl_NewWideIntObj(nVisit)); + sqlite3_stmt_scanstatus(pStmt, idx, SQLITE_SCANSTAT_EST, (void*)&rEst); + Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj("nEst", -1)); + Tcl_ListObjAppendElement(0, pRet, Tcl_NewDoubleObj(rEst)); + sqlite3_stmt_scanstatus(pStmt, idx, SQLITE_SCANSTAT_NAME, (void*)&zName); + Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj("zName", -1)); + Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zName, -1)); + sqlite3_stmt_scanstatus(pStmt, idx, SQLITE_SCANSTAT_EXPLAIN, (void*)&zExplain); + Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj("zExplain", -1)); + Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zExplain, -1)); + Tcl_SetObjResult(interp, pRet); + }else{ + Tcl_ResetResult(interp); + } + return TCL_OK; +} + +/* +** Usage: sqlite3_stmt_scanstatus_reset STMT +*/ +static int test_stmt_scanstatus_reset( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3_stmt *pStmt; /* First argument */ + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "STMT"); + return TCL_ERROR; + } + if( getStmtPointer(interp, Tcl_GetString(objv[1]), &pStmt) ) return TCL_ERROR; + sqlite3_stmt_scanstatus_reset(pStmt); + return TCL_OK; +} +#endif + /* ** Usage: sqlite3_next_stmt DB STMT ** @@ -2605,7 +2526,7 @@ static int test_bind( ** SQLite selected to call. The TCL test script implements the ** "test_collate" proc. ** -** Note that this will only work with one intepreter at a time, as the +** Note that this will only work with one interpreter at a time, as the ** interp pointer to use when evaluating the TCL script is stored in ** pTestCollateInterp. */ @@ -2717,6 +2638,46 @@ bad_args: return TCL_ERROR; } +/* +** Usage: add_test_utf16bin_collate +** +** Add a utf-16 collation sequence named "utf16bin" to the database +** handle. This collation sequence compares arguments in the same way as the +** built-in collation "binary". +*/ +static int test_utf16bin_collate_func( + void *pCtx, + int nA, const void *zA, + int nB, const void *zB +){ + int nCmp = (nA>nB ? nB : nA); + int res = memcmp(zA, zB, nCmp); + if( res==0 ) res = nA - nB; + return res; +} +static int test_utf16bin_collate( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3 *db; + int rc; + + if( objc!=2 ) goto bad_args; + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; + + rc = sqlite3_create_collation(db, "utf16bin", SQLITE_UTF16, 0, + test_utf16bin_collate_func + ); + if( sqlite3TestErrCode(interp, db, rc) ) return TCL_ERROR; + return TCL_OK; + +bad_args: + Tcl_WrongNumArgs(interp, 1, objv, "DB"); + return TCL_ERROR; +} + /* ** When the collation needed callback is invoked, record the name of ** the requested collating function here. The recorded name is linked @@ -3675,6 +3636,7 @@ static int test_prepare_v2( ){ sqlite3 *db; const char *zSql; + char *zCopy = 0; /* malloc() copy of zSql */ int bytes; const char *zTail = 0; sqlite3_stmt *pStmt = 0; @@ -3690,11 +3652,25 @@ static int test_prepare_v2( zSql = Tcl_GetString(objv[2]); if( Tcl_GetIntFromObj(interp, objv[3], &bytes) ) return TCL_ERROR; - rc = sqlite3_prepare_v2(db, zSql, bytes, &pStmt, objc>=5 ? &zTail : 0); + /* Instead of using zSql directly, make a copy into a buffer obtained + ** directly from malloc(). The idea is to make it easier for valgrind + ** to spot buffer overreads. */ + if( bytes>=0 ){ + zCopy = malloc(bytes); + memcpy(zCopy, zSql, bytes); + }else{ + int n = (int)strlen(zSql) + 1; + zCopy = malloc(n); + memcpy(zCopy, zSql, n); + } + rc = sqlite3_prepare_v2(db, zCopy, bytes, &pStmt, objc>=5 ? &zTail : 0); + free(zCopy); + zTail = &zSql[(zTail - zCopy)]; + assert(rc==SQLITE_OK || pStmt==0); Tcl_ResetResult(interp); if( sqlite3TestErrCode(interp, db, rc) ) return TCL_ERROR; - if( zTail && objc>=5 ){ + if( rc==SQLITE_OK && zTail && objc>=5 ){ if( bytes>=0 ){ bytes = bytes - (int)(zTail-zSql); } @@ -3718,7 +3694,7 @@ static int test_prepare_v2( ** Usage: sqlite3_prepare_tkt3134 DB ** ** Generate a prepared statement for a zero-byte string as a test -** for ticket #3134. The string should be preceeded by a zero byte. +** for ticket #3134. The string should be preceded by a zero byte. */ static int test_prepare_tkt3134( void * clientData, @@ -5473,10 +5449,11 @@ static int test_limit( { "SQLITE_LIMIT_LIKE_PATTERN_LENGTH", SQLITE_LIMIT_LIKE_PATTERN_LENGTH }, { "SQLITE_LIMIT_VARIABLE_NUMBER", SQLITE_LIMIT_VARIABLE_NUMBER }, { "SQLITE_LIMIT_TRIGGER_DEPTH", SQLITE_LIMIT_TRIGGER_DEPTH }, + { "SQLITE_LIMIT_WORKER_THREADS", SQLITE_LIMIT_WORKER_THREADS }, /* Out of range test cases */ { "SQLITE_LIMIT_TOOSMALL", -1, }, - { "SQLITE_LIMIT_TOOBIG", SQLITE_LIMIT_TRIGGER_DEPTH+1 }, + { "SQLITE_LIMIT_TOOBIG", SQLITE_LIMIT_WORKER_THREADS+1 }, }; int i, id; int val; @@ -5711,10 +5688,11 @@ static int test_wal_checkpoint_v2( int nCkpt = -555; Tcl_Obj *pRet; - const char * aMode[] = { "passive", "full", "restart", 0 }; + const char * aMode[] = { "passive", "full", "restart", "truncate", 0 }; assert( SQLITE_CHECKPOINT_PASSIVE==0 ); assert( SQLITE_CHECKPOINT_FULL==1 ); assert( SQLITE_CHECKPOINT_RESTART==2 ); + assert( SQLITE_CHECKPOINT_TRUNCATE==3 ); if( objc!=3 && objc!=4 ){ Tcl_WrongNumArgs(interp, 1, objv, "DB MODE ?NAME?"); @@ -5724,15 +5702,18 @@ static int test_wal_checkpoint_v2( if( objc==4 ){ zDb = Tcl_GetString(objv[3]); } - if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) - || Tcl_GetIndexFromObj(interp, objv[2], aMode, "mode", 0, &eMode) - ){ + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) || ( + TCL_OK!=Tcl_GetIntFromObj(0, objv[2], &eMode) + && TCL_OK!=Tcl_GetIndexFromObj(interp, objv[2], aMode, "mode", 0, &eMode) + )){ return TCL_ERROR; } rc = sqlite3_wal_checkpoint_v2(db, zDb, eMode, &nLog, &nCkpt); if( rc!=SQLITE_OK && rc!=SQLITE_BUSY ){ - Tcl_SetResult(interp, (char *)sqlite3_errmsg(db), TCL_VOLATILE); + const char *zErrCode = sqlite3ErrName(rc); + Tcl_ResetResult(interp); + Tcl_AppendResult(interp, zErrCode, " - ", (char *)sqlite3_errmsg(db), 0); return TCL_ERROR; } @@ -5745,6 +5726,43 @@ static int test_wal_checkpoint_v2( return TCL_OK; } +/* +** tclcmd: sqlite3_wal_autocheckpoint db VALUE +*/ +static int test_wal_autocheckpoint( + ClientData clientData, /* Unused */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + sqlite3 *db; + int rc; + int iVal; + + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB VALUE"); + return TCL_ERROR; + } + + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) + || Tcl_GetIntFromObj(0, objv[2], &iVal) + ){ + return TCL_ERROR; + } + + rc = sqlite3_wal_autocheckpoint(db, iVal); + Tcl_ResetResult(interp); + if( rc!=SQLITE_OK ){ + const char *zErrCode = sqlite3ErrName(rc); + Tcl_SetObjResult(interp, Tcl_NewStringObj(zErrCode, -1)); + return TCL_ERROR; + } + + return TCL_OK; +} + + /* ** tclcmd: test_sqlite3_log ?SCRIPT? */ @@ -5895,6 +5913,7 @@ static int test_test_control( int i; } aVerb[] = { { "SQLITE_TESTCTRL_LOCALTIME_FAULT", SQLITE_TESTCTRL_LOCALTIME_FAULT }, + { "SQLITE_TESTCTRL_SORTER_MMAP", SQLITE_TESTCTRL_SORTER_MMAP }, }; int iVerb; int iFlag; @@ -5922,6 +5941,19 @@ static int test_test_control( sqlite3_test_control(SQLITE_TESTCTRL_LOCALTIME_FAULT, val); break; } + + case SQLITE_TESTCTRL_SORTER_MMAP: { + int val; + sqlite3 *db; + if( objc!=4 ){ + Tcl_WrongNumArgs(interp, 2, objv, "DB LIMIT"); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[2]), &db) ) return TCL_ERROR; + if( Tcl_GetIntFromObj(interp, objv[3], &val) ) return TCL_ERROR; + sqlite3_test_control(SQLITE_TESTCTRL_SORTER_MMAP, db, val); + break; + } } Tcl_ResetResult(interp); @@ -6238,7 +6270,8 @@ static int optimization_control( { "transitive", SQLITE_Transitive }, { "subquery-coroutine", SQLITE_SubqCoroutine }, { "omit-noop-join", SQLITE_OmitNoopJoin }, - { "stat3", SQLITE_Stat3 }, + { "stat3", SQLITE_Stat34 }, + { "stat4", SQLITE_Stat34 }, }; if( objc!=4 ){ @@ -6281,6 +6314,7 @@ static int tclLoadStaticExtensionCmd( ){ extern int sqlite3_amatch_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_closure_init(sqlite3*,char**,const sqlite3_api_routines*); + extern int sqlite3_eval_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_fileio_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_fuzzer_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_ieee_init(sqlite3*,char**,const sqlite3_api_routines*); @@ -6296,6 +6330,7 @@ static int tclLoadStaticExtensionCmd( } aExtension[] = { { "amatch", sqlite3_amatch_init }, { "closure", sqlite3_closure_init }, + { "eval", sqlite3_eval_init }, { "fileio", sqlite3_fileio_init }, { "fuzzer", sqlite3_fuzzer_init }, { "ieee754", sqlite3_ieee_init }, @@ -6335,6 +6370,239 @@ static int tclLoadStaticExtensionCmd( return TCL_OK; } +/* +** sorter_test_fakeheap BOOL +** +*/ +static int sorter_test_fakeheap( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + int bArg; + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "BOOL"); + return TCL_ERROR; + } + + if( Tcl_GetBooleanFromObj(interp, objv[1], &bArg) ){ + return TCL_ERROR; + } + + if( bArg ){ + if( sqlite3GlobalConfig.pHeap==0 ){ + sqlite3GlobalConfig.pHeap = SQLITE_INT_TO_PTR(-1); + } + }else{ + if( sqlite3GlobalConfig.pHeap==SQLITE_INT_TO_PTR(-1) ){ + sqlite3GlobalConfig.pHeap = 0; + } + } + + Tcl_ResetResult(interp); + return TCL_OK; +} + +/* +** sorter_test_sort4_helper DB SQL1 NSTEP SQL2 +** +** Compile SQL statement $SQL1 and step it $NSTEP times. For each row, +** check that the leftmost and rightmost columns returned are both integers, +** and that both contain the same value. +** +** Then execute statement $SQL2. Check that the statement returns the same +** set of integers in the same order as in the previous step (using $SQL1). +*/ +static int sorter_test_sort4_helper( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + const char *zSql1; + const char *zSql2; + int nStep; + int iStep; + int iCksum1 = 0; + int iCksum2 = 0; + int rc; + int iB; + sqlite3 *db; + sqlite3_stmt *pStmt; + + if( objc!=5 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB SQL1 NSTEP SQL2"); + return TCL_ERROR; + } + + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; + zSql1 = Tcl_GetString(objv[2]); + if( Tcl_GetIntFromObj(interp, objv[3], &nStep) ) return TCL_ERROR; + zSql2 = Tcl_GetString(objv[4]); + + rc = sqlite3_prepare_v2(db, zSql1, -1, &pStmt, 0); + if( rc!=SQLITE_OK ) goto sql_error; + + iB = sqlite3_column_count(pStmt)-1; + for(iStep=0; iStep +#include +#include + +/* These functions are implemented in main.c. */ +extern const char *sqlite3ErrName(int); + +/* From test1.c: */ +extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb); +extern void *sqlite3TestTextToPtr(const char *z); + +/* +** Return a pointer to a buffer containing a text representation of the +** pointer passed as the only argument. The original pointer may be extracted +** from the text using sqlite3TestTextToPtr(). +*/ +static char *ptrToText(void *p){ + static char buf[100]; + sqlite3_snprintf(sizeof(buf)-1, buf, "%p", p); + return buf; +} + +/* +** Attempt to extract a blob handle (type sqlite3_blob*) from the Tcl +** object passed as the second argument. If successful, set *ppBlob to +** point to the blob handle and return TCL_OK. Otherwise, store an error +** message in the tcl interpreter and return TCL_ERROR. The final value +** of *ppBlob is undefined in this case. +** +** If the object contains a string that begins with "incrblob_", then it +** is assumed to be the name of a Tcl channel opened using the [db incrblob] +** command (see tclsqlite.c). Otherwise, it is assumed to be a pointer +** encoded using the ptrToText() routine or similar. +*/ +static int blobHandleFromObj( + Tcl_Interp *interp, + Tcl_Obj *pObj, + sqlite3_blob **ppBlob +){ + char *z; + int n; + + z = Tcl_GetStringFromObj(pObj, &n); + if( n==0 ){ + *ppBlob = 0; + }else if( n>9 && 0==memcmp("incrblob_", z, 9) ){ + int notUsed; + Tcl_Channel channel; + ClientData instanceData; + + channel = Tcl_GetChannel(interp, z, ¬Used); + if( !channel ) return TCL_ERROR; + + Tcl_Flush(channel); + Tcl_Seek(channel, 0, SEEK_SET); + + instanceData = Tcl_GetChannelInstanceData(channel); + *ppBlob = *((sqlite3_blob **)instanceData); + }else{ + *ppBlob = (sqlite3_blob*)sqlite3TestTextToPtr(z); + } + + return TCL_OK; +} + +/* +** Like Tcl_GetString(), except that if the string is 0 bytes in size, a +** NULL Pointer is returned. +*/ +static char *blobStringFromObj(Tcl_Obj *pObj){ + int n; + char *z; + z = Tcl_GetStringFromObj(pObj, &n); + return (n ? z : 0); +} + +/* +** sqlite3_blob_open DB DATABASE TABLE COLUMN ROWID FLAGS VARNAME +** +** Tcl test harness for the sqlite3_blob_open() function. +*/ +static int test_blob_open( + ClientData clientData, /* Not used */ + Tcl_Interp *interp, /* Calling TCL interpreter */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + sqlite3 *db; + const char *zDb; + const char *zTable; + const char *zColumn; + sqlite_int64 iRowid; + int flags; + const char *zVarname; + int nVarname; + + sqlite3_blob *pBlob = (sqlite3_blob*)0xFFFFFFFF; + int rc; + + if( objc!=8 ){ + const char *zUsage = "DB DATABASE TABLE COLUMN ROWID FLAGS VARNAME"; + Tcl_WrongNumArgs(interp, 1, objv, zUsage); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; + zDb = Tcl_GetString(objv[2]); + zTable = blobStringFromObj(objv[3]); + zColumn = Tcl_GetString(objv[4]); + if( Tcl_GetWideIntFromObj(interp, objv[5], &iRowid) ) return TCL_ERROR; + if( Tcl_GetIntFromObj(interp, objv[6], &flags) ) return TCL_ERROR; + zVarname = Tcl_GetStringFromObj(objv[7], &nVarname); + + if( nVarname>0 ){ + rc = sqlite3_blob_open(db, zDb, zTable, zColumn, iRowid, flags, &pBlob); + Tcl_SetVar(interp, zVarname, ptrToText(pBlob), 0); + }else{ + rc = sqlite3_blob_open(db, zDb, zTable, zColumn, iRowid, flags, 0); + } + + if( rc==SQLITE_OK ){ + Tcl_ResetResult(interp); + }else{ + Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE); + return TCL_ERROR; + } + return TCL_OK; +} + + +/* +** sqlite3_blob_close HANDLE +*/ +static int test_blob_close( + ClientData clientData, /* Not used */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + sqlite3_blob *pBlob; + int rc; + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "HANDLE"); + return TCL_ERROR; + } + + if( blobHandleFromObj(interp, objv[1], &pBlob) ) return TCL_ERROR; + rc = sqlite3_blob_close(pBlob); + + if( rc ){ + Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE); + }else{ + Tcl_ResetResult(interp); + } + return TCL_OK; +} + +/* +** sqlite3_blob_bytes HANDLE +*/ +static int test_blob_bytes( + ClientData clientData, /* Not used */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + sqlite3_blob *pBlob; + int nByte; + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "HANDLE"); + return TCL_ERROR; + } + + if( blobHandleFromObj(interp, objv[1], &pBlob) ) return TCL_ERROR; + nByte = sqlite3_blob_bytes(pBlob); + Tcl_SetObjResult(interp, Tcl_NewIntObj(nByte)); + + return TCL_OK; +} + +/* +** sqlite3_blob_read CHANNEL OFFSET N +** +** This command is used to test the sqlite3_blob_read() in ways that +** the Tcl channel interface does not. The first argument should +** be the name of a valid channel created by the [incrblob] method +** of a database handle. This function calls sqlite3_blob_read() +** to read N bytes from offset OFFSET from the underlying SQLite +** blob handle. +** +** On success, a byte-array object containing the read data is +** returned. On failure, the interpreter result is set to the +** text representation of the returned error code (i.e. "SQLITE_NOMEM") +** and a Tcl exception is thrown. +*/ +static int test_blob_read( + ClientData clientData, /* Not used */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + sqlite3_blob *pBlob; + int nByte; + int iOffset; + unsigned char *zBuf = 0; + int rc; + + if( objc!=4 ){ + Tcl_WrongNumArgs(interp, 1, objv, "CHANNEL OFFSET N"); + return TCL_ERROR; + } + + if( blobHandleFromObj(interp, objv[1], &pBlob) ) return TCL_ERROR; + if( TCL_OK!=Tcl_GetIntFromObj(interp, objv[2], &iOffset) + || TCL_OK!=Tcl_GetIntFromObj(interp, objv[3], &nByte) + ){ + return TCL_ERROR; + } + + if( nByte>0 ){ + zBuf = (unsigned char *)Tcl_Alloc(nByte); + } + rc = sqlite3_blob_read(pBlob, zBuf, nByte, iOffset); + if( rc==SQLITE_OK ){ + Tcl_SetObjResult(interp, Tcl_NewByteArrayObj(zBuf, nByte)); + }else{ + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); + } + Tcl_Free((char *)zBuf); + + return (rc==SQLITE_OK ? TCL_OK : TCL_ERROR); +} + +/* +** sqlite3_blob_write HANDLE OFFSET DATA ?NDATA? +** +** This command is used to test the sqlite3_blob_write() in ways that +** the Tcl channel interface does not. The first argument should +** be the name of a valid channel created by the [incrblob] method +** of a database handle. This function calls sqlite3_blob_write() +** to write the DATA byte-array to the underlying SQLite blob handle. +** at offset OFFSET. +** +** On success, an empty string is returned. On failure, the interpreter +** result is set to the text representation of the returned error code +** (i.e. "SQLITE_NOMEM") and a Tcl exception is thrown. +*/ +static int test_blob_write( + ClientData clientData, /* Not used */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + sqlite3_blob *pBlob; + int iOffset; + int rc; + + unsigned char *zBuf; + int nBuf; + + if( objc!=4 && objc!=5 ){ + Tcl_WrongNumArgs(interp, 1, objv, "HANDLE OFFSET DATA ?NDATA?"); + return TCL_ERROR; + } + + if( blobHandleFromObj(interp, objv[1], &pBlob) ) return TCL_ERROR; + if( TCL_OK!=Tcl_GetIntFromObj(interp, objv[2], &iOffset) ){ + return TCL_ERROR; + } + + zBuf = Tcl_GetByteArrayFromObj(objv[3], &nBuf); + if( objc==5 && Tcl_GetIntFromObj(interp, objv[4], &nBuf) ){ + return TCL_ERROR; + } + rc = sqlite3_blob_write(pBlob, zBuf, nBuf, iOffset); + if( rc!=SQLITE_OK ){ + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); + } + + return (rc==SQLITE_OK ? TCL_OK : TCL_ERROR); +} + + +/* +** Register commands with the TCL interpreter. +*/ +int Sqlitetest_blob_Init(Tcl_Interp *interp){ + static struct { + char *zName; + Tcl_ObjCmdProc *xProc; + } aObjCmd[] = { + { "sqlite3_blob_open", test_blob_open }, + { "sqlite3_blob_close", test_blob_close }, + { "sqlite3_blob_bytes", test_blob_bytes }, + { "sqlite3_blob_read", test_blob_read }, + { "sqlite3_blob_write", test_blob_write }, + }; + int i; + for(i=0; ibTruncate = 1; + }else + if( sqlite3_stricmp(aFcntl[2], "off")==0 + || sqlite3_stricmp(aFcntl[2], "0")==0 ){ + pGroup->bTruncate = 0; + } + } + aFcntl[0] = sqlite3_mprintf(pGroup->bTruncate ? "on" : "off"); + rc = SQLITE_OK; + break; + } + /* If the multiplexor does not handle the pragma, pass it through + ** into the default case. */ + } default: pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL, 0); if( pSubOpen ){ @@ -1176,14 +1196,20 @@ int sqlite3_multiplex_initialize(const char *zOrigVfsName, int makeDefault){ ** THIS ROUTINE IS NOT THREADSAFE. Call this routine exactly once while ** shutting down in order to free all remaining multiplex groups. */ -int sqlite3_multiplex_shutdown(void){ +int sqlite3_multiplex_shutdown(int eForce){ + int rc = SQLITE_OK; if( gMultiplex.isInitialized==0 ) return SQLITE_MISUSE; - if( gMultiplex.pGroups ) return SQLITE_MISUSE; + if( gMultiplex.pGroups ){ + sqlite3_log(SQLITE_MISUSE, "sqlite3_multiplex_shutdown() called " + "while database connections are still open"); + if( !eForce ) return SQLITE_MISUSE; + rc = SQLITE_MISUSE; + } gMultiplex.isInitialized = 0; sqlite3_mutex_free(gMultiplex.pMutex); sqlite3_vfs_unregister(&gMultiplex.sThisVfs); memset(&gMultiplex, 0, sizeof(gMultiplex)); - return SQLITE_OK; + return rc; } /***************************** Test Code ***********************************/ @@ -1236,13 +1262,16 @@ static int test_multiplex_shutdown( UNUSED_PARAMETER(clientData); - if( objc!=1 ){ - Tcl_WrongNumArgs(interp, 1, objv, ""); + if( objc==2 && strcmp(Tcl_GetString(objv[1]),"-force")!=0 ){ + objc = 3; + } + if( (objc!=1 && objc!=2) ){ + Tcl_WrongNumArgs(interp, 1, objv, "?-force?"); return TCL_ERROR; } /* Call sqlite3_multiplex_shutdown() */ - rc = sqlite3_multiplex_shutdown(); + rc = sqlite3_multiplex_shutdown(objc==2); Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); return TCL_OK; diff --git a/src/test_multiplex.h b/src/test_multiplex.h index 9fe2f0f2b2..d973e4af27 100644 --- a/src/test_multiplex.h +++ b/src/test_multiplex.h @@ -90,7 +90,7 @@ extern int sqlite3_multiplex_initialize(const char *zOrigVfsName, int makeDefaul ** THIS ROUTINE IS NOT THREADSAFE. Call this routine exactly once while ** shutting down in order to free all remaining multiplex groups. */ -extern int sqlite3_multiplex_shutdown(void); +extern int sqlite3_multiplex_shutdown(int eForce); #ifdef __cplusplus } /* End of the 'extern "C"' block */ diff --git a/src/test_schema.c b/src/test_schema.c index 00a9f4dd90..4ee18193b0 100644 --- a/src/test_schema.c +++ b/src/test_schema.c @@ -189,7 +189,7 @@ static int schemaNext(sqlite3_vtab_cursor *cur){ /* Set zSql to the SQL to pull the list of tables from the ** sqlite_master (or sqlite_temp_master) table of the database - ** identfied by the row pointed to by the SQL statement pCur->pDbList + ** identified by the row pointed to by the SQL statement pCur->pDbList ** (iterating through a "PRAGMA database_list;" statement). */ if( sqlite3_column_int(pCur->pDbList, 0)==1 ){ diff --git a/src/threads.c b/src/threads.c new file mode 100644 index 0000000000..18d7320a12 --- /dev/null +++ b/src/threads.c @@ -0,0 +1,262 @@ +/* +** 2012 July 21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file presents a simple cross-platform threading interface for +** use internally by SQLite. +** +** A "thread" can be created using sqlite3ThreadCreate(). This thread +** runs independently of its creator until it is joined using +** sqlite3ThreadJoin(), at which point it terminates. +** +** Threads do not have to be real. It could be that the work of the +** "thread" is done by the main thread at either the sqlite3ThreadCreate() +** or sqlite3ThreadJoin() call. This is, in fact, what happens in +** single threaded systems. Nothing in SQLite requires multiple threads. +** This interface exists so that applications that want to take advantage +** of multiple cores can do so, while also allowing applications to stay +** single-threaded if desired. +*/ +#include "sqliteInt.h" + +#if SQLITE_MAX_WORKER_THREADS>0 + +/********************************* Unix Pthreads ****************************/ +#if SQLITE_OS_UNIX && defined(SQLITE_MUTEX_PTHREADS) && SQLITE_THREADSAFE>0 + +#define SQLITE_THREADS_IMPLEMENTED 1 /* Prevent the single-thread code below */ +#include + +/* A running thread */ +struct SQLiteThread { + pthread_t tid; /* Thread ID */ + int done; /* Set to true when thread finishes */ + void *pOut; /* Result returned by the thread */ + void *(*xTask)(void*); /* The thread routine */ + void *pIn; /* Argument to the thread */ +}; + +/* Create a new thread */ +int sqlite3ThreadCreate( + SQLiteThread **ppThread, /* OUT: Write the thread object here */ + void *(*xTask)(void*), /* Routine to run in a separate thread */ + void *pIn /* Argument passed into xTask() */ +){ + SQLiteThread *p; + int rc; + + assert( ppThread!=0 ); + assert( xTask!=0 ); + /* This routine is never used in single-threaded mode */ + assert( sqlite3GlobalConfig.bCoreMutex!=0 ); + + *ppThread = 0; + p = sqlite3Malloc(sizeof(*p)); + if( p==0 ) return SQLITE_NOMEM; + memset(p, 0, sizeof(*p)); + p->xTask = xTask; + p->pIn = pIn; + if( sqlite3FaultSim(200) ){ + rc = 1; + }else{ + rc = pthread_create(&p->tid, 0, xTask, pIn); + } + if( rc ){ + p->done = 1; + p->pOut = xTask(pIn); + } + *ppThread = p; + return SQLITE_OK; +} + +/* Get the results of the thread */ +int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ + int rc; + + assert( ppOut!=0 ); + if( NEVER(p==0) ) return SQLITE_NOMEM; + if( p->done ){ + *ppOut = p->pOut; + rc = SQLITE_OK; + }else{ + rc = pthread_join(p->tid, ppOut) ? SQLITE_ERROR : SQLITE_OK; + } + sqlite3_free(p); + return rc; +} + +#endif /* SQLITE_OS_UNIX && defined(SQLITE_MUTEX_PTHREADS) */ +/******************************** End Unix Pthreads *************************/ + + +/********************************* Win32 Threads ****************************/ +#if SQLITE_OS_WIN && !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_THREADSAFE>0 + +#define SQLITE_THREADS_IMPLEMENTED 1 /* Prevent the single-thread code below */ +#include + +/* A running thread */ +struct SQLiteThread { + void *tid; /* The thread handle */ + unsigned id; /* The thread identifier */ + void *(*xTask)(void*); /* The routine to run as a thread */ + void *pIn; /* Argument to xTask */ + void *pResult; /* Result of xTask */ +}; + +/* Thread procedure Win32 compatibility shim */ +static unsigned __stdcall sqlite3ThreadProc( + void *pArg /* IN: Pointer to the SQLiteThread structure */ +){ + SQLiteThread *p = (SQLiteThread *)pArg; + + assert( p!=0 ); +#if 0 + /* + ** This assert appears to trigger spuriously on certain + ** versions of Windows, possibly due to _beginthreadex() + ** and/or CreateThread() not fully setting their thread + ** ID parameter before starting the thread. + */ + assert( p->id==GetCurrentThreadId() ); +#endif + assert( p->xTask!=0 ); + p->pResult = p->xTask(p->pIn); + + _endthreadex(0); + return 0; /* NOT REACHED */ +} + +/* Create a new thread */ +int sqlite3ThreadCreate( + SQLiteThread **ppThread, /* OUT: Write the thread object here */ + void *(*xTask)(void*), /* Routine to run in a separate thread */ + void *pIn /* Argument passed into xTask() */ +){ + SQLiteThread *p; + + assert( ppThread!=0 ); + assert( xTask!=0 ); + *ppThread = 0; + p = sqlite3Malloc(sizeof(*p)); + if( p==0 ) return SQLITE_NOMEM; + if( sqlite3GlobalConfig.bCoreMutex==0 ){ + memset(p, 0, sizeof(*p)); + }else{ + p->xTask = xTask; + p->pIn = pIn; + p->tid = (void*)_beginthreadex(0, 0, sqlite3ThreadProc, p, 0, &p->id); + if( p->tid==0 ){ + memset(p, 0, sizeof(*p)); + } + } + if( p->xTask==0 ){ + p->id = GetCurrentThreadId(); + p->pResult = xTask(pIn); + } + *ppThread = p; + return SQLITE_OK; +} + +DWORD sqlite3Win32Wait(HANDLE hObject); /* os_win.c */ + +/* Get the results of the thread */ +int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ + DWORD rc; + BOOL bRc; + + assert( ppOut!=0 ); + if( NEVER(p==0) ) return SQLITE_NOMEM; + if( p->xTask==0 ){ + assert( p->id==GetCurrentThreadId() ); + rc = WAIT_OBJECT_0; + assert( p->tid==0 ); + }else{ + assert( p->id!=0 && p->id!=GetCurrentThreadId() ); + rc = sqlite3Win32Wait((HANDLE)p->tid); + assert( rc!=WAIT_IO_COMPLETION ); + bRc = CloseHandle((HANDLE)p->tid); + assert( bRc ); + } + if( rc==WAIT_OBJECT_0 ) *ppOut = p->pResult; + sqlite3_free(p); + return (rc==WAIT_OBJECT_0) ? SQLITE_OK : SQLITE_ERROR; +} + +#endif /* SQLITE_OS_WIN && !SQLITE_OS_WINCE && !SQLITE_OS_WINRT */ +/******************************** End Win32 Threads *************************/ + + +/********************************* Single-Threaded **************************/ +#ifndef SQLITE_THREADS_IMPLEMENTED +/* +** This implementation does not actually create a new thread. It does the +** work of the thread in the main thread, when either the thread is created +** or when it is joined +*/ + +/* A running thread */ +struct SQLiteThread { + void *(*xTask)(void*); /* The routine to run as a thread */ + void *pIn; /* Argument to xTask */ + void *pResult; /* Result of xTask */ +}; + +/* Create a new thread */ +int sqlite3ThreadCreate( + SQLiteThread **ppThread, /* OUT: Write the thread object here */ + void *(*xTask)(void*), /* Routine to run in a separate thread */ + void *pIn /* Argument passed into xTask() */ +){ + SQLiteThread *p; + + assert( ppThread!=0 ); + assert( xTask!=0 ); + *ppThread = 0; + p = sqlite3Malloc(sizeof(*p)); + if( p==0 ) return SQLITE_NOMEM; + if( (SQLITE_PTR_TO_INT(p)/17)&1 ){ + p->xTask = xTask; + p->pIn = pIn; + }else{ + p->xTask = 0; + p->pResult = xTask(pIn); + } + *ppThread = p; + return SQLITE_OK; +} + +/* Get the results of the thread */ +int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ + + assert( ppOut!=0 ); + if( NEVER(p==0) ) return SQLITE_NOMEM; + if( p->xTask ){ + *ppOut = p->xTask(p->pIn); + }else{ + *ppOut = p->pResult; + } + sqlite3_free(p); + +#if defined(SQLITE_TEST) + { + void *pTstAlloc = sqlite3Malloc(10); + if (!pTstAlloc) return SQLITE_NOMEM; + sqlite3_free(pTstAlloc); + } +#endif + + return SQLITE_OK; +} + +#endif /* !defined(SQLITE_THREADS_IMPLEMENTED) */ +/****************************** End Single-Threaded *************************/ +#endif /* SQLITE_MAX_WORKER_THREADS>0 */ diff --git a/src/tokenize.c b/src/tokenize.c index 87553e25b0..f0360eef61 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -77,7 +77,7 @@ const unsigned char ebcdicToAscii[] = { ** end result. ** ** Ticket #1066. the SQL standard does not allow '$' in the -** middle of identfiers. But many SQL implementations do. +** middle of identifiers. But many SQL implementations do. ** SQLite will allow '$' in identifiers for compatibility. ** But the feature is undocumented. */ @@ -102,6 +102,7 @@ const char sqlite3IsEbcdicIdChar[] = { }; #define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40])) #endif +int sqlite3IsIdChar(u8 c){ return IdChar(c); } /* @@ -270,6 +271,12 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){ testcase( z[0]=='6' ); testcase( z[0]=='7' ); testcase( z[0]=='8' ); testcase( z[0]=='9' ); *tokenType = TK_INTEGER; +#ifndef SQLITE_OMIT_HEX_INTEGER + if( z[0]=='0' && (z[1]=='x' || z[1]=='X') && sqlite3Isxdigit(z[2]) ){ + for(i=3; sqlite3Isxdigit(z[i]); i++){} + return i; + } +#endif for(i=0; sqlite3Isdigit(z[i]); i++){} #ifndef SQLITE_OMIT_FLOATING_POINT if( z[i]=='.' ){ @@ -384,6 +391,9 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ int mxSqlLen; /* Max length of an SQL string */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( zSql==0 || pzErrMsg==0 ) return SQLITE_MISUSE_BKPT; +#endif mxSqlLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH]; if( db->nVdbeActive==0 ){ db->u1.isInterrupted = 0; @@ -392,7 +402,7 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ pParse->zTail = zSql; i = 0; assert( pzErrMsg!=0 ); - pEngine = sqlite3ParserAlloc((void*(*)(size_t))sqlite3Malloc); + pEngine = sqlite3ParserAlloc(sqlite3Malloc); if( pEngine==0 ){ db->mallocFailed = 1; return SQLITE_NOMEM; diff --git a/src/trigger.c b/src/trigger.c index 01f7b21f74..d2e7b5a1e6 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -127,7 +127,7 @@ void sqlite3BeginTrigger( ** ^^^^^^^^ ** ** To maintain backwards compatibility, ignore the database - ** name on pTableName if we are reparsing our of SQLITE_MASTER. + ** name on pTableName if we are reparsing out of SQLITE_MASTER. */ if( db->init.busy && iDb!=1 ){ sqlite3DbFree(db, pTableName->a[0].zDatabase); @@ -180,8 +180,7 @@ void sqlite3BeginTrigger( goto trigger_cleanup; } assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - if( sqlite3HashFind(&(db->aDb[iDb].pSchema->trigHash), - zName, sqlite3Strlen30(zName)) ){ + if( sqlite3HashFind(&(db->aDb[iDb].pSchema->trigHash),zName) ){ if( !noErr ){ sqlite3ErrorMsg(pParse, "trigger %T already exists", pName); }else{ @@ -324,13 +323,12 @@ void sqlite3FinishTrigger( Trigger *pLink = pTrig; Hash *pHash = &db->aDb[iDb].pSchema->trigHash; assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - pTrig = sqlite3HashInsert(pHash, zName, sqlite3Strlen30(zName), pTrig); + pTrig = sqlite3HashInsert(pHash, zName, pTrig); if( pTrig ){ db->mallocFailed = 1; }else if( pLink->pSchema==pLink->pTabSchema ){ Table *pTab; - int n = sqlite3Strlen30(pLink->table); - pTab = sqlite3HashFind(&pLink->pTabSchema->tblHash, pLink->table, n); + pTab = sqlite3HashFind(&pLink->pTabSchema->tblHash, pLink->table); assert( pTab!=0 ); pLink->pNext = pTab->pTrigger; pTab->pTrigger = pLink; @@ -489,7 +487,6 @@ void sqlite3DropTrigger(Parse *pParse, SrcList *pName, int noErr){ int i; const char *zDb; const char *zName; - int nName; sqlite3 *db = pParse->db; if( db->mallocFailed ) goto drop_trigger_cleanup; @@ -500,13 +497,12 @@ void sqlite3DropTrigger(Parse *pParse, SrcList *pName, int noErr){ assert( pName->nSrc==1 ); zDb = pName->a[0].zDatabase; zName = pName->a[0].zName; - nName = sqlite3Strlen30(zName); assert( zDb!=0 || sqlite3BtreeHoldsAllMutexes(db) ); for(i=OMIT_TEMPDB; inDb; i++){ int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ if( zDb && sqlite3StrICmp(db->aDb[j].zName, zDb) ) continue; assert( sqlite3SchemaMutexHeld(db, j, 0) ); - pTrigger = sqlite3HashFind(&(db->aDb[j].pSchema->trigHash), zName, nName); + pTrigger = sqlite3HashFind(&(db->aDb[j].pSchema->trigHash), zName); if( pTrigger ) break; } if( !pTrigger ){ @@ -529,8 +525,7 @@ drop_trigger_cleanup: ** is set on. */ static Table *tableOfTrigger(Trigger *pTrigger){ - int n = sqlite3Strlen30(pTrigger->table); - return sqlite3HashFind(&pTrigger->pTabSchema->tblHash, pTrigger->table, n); + return sqlite3HashFind(&pTrigger->pTabSchema->tblHash, pTrigger->table); } @@ -602,7 +597,7 @@ void sqlite3UnlinkAndDeleteTrigger(sqlite3 *db, int iDb, const char *zName){ assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); pHash = &(db->aDb[iDb].pSchema->trigHash); - pTrigger = sqlite3HashInsert(pHash, zName, sqlite3Strlen30(zName), 0); + pTrigger = sqlite3HashInsert(pHash, zName, 0); if( ALWAYS(pTrigger) ){ if( pTrigger->pSchema==pTrigger->pTabSchema ){ Table *pTab = tableOfTrigger(pTrigger); diff --git a/src/update.c b/src/update.c index 3e04e00316..3af4017f1b 100644 --- a/src/update.c +++ b/src/update.c @@ -327,7 +327,7 @@ void sqlite3Update( } /* If we are trying to update a view, realize that view into - ** a ephemeral table. + ** an ephemeral table. */ #if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) if( isView ){ @@ -431,14 +431,15 @@ void sqlite3Update( /* Top of the update loop */ if( okOnePass ){ - if( aToOpen[iDataCur-iBaseCur] ){ - assert( pPk!=0 ); + if( aToOpen[iDataCur-iBaseCur] && !isView ){ + assert( pPk ); sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelBreak, regKey, nKey); VdbeCoverageNeverTaken(v); } labelContinue = labelBreak; sqlite3VdbeAddOp2(v, OP_IsNull, pPk ? regKey : regOldRowid, labelBreak); - VdbeCoverage(v); + VdbeCoverageIf(v, pPk==0); + VdbeCoverageIf(v, pPk!=0); }else if( pPk ){ labelContinue = sqlite3VdbeMakeLabel(v); sqlite3VdbeAddOp2(v, OP_Rewind, iEph, labelBreak); VdbeCoverage(v); @@ -487,7 +488,7 @@ void sqlite3Update( } /* Populate the array of registers beginning at regNew with the new - ** row data. This array is used to check constaints, create the new + ** row data. This array is used to check constants, create the new ** table and index records, and as the values for any new.* references ** made by triggers. ** @@ -667,7 +668,7 @@ update_cleanup: return; } /* Make sure "isView" and other macros defined above are undefined. Otherwise -** thely may interfere with compilation of other functions in this file +** they may interfere with compilation of other functions in this file ** (or in another file, if this file becomes part of the amalgamation). */ #ifdef isView #undef isView @@ -680,7 +681,7 @@ update_cleanup: /* ** Generate code for an UPDATE of a virtual table. ** -** The strategy is that we create an ephemerial table that contains +** The strategy is that we create an ephemeral table that contains ** for each row to be changed: ** ** (A) The original rowid of that row. @@ -688,7 +689,7 @@ update_cleanup: ** (C) The content of every column in the row. ** ** Then we loop over this ephemeral table and for each row in -** the ephermeral table call VUpdate. +** the ephemeral table call VUpdate. ** ** When finished, drop the ephemeral table. ** diff --git a/src/utf.c b/src/utf.c index 97898746a2..25f4dadf0c 100644 --- a/src/utf.c +++ b/src/utf.c @@ -199,7 +199,7 @@ u32 sqlite3Utf8Read( ** desiredEnc. It is an error if the string is already of the desired ** encoding, or if *pMem does not contain a string value. */ -int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){ +SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){ int len; /* Maximum length of output string in bytes */ unsigned char *zOut; /* Output buffer */ unsigned char *zIn; /* Input iterator */ @@ -314,12 +314,13 @@ int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){ *z = 0; assert( (pMem->n+(desiredEnc==SQLITE_UTF8?1:2))<=len ); + c = pMem->flags; sqlite3VdbeMemRelease(pMem); - pMem->flags &= ~(MEM_Static|MEM_Dyn|MEM_Ephem); + pMem->flags = MEM_Str|MEM_Term|(c&MEM_AffMask); pMem->enc = desiredEnc; - pMem->flags |= (MEM_Term); pMem->z = (char*)zOut; pMem->zMalloc = pMem->z; + pMem->szMalloc = sqlite3DbMallocSize(pMem->db, pMem->z); translate_out: #if defined(TRANSLATE_TRACE) && defined(SQLITE_DEBUG) diff --git a/src/util.c b/src/util.c index 4fe07a862a..ab409fa256 100644 --- a/src/util.c +++ b/src/util.c @@ -111,6 +111,15 @@ int sqlite3Strlen30(const char *z){ return 0x3fffffff & (int)(z2 - z); } +/* +** Set the current error code to err_code and clear any prior error message. +*/ +void sqlite3Error(sqlite3 *db, int err_code){ + assert( db!=0 ); + db->errCode = err_code; + if( db->pErr ) sqlite3ValueSetNull(db->pErr); +} + /* ** Set the most recent error code and error string for the sqlite ** handle "db". The error code is set to "err_code". @@ -132,18 +141,18 @@ int sqlite3Strlen30(const char *z){ ** should be called with err_code set to SQLITE_OK and zFormat set ** to NULL. */ -void sqlite3Error(sqlite3 *db, int err_code, const char *zFormat, ...){ +void sqlite3ErrorWithMsg(sqlite3 *db, int err_code, const char *zFormat, ...){ assert( db!=0 ); db->errCode = err_code; - if( zFormat && (db->pErr || (db->pErr = sqlite3ValueNew(db))!=0) ){ + if( zFormat==0 ){ + sqlite3Error(db, err_code); + }else if( db->pErr || (db->pErr = sqlite3ValueNew(db))!=0 ){ char *z; va_list ap; va_start(ap, zFormat); z = sqlite3VMPrintf(db, zFormat, ap); va_end(ap); sqlite3ValueSetStr(db->pErr, -1, z, SQLITE_UTF8, SQLITE_DYNAMIC); - }else if( db->pErr ){ - sqlite3ValueSetNull(db->pErr); } } @@ -157,12 +166,12 @@ void sqlite3Error(sqlite3 *db, int err_code, const char *zFormat, ...){ ** %T Insert a token ** %S Insert the first element of a SrcList ** -** This function should be used to report any error that occurs whilst +** This function should be used to report any error that occurs while ** compiling an SQL statement (i.e. within sqlite3_prepare()). The ** last thing the sqlite3_prepare() function does is copy the error ** stored by this function into the database handle using sqlite3Error(). -** Function sqlite3Error() should be used during statement execution -** (sqlite3_step() etc.). +** Functions sqlite3Error() or sqlite3ErrorWithMsg() should be used +** during statement execution (sqlite3_step() etc.). */ void sqlite3ErrorMsg(Parse *pParse, const char *zFormat, ...){ char *zMsg; @@ -195,7 +204,7 @@ void sqlite3ErrorMsg(Parse *pParse, const char *zFormat, ...){ ** occur. ** ** 2002-Feb-14: This routine is extended to remove MS-Access style -** brackets from around identifers. For example: "[a-b-c]" becomes +** brackets from around identifiers. For example: "[a-b-c]" becomes ** "a-b-c". */ int sqlite3Dequote(char *z){ @@ -242,6 +251,11 @@ int sqlite3Dequote(char *z){ */ int sqlite3_stricmp(const char *zLeft, const char *zRight){ register unsigned char *a, *b; + if( zLeft==0 ){ + return zRight ? -1 : 0; + }else if( zRight==0 ){ + return 1; + } a = (unsigned char *)zLeft; b = (unsigned char *)zRight; while( *a!=0 && UpperToLower[*a]==UpperToLower[*b]){ a++; b++; } @@ -249,6 +263,11 @@ int sqlite3_stricmp(const char *zLeft, const char *zRight){ } int sqlite3_strnicmp(const char *zLeft, const char *zRight, int N){ register unsigned char *a, *b; + if( zLeft==0 ){ + return zRight ? -1 : 0; + }else if( zRight==0 ){ + return 1; + } a = (unsigned char *)zLeft; b = (unsigned char *)zRight; while( N-- > 0 && *a!=0 && UpperToLower[*a]==UpperToLower[*b]){ a++; b++; } @@ -475,9 +494,9 @@ static int compare2pow63(const char *zNum, int incr){ return c; } - /* -** Convert zNum to a 64-bit signed integer. +** Convert zNum to a 64-bit signed integer. zNum must be decimal. This +** routine does *not* accept hexadecimal notation. ** ** If the zNum value is representable as a 64-bit twos-complement ** integer, then write that value into *pNum and return 0. @@ -565,10 +584,44 @@ int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc){ } } +/* +** Transform a UTF-8 integer literal, in either decimal or hexadecimal, +** into a 64-bit signed integer. This routine accepts hexadecimal literals, +** whereas sqlite3Atoi64() does not. +** +** Returns: +** +** 0 Successful transformation. Fits in a 64-bit signed integer. +** 1 Integer too large for a 64-bit signed integer or is malformed +** 2 Special case of 9223372036854775808 +*/ +int sqlite3DecOrHexToI64(const char *z, i64 *pOut){ +#ifndef SQLITE_OMIT_HEX_INTEGER + if( z[0]=='0' + && (z[1]=='x' || z[1]=='X') + && sqlite3Isxdigit(z[2]) + ){ + u64 u = 0; + int i, k; + for(i=2; z[i]=='0'; i++){} + for(k=i; sqlite3Isxdigit(z[k]); k++){ + u = u*16 + sqlite3HexToInt(z[k]); + } + memcpy(pOut, &u, 8); + return (z[k]==0 && k-i<=16) ? 0 : 1; + }else +#endif /* SQLITE_OMIT_HEX_INTEGER */ + { + return sqlite3Atoi64(z, pOut, sqlite3Strlen30(z), SQLITE_UTF8); + } +} + /* ** If zNum represents an integer that will fit in 32-bits, then set ** *pValue to that integer and return true. Otherwise return false. ** +** This routine accepts both decimal and hexadecimal notation for integers. +** ** Any non-numeric characters that following zNum are ignored. ** This is different from sqlite3Atoi64() which requires the ** input number to be zero-terminated. @@ -583,7 +636,25 @@ int sqlite3GetInt32(const char *zNum, int *pValue){ }else if( zNum[0]=='+' ){ zNum++; } - while( zNum[0]=='0' ) zNum++; +#ifndef SQLITE_OMIT_HEX_INTEGER + else if( zNum[0]=='0' + && (zNum[1]=='x' || zNum[1]=='X') + && sqlite3Isxdigit(zNum[2]) + ){ + u32 u = 0; + zNum += 2; + while( zNum[0]=='0' ) zNum++; + for(i=0; sqlite3Isxdigit(zNum[i]) && i<8; i++){ + u = u*16 + sqlite3HexToInt(zNum[i]); + } + if( (u&0x80000000)==0 && sqlite3Isxdigit(zNum[i])==0 ){ + memcpy(pValue, &u, 4); + return 1; + }else{ + return 0; + } + } +#endif for(i=0; i<11 && (c = zNum[i] - '0')>=0 && c<=9; i++){ v = v*10 + c; } @@ -647,7 +718,7 @@ int sqlite3Atoi(const char *z){ ** bit clear. Except, if we get to the 9th byte, it stores the full ** 8 bits and is the last byte. */ -int sqlite3PutVarint(unsigned char *p, u64 v){ +static int SQLITE_NOINLINE putVarint64(unsigned char *p, u64 v){ int i, j, n; u8 buf[10]; if( v & (((u64)0xff000000)<<32) ){ @@ -671,28 +742,17 @@ int sqlite3PutVarint(unsigned char *p, u64 v){ } return n; } - -/* -** This routine is a faster version of sqlite3PutVarint() that only -** works for 32-bit positive integers and which is optimized for -** the common case of small integers. A MACRO version, putVarint32, -** is provided which inlines the single-byte case. All code should use -** the MACRO version as this function assumes the single-byte case has -** already been handled. -*/ -int sqlite3PutVarint32(unsigned char *p, u32 v){ -#ifndef putVarint32 - if( (v & ~0x7f)==0 ){ - p[0] = v; +int sqlite3PutVarint(unsigned char *p, u64 v){ + if( v<=0x7f ){ + p[0] = v&0x7f; return 1; } -#endif - if( (v & ~0x3fff)==0 ){ - p[0] = (u8)((v>>7) | 0x80); - p[1] = (u8)(v & 0x7f); + if( v<=0x3fff ){ + p[0] = ((v>>7)&0x7f)|0x80; + p[1] = v&0x7f; return 2; } - return sqlite3PutVarint(p, v); + return putVarint64(p,v); } /* diff --git a/src/vacuum.c b/src/vacuum.c index 936a44a7fe..9df8e08b22 100644 --- a/src/vacuum.c +++ b/src/vacuum.c @@ -87,14 +87,14 @@ static int execExecSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ ** step (3) requires additional temporary disk space approximately equal ** to the size of the original database for the rollback journal. ** Hence, temporary disk space that is approximately 2x the size of the -** orginal database is required. Every page of the database is written +** original database is required. Every page of the database is written ** approximately 3 times: Once for step (2) and twice for step (3). ** Two writes per page are required in step (3) because the original ** database content must be written into the rollback journal prior to ** overwriting the database with the vacuumed content. ** ** Only 1x temporary space and only 1x writes would be required if -** the copy of step (3) were replace by deleting the original database +** the copy of step (3) were replaced by deleting the original database ** and renaming the transient database as the original. But that will ** not work if other processes are attached to the original database. ** And a power loss in between deleting the original and renaming the diff --git a/src/vdbe.c b/src/vdbe.c index 11aa035026..3dac74dfd4 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -116,6 +116,12 @@ int sqlite3_found_count = 0; ** branch can go. It is usually 2. "I" is the direction the branch ** goes. 0 means falls through. 1 means branch is taken. 2 means the ** second alternative branch is taken. +** +** iSrcLine is the source code line (from the __LINE__ macro) that +** generated the VDBE instruction. This instrumentation assumes that all +** source code is in a single file (the amalgamation). Special values 1 +** and 2 for the iSrcLine parameter mean that this particular branch is +** always taken or never taken, respectively. */ #if !defined(SQLITE_VDBE_COVERAGE) # define VdbeBranchTaken(I,M) @@ -140,7 +146,7 @@ int sqlite3_found_count = 0; ** already. Return non-zero if a malloc() fails. */ #define Stringify(P, enc) \ - if(((P)->flags&(MEM_Str|MEM_Blob))==0 && sqlite3VdbeMemStringify(P,enc)) \ + if(((P)->flags&(MEM_Str|MEM_Blob))==0 && sqlite3VdbeMemStringify(P,enc,0)) \ { goto no_mem; } /* @@ -203,11 +209,12 @@ static VdbeCursor *allocateCursor( sqlite3VdbeFreeCursor(p, p->apCsr[iCur]); p->apCsr[iCur] = 0; } - if( SQLITE_OK==sqlite3VdbeMemGrow(pMem, nByte, 0) ){ + if( SQLITE_OK==sqlite3VdbeMemClearAndResize(pMem, nByte) ){ p->apCsr[iCur] = pCx = (VdbeCursor*)pMem->z; memset(pCx, 0, sizeof(VdbeCursor)); pCx->iDb = iDb; pCx->nField = nField; + pCx->aOffset = &pCx->aType[nField]; if( isBtreeCursor ){ pCx->pCursor = (BtCursor*) &pMem->z[ROUND8(sizeof(VdbeCursor))+2*sizeof(u32)*nField]; @@ -222,21 +229,29 @@ static VdbeCursor *allocateCursor( ** do so without loss of information. In other words, if the string ** looks like a number, convert it into a number. If it does not ** look like a number, leave it alone. +** +** If the bTryForInt flag is true, then extra effort is made to give +** an integer representation. Strings that look like floating point +** values but which have no fractional component (example: '48.00') +** will have a MEM_Int representation when bTryForInt is true. +** +** If bTryForInt is false, then if the input string contains a decimal +** point or exponential notation, the result is only MEM_Real, even +** if there is an exact integer representation of the quantity. */ -static void applyNumericAffinity(Mem *pRec){ - if( (pRec->flags & (MEM_Real|MEM_Int))==0 ){ - double rValue; - i64 iValue; - u8 enc = pRec->enc; - if( (pRec->flags&MEM_Str)==0 ) return; - if( sqlite3AtoF(pRec->z, &rValue, pRec->n, enc)==0 ) return; - if( 0==sqlite3Atoi64(pRec->z, &iValue, pRec->n, enc) ){ - pRec->u.i = iValue; - pRec->flags |= MEM_Int; - }else{ - pRec->r = rValue; - pRec->flags |= MEM_Real; - } +static void applyNumericAffinity(Mem *pRec, int bTryForInt){ + double rValue; + i64 iValue; + u8 enc = pRec->enc; + assert( (pRec->flags & (MEM_Str|MEM_Int|MEM_Real))==MEM_Str ); + if( sqlite3AtoF(pRec->z, &rValue, pRec->n, enc)==0 ) return; + if( 0==sqlite3Atoi64(pRec->z, &iValue, pRec->n, enc) ){ + pRec->u.i = iValue; + pRec->flags |= MEM_Int; + }else{ + pRec->u.r = rValue; + pRec->flags |= MEM_Real; + if( bTryForInt ) sqlite3VdbeIntegerAffinity(pRec); } } @@ -263,21 +278,23 @@ static void applyAffinity( char affinity, /* The affinity to be applied */ u8 enc /* Use this text encoding */ ){ - if( affinity==SQLITE_AFF_TEXT ){ + if( affinity>=SQLITE_AFF_NUMERIC ){ + assert( affinity==SQLITE_AFF_INTEGER || affinity==SQLITE_AFF_REAL + || affinity==SQLITE_AFF_NUMERIC ); + if( (pRec->flags & MEM_Int)==0 ){ + if( (pRec->flags & MEM_Real)==0 ){ + if( pRec->flags & MEM_Str ) applyNumericAffinity(pRec,1); + }else{ + sqlite3VdbeIntegerAffinity(pRec); + } + } + }else if( affinity==SQLITE_AFF_TEXT ){ /* Only attempt the conversion to TEXT if there is an integer or real ** representation (blob and NULL do not get converted) but no string ** representation. */ if( 0==(pRec->flags&MEM_Str) && (pRec->flags&(MEM_Real|MEM_Int)) ){ - sqlite3VdbeMemStringify(pRec, enc); - } - pRec->flags &= ~(MEM_Real|MEM_Int); - }else if( affinity!=SQLITE_AFF_NONE ){ - assert( affinity==SQLITE_AFF_INTEGER || affinity==SQLITE_AFF_REAL - || affinity==SQLITE_AFF_NUMERIC ); - applyNumericAffinity(pRec); - if( pRec->flags & MEM_Real ){ - sqlite3VdbeIntegerAffinity(pRec); + sqlite3VdbeMemStringify(pRec, enc, 1); } } } @@ -292,7 +309,7 @@ int sqlite3_value_numeric_type(sqlite3_value *pVal){ int eType = sqlite3_value_type(pVal); if( eType==SQLITE_TEXT ){ Mem *pMem = (Mem*)pVal; - applyNumericAffinity(pMem); + applyNumericAffinity(pMem, 0); eType = sqlite3_value_type(pVal); } return eType; @@ -310,25 +327,37 @@ void sqlite3ValueApplyAffinity( applyAffinity((Mem *)pVal, affinity, enc); } +/* +** pMem currently only holds a string type (or maybe a BLOB that we can +** interpret as a string if we want to). Compute its corresponding +** numeric type, if has one. Set the pMem->u.r and pMem->u.i fields +** accordingly. +*/ +static u16 SQLITE_NOINLINE computeNumericType(Mem *pMem){ + assert( (pMem->flags & (MEM_Int|MEM_Real))==0 ); + assert( (pMem->flags & (MEM_Str|MEM_Blob))!=0 ); + if( sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc)==0 ){ + return 0; + } + if( sqlite3Atoi64(pMem->z, &pMem->u.i, pMem->n, pMem->enc)==SQLITE_OK ){ + return MEM_Int; + } + return MEM_Real; +} + /* ** Return the numeric type for pMem, either MEM_Int or MEM_Real or both or ** none. ** ** Unlike applyNumericAffinity(), this routine does not modify pMem->flags. -** But it does set pMem->r and pMem->u.i appropriately. +** But it does set pMem->u.r and pMem->u.i appropriately. */ static u16 numericType(Mem *pMem){ if( pMem->flags & (MEM_Int|MEM_Real) ){ return pMem->flags & (MEM_Int|MEM_Real); } if( pMem->flags & (MEM_Str|MEM_Blob) ){ - if( sqlite3AtoF(pMem->z, &pMem->r, pMem->n, pMem->enc)==0 ){ - return 0; - } - if( sqlite3Atoi64(pMem->z, &pMem->u.i, pMem->n, pMem->enc)==SQLITE_OK ){ - return MEM_Int; - } - return MEM_Real; + return computeNumericType(pMem); } return 0; } @@ -431,7 +460,7 @@ static void memTracePrint(Mem *p){ printf(" i:%lld", p->u.i); #ifndef SQLITE_OMIT_FLOATING_POINT }else if( p->flags & MEM_Real ){ - printf(" r:%g", p->r); + printf(" r:%g", p->u.r); #endif }else if( p->flags & MEM_RowSet ){ printf(" (rowset)"); @@ -579,6 +608,9 @@ int sqlite3VdbeExec( #endif nVmStep++; pOp = &aOp[pc]; +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + if( p->anExec ) p->anExec[pc]++; +#endif /* Only allow tracing if SQLITE_DEBUG is defined. */ @@ -612,7 +644,7 @@ int sqlite3VdbeExec( assert( pOp->p2<=(p->nMem-p->nCursor) ); pOut = &aMem[pOp->p2]; memAboutToChange(p, pOut); - VdbeMemRelease(pOut); + if( VdbeMemDynamic(pOut) ) sqlite3VdbeMemSetNull(pOut); pOut->flags = MEM_Int; } @@ -767,12 +799,14 @@ case OP_Return: { /* in1 */ /* Opcode: InitCoroutine P1 P2 P3 * * ** -** Set up register P1 so that it will OP_Yield to the co-routine +** Set up register P1 so that it will Yield to the coroutine ** located at address P3. ** -** If P2!=0 then the co-routine implementation immediately follows -** this opcode. So jump over the co-routine implementation to +** If P2!=0 then the coroutine implementation immediately follows +** this opcode. So jump over the coroutine implementation to ** address P2. +** +** See also: EndCoroutine */ case OP_InitCoroutine: { /* jump */ assert( pOp->p1>0 && pOp->p1<=(p->nMem-p->nCursor) ); @@ -788,9 +822,11 @@ case OP_InitCoroutine: { /* jump */ /* Opcode: EndCoroutine P1 * * * * ** -** The instruction at the address in register P1 is an OP_Yield. -** Jump to the P2 parameter of that OP_Yield. +** The instruction at the address in register P1 is a Yield. +** Jump to the P2 parameter of that Yield. ** After the jump, register P1 becomes undefined. +** +** See also: InitCoroutine */ case OP_EndCoroutine: { /* in1 */ VdbeOp *pCaller; @@ -807,11 +843,16 @@ case OP_EndCoroutine: { /* in1 */ /* Opcode: Yield P1 P2 * * * ** -** Swap the program counter with the value in register P1. +** Swap the program counter with the value in register P1. This +** has the effect of yielding to a coroutine. ** -** If the co-routine ends with OP_Yield or OP_Return then continue -** to the next instruction. But if the co-routine ends with -** OP_EndCoroutine, jump immediately to P2. +** If the coroutine that is launched by this instruction ends with +** Yield or Return then continue to the next instruction. But if +** the coroutine launched by this instruction ends with +** EndCoroutine, then jump to P2 rather than continuing with the +** next instruction. +** +** See also: InitCoroutine */ case OP_Yield: { /* in1, jump */ int pcDest; @@ -965,7 +1006,7 @@ case OP_Int64: { /* out2-prerelease */ case OP_Real: { /* same as TK_FLOAT, out2-prerelease */ pOut->flags = MEM_Real; assert( !sqlite3IsNaN(*pOp->p4.pReal) ); - pOut->r = *pOp->p4.pReal; + pOut->u.r = *pOp->p4.pReal; break; } #endif @@ -974,7 +1015,7 @@ case OP_Real: { /* same as TK_FLOAT, out2-prerelease */ ** Synopsis: r[P2]='P4' ** ** P4 points to a nul terminated UTF-8 string. This opcode is transformed -** into an OP_String before it is executed for the first time. During +** into a String before it is executed for the first time. During ** this transformation, the length of string P4 is computed and stored ** as the P1 parameter. */ @@ -988,9 +1029,9 @@ case OP_String8: { /* same as TK_STRING, out2-prerelease */ rc = sqlite3VdbeMemSetStr(pOut, pOp->p4.z, -1, SQLITE_UTF8, SQLITE_STATIC); if( rc==SQLITE_TOOBIG ) goto too_big; if( SQLITE_OK!=sqlite3VdbeChangeEncoding(pOut, encoding) ) goto no_mem; - assert( pOut->zMalloc==pOut->z ); + assert( pOut->szMalloc>0 && pOut->zMalloc==pOut->z ); assert( VdbeMemDynamic(pOut)==0 ); - pOut->zMalloc = 0; + pOut->szMalloc = 0; pOut->flags |= MEM_Static; if( pOp->p4type==P4_DYNAMIC ){ sqlite3DbFree(db, pOp->p4.z); @@ -1042,7 +1083,7 @@ case OP_Null: { /* out2-prerelease */ while( cnt>0 ){ pOut++; memAboutToChange(p, pOut); - VdbeMemRelease(pOut); + sqlite3VdbeMemSetNull(pOut); pOut->flags = nullFlag; cnt--; } @@ -1110,7 +1151,6 @@ case OP_Variable: { /* out2-prerelease */ ** for P3 to be less than 1. */ case OP_Move: { - char *zMalloc; /* Holding variable for allocated memory */ int n; /* Number of registers left to copy */ int p1; /* Register to copy from */ int p2; /* Register to copy to */ @@ -1128,17 +1168,12 @@ case OP_Move: { assert( pIn1<=&aMem[(p->nMem-p->nCursor)] ); assert( memIsValid(pIn1) ); memAboutToChange(p, pOut); - VdbeMemRelease(pOut); - zMalloc = pOut->zMalloc; - memcpy(pOut, pIn1, sizeof(Mem)); + sqlite3VdbeMemMove(pOut, pIn1); #ifdef SQLITE_DEBUG if( pOut->pScopyFrom>=&aMem[p1] && pOut->pScopyFrom<&aMem[p1+pOp->p3] ){ pOut->pScopyFrom += p1 - pOp->p2; } #endif - pIn1->flags = MEM_Undefined; - pIn1->xDel = 0; - pIn1->zMalloc = zMalloc; REGISTER_TRACE(p2++, pOut); pIn1++; pOut++; @@ -1443,7 +1478,7 @@ fp_math: if( sqlite3IsNaN(rB) ){ goto arithmetic_result_is_null; } - pOut->r = rB; + pOut->u.r = rB; MemSetTypeFlag(pOut, MEM_Real); if( ((type1|type2)&MEM_Real)==0 && !bIntint ){ sqlite3VdbeIntegerAffinity(pOut); @@ -1508,8 +1543,8 @@ case OP_Function: { apVal = p->apArg; assert( apVal || n==0 ); assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); - pOut = &aMem[pOp->p3]; - memAboutToChange(p, pOut); + ctx.pOut = &aMem[pOp->p3]; + memAboutToChange(p, ctx.pOut); assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem-p->nCursor)+1) ); assert( pOp->p3p2 || pOp->p3>=pOp->p2+n ); @@ -1525,65 +1560,29 @@ case OP_Function: { ctx.pFunc = pOp->p4.pFunc; ctx.iOp = pc; ctx.pVdbe = p; - - /* The output cell may already have a buffer allocated. Move - ** the pointer to ctx.s so in case the user-function can use - ** the already allocated buffer instead of allocating a new one. - */ - memcpy(&ctx.s, pOut, sizeof(Mem)); - pOut->flags = MEM_Null; - pOut->xDel = 0; - pOut->zMalloc = 0; - MemSetTypeFlag(&ctx.s, MEM_Null); - + MemSetTypeFlag(ctx.pOut, MEM_Null); ctx.fErrorOrAux = 0; - if( ctx.pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){ - assert( pOp>aOp ); - assert( pOp[-1].p4type==P4_COLLSEQ ); - assert( pOp[-1].opcode==OP_CollSeq ); - ctx.pColl = pOp[-1].p4.pColl; - } db->lastRowid = lastRowid; (*ctx.pFunc->xFunc)(&ctx, n, apVal); /* IMP: R-24505-23230 */ - lastRowid = db->lastRowid; - - if( db->mallocFailed ){ - /* Even though a malloc() has failed, the implementation of the - ** user function may have called an sqlite3_result_XXX() function - ** to return a value. The following call releases any resources - ** associated with such a value. - */ - sqlite3VdbeMemRelease(&ctx.s); - goto no_mem; - } + lastRowid = db->lastRowid; /* Remember rowid changes made by xFunc */ /* If the function returned an error, throw an exception */ if( ctx.fErrorOrAux ){ if( ctx.isError ){ - sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&ctx.s)); + sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(ctx.pOut)); rc = ctx.isError; } sqlite3VdbeDeleteAuxData(p, pc, pOp->p1); } /* Copy the result of the function into register P3 */ - sqlite3VdbeChangeEncoding(&ctx.s, encoding); - assert( pOut->flags==MEM_Null ); - memcpy(pOut, &ctx.s, sizeof(Mem)); - if( sqlite3VdbeMemTooBig(pOut) ){ + sqlite3VdbeChangeEncoding(ctx.pOut, encoding); + if( sqlite3VdbeMemTooBig(ctx.pOut) ){ goto too_big; } -#if 0 - /* The app-defined function has done something that as caused this - ** statement to expire. (Perhaps the function called sqlite3_exec() - ** with a CREATE TABLE statement.) - */ - if( p->expired ) rc = SQLITE_ABORT; -#endif - - REGISTER_TRACE(pOp->p3, pOut); - UPDATE_MAX_BLOBSIZE(pOut); + REGISTER_TRACE(pOp->p3, ctx.pOut); + UPDATE_MAX_BLOBSIZE(ctx.pOut); break; } @@ -1731,106 +1730,37 @@ case OP_RealAffinity: { /* in1 */ #endif #ifndef SQLITE_OMIT_CAST -/* Opcode: ToText P1 * * * * +/* Opcode: Cast P1 P2 * * * +** Synopsis: affinity(r[P1]) ** -** Force the value in register P1 to be text. -** If the value is numeric, convert it to a string using the -** equivalent of sprintf(). Blob values are unchanged and -** are afterwards simply interpreted as text. +** Force the value in register P1 to be the type defined by P2. +** +**
      +**
    • TEXT +**
    • BLOB +**
    • NUMERIC +**
    • INTEGER +**
    • REAL +**
    ** ** A NULL value is not changed by this routine. It remains NULL. */ -case OP_ToText: { /* same as TK_TO_TEXT, in1 */ +case OP_Cast: { /* in1 */ + assert( pOp->p2>=SQLITE_AFF_NONE && pOp->p2<=SQLITE_AFF_REAL ); + testcase( pOp->p2==SQLITE_AFF_TEXT ); + testcase( pOp->p2==SQLITE_AFF_NONE ); + testcase( pOp->p2==SQLITE_AFF_NUMERIC ); + testcase( pOp->p2==SQLITE_AFF_INTEGER ); + testcase( pOp->p2==SQLITE_AFF_REAL ); pIn1 = &aMem[pOp->p1]; memAboutToChange(p, pIn1); - if( pIn1->flags & MEM_Null ) break; - assert( MEM_Str==(MEM_Blob>>3) ); - pIn1->flags |= (pIn1->flags&MEM_Blob)>>3; - applyAffinity(pIn1, SQLITE_AFF_TEXT, encoding); rc = ExpandBlob(pIn1); - assert( pIn1->flags & MEM_Str || db->mallocFailed ); - pIn1->flags &= ~(MEM_Int|MEM_Real|MEM_Blob|MEM_Zero); + sqlite3VdbeMemCast(pIn1, pOp->p2, encoding); UPDATE_MAX_BLOBSIZE(pIn1); break; } - -/* Opcode: ToBlob P1 * * * * -** -** Force the value in register P1 to be a BLOB. -** If the value is numeric, convert it to a string first. -** Strings are simply reinterpreted as blobs with no change -** to the underlying data. -** -** A NULL value is not changed by this routine. It remains NULL. -*/ -case OP_ToBlob: { /* same as TK_TO_BLOB, in1 */ - pIn1 = &aMem[pOp->p1]; - if( pIn1->flags & MEM_Null ) break; - if( (pIn1->flags & MEM_Blob)==0 ){ - applyAffinity(pIn1, SQLITE_AFF_TEXT, encoding); - assert( pIn1->flags & MEM_Str || db->mallocFailed ); - MemSetTypeFlag(pIn1, MEM_Blob); - }else{ - pIn1->flags &= ~(MEM_TypeMask&~MEM_Blob); - } - UPDATE_MAX_BLOBSIZE(pIn1); - break; -} - -/* Opcode: ToNumeric P1 * * * * -** -** Force the value in register P1 to be numeric (either an -** integer or a floating-point number.) -** If the value is text or blob, try to convert it to an using the -** equivalent of atoi() or atof() and store 0 if no such conversion -** is possible. -** -** A NULL value is not changed by this routine. It remains NULL. -*/ -case OP_ToNumeric: { /* same as TK_TO_NUMERIC, in1 */ - pIn1 = &aMem[pOp->p1]; - sqlite3VdbeMemNumerify(pIn1); - break; -} #endif /* SQLITE_OMIT_CAST */ -/* Opcode: ToInt P1 * * * * -** -** Force the value in register P1 to be an integer. If -** The value is currently a real number, drop its fractional part. -** If the value is text or blob, try to convert it to an integer using the -** equivalent of atoi() and store 0 if no such conversion is possible. -** -** A NULL value is not changed by this routine. It remains NULL. -*/ -case OP_ToInt: { /* same as TK_TO_INT, in1 */ - pIn1 = &aMem[pOp->p1]; - if( (pIn1->flags & MEM_Null)==0 ){ - sqlite3VdbeMemIntegerify(pIn1); - } - break; -} - -#if !defined(SQLITE_OMIT_CAST) && !defined(SQLITE_OMIT_FLOATING_POINT) -/* Opcode: ToReal P1 * * * * -** -** Force the value in register P1 to be a floating point number. -** If The value is currently an integer, convert it. -** If the value is text or blob, try to convert it to an integer using the -** equivalent of atoi() and store 0.0 if no such conversion is possible. -** -** A NULL value is not changed by this routine. It remains NULL. -*/ -case OP_ToReal: { /* same as TK_TO_REAL, in1 */ - pIn1 = &aMem[pOp->p1]; - memAboutToChange(p, pIn1); - if( (pIn1->flags & MEM_Null)==0 ){ - sqlite3VdbeMemRealify(pIn1); - } - break; -} -#endif /* !defined(SQLITE_OMIT_CAST) && !defined(SQLITE_OMIT_FLOATING_POINT) */ - /* Opcode: Lt P1 P2 P3 P4 P5 ** Synopsis: if r[P1]p5 & SQLITE_AFF_MASK; - if( affinity ){ - applyAffinity(pIn1, affinity, encoding); - applyAffinity(pIn3, affinity, encoding); - if( db->mallocFailed ) goto no_mem; + if( affinity>=SQLITE_AFF_NUMERIC ){ + if( (pIn1->flags & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){ + applyNumericAffinity(pIn1,0); + } + if( (pIn3->flags & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){ + applyNumericAffinity(pIn3,0); + } + }else if( affinity==SQLITE_AFF_TEXT ){ + if( (pIn1->flags & MEM_Str)==0 && (pIn1->flags & (MEM_Int|MEM_Real))!=0 ){ + testcase( pIn1->flags & MEM_Int ); + testcase( pIn1->flags & MEM_Real ); + sqlite3VdbeMemStringify(pIn1, encoding, 1); + } + if( (pIn3->flags & MEM_Str)==0 && (pIn3->flags & (MEM_Int|MEM_Real))!=0 ){ + testcase( pIn3->flags & MEM_Int ); + testcase( pIn3->flags & MEM_Real ); + sqlite3VdbeMemStringify(pIn3, encoding, 1); + } } - assert( pOp->p4type==P4_COLLSEQ || pOp->p4.pColl==0 ); - ExpandBlob(pIn1); - ExpandBlob(pIn3); + if( pIn1->flags & MEM_Zero ){ + sqlite3VdbeMemExpandBlob(pIn1); + flags1 &= ~MEM_Zero; + } + if( pIn3->flags & MEM_Zero ){ + sqlite3VdbeMemExpandBlob(pIn3); + flags3 &= ~MEM_Zero; + } + if( db->mallocFailed ) goto no_mem; res = sqlite3MemCompare(pIn3, pIn1, pOp->p4.pColl); } switch( pOp->opcode ){ @@ -1999,8 +1949,8 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ } } /* Undo any changes made by applyAffinity() to the input registers. */ - pIn1->flags = (pIn1->flags&~MEM_TypeMask) | (flags1&MEM_TypeMask); - pIn3->flags = (pIn3->flags&~MEM_TypeMask) | (flags3&MEM_TypeMask); + pIn1->flags = flags1; + pIn3->flags = flags3; break; } @@ -2168,10 +2118,10 @@ case OP_Or: { /* same as TK_OR, in1, in2, out3 */ case OP_Not: { /* same as TK_NOT, in1, out2 */ pIn1 = &aMem[pOp->p1]; pOut = &aMem[pOp->p2]; - if( pIn1->flags & MEM_Null ){ - sqlite3VdbeMemSetNull(pOut); - }else{ - sqlite3VdbeMemSetInt64(pOut, !sqlite3VdbeIntValue(pIn1)); + sqlite3VdbeMemSetNull(pOut); + if( (pIn1->flags & MEM_Null)==0 ){ + pOut->flags = MEM_Int; + pOut->u.i = !sqlite3VdbeIntValue(pIn1); } break; } @@ -2186,20 +2136,24 @@ case OP_Not: { /* same as TK_NOT, in1, out2 */ case OP_BitNot: { /* same as TK_BITNOT, in1, out2 */ pIn1 = &aMem[pOp->p1]; pOut = &aMem[pOp->p2]; - if( pIn1->flags & MEM_Null ){ - sqlite3VdbeMemSetNull(pOut); - }else{ - sqlite3VdbeMemSetInt64(pOut, ~sqlite3VdbeIntValue(pIn1)); + sqlite3VdbeMemSetNull(pOut); + if( (pIn1->flags & MEM_Null)==0 ){ + pOut->flags = MEM_Int; + pOut->u.i = ~sqlite3VdbeIntValue(pIn1); } break; } /* Opcode: Once P1 P2 * * * ** -** Check if OP_Once flag P1 is set. If so, jump to instruction P2. Otherwise, -** set the flag and fall through to the next instruction. In other words, -** this opcode causes all following opcodes up through P2 (but not including -** P2) to run just once and to be skipped on subsequent times through the loop. +** Check the "once" flag number P1. If it is set, jump to instruction P2. +** Otherwise, set the flag and fall through to the next instruction. +** In other words, this opcode causes all following opcodes up through P2 +** (but not including P2) to run just once and to be skipped on subsequent +** times through the loop. +** +** All "once" flags are initially cleared whenever a prepared statement +** first begins to run. */ case OP_Once: { /* jump */ assert( pOp->p1nOnceFlag ); @@ -2216,13 +2170,13 @@ case OP_Once: { /* jump */ ** ** Jump to P2 if the value in register P1 is true. The value ** is considered true if it is numeric and non-zero. If the value -** in P1 is NULL then take the jump if P3 is non-zero. +** in P1 is NULL then take the jump if and only if P3 is non-zero. */ /* Opcode: IfNot P1 P2 P3 * * ** ** Jump to P2 if the value in register P1 is False. The value ** is considered false if it has a numeric value of zero. If the value -** in P1 is NULL then take the jump if P3 is zero. +** in P1 is NULL then take the jump if and only if P3 is non-zero. */ case OP_If: /* jump, in1 */ case OP_IfNot: { /* jump, in1 */ @@ -2303,7 +2257,6 @@ case OP_Column: { int p2; /* column number to retrieve */ VdbeCursor *pC; /* The VDBE cursor */ BtCursor *pCrsr; /* The BTree cursor */ - u32 *aType; /* aType[i] holds the numeric type of the i-th column */ u32 *aOffset; /* aOffset[i] is offset to start of data for i-th column */ int len; /* The length of the serialized data for the column */ int i; /* Loop counter */ @@ -2316,6 +2269,7 @@ case OP_Column: { u32 szField; /* Number of bytes in the content of a field */ u32 avail; /* Number of bytes of available data */ u32 t; /* A type code from the record header */ + u16 fx; /* pDest->flags value */ Mem *pReg; /* PseudoTable input register */ p2 = pOp->p2; @@ -2326,8 +2280,7 @@ case OP_Column: { pC = p->apCsr[pOp->p1]; assert( pC!=0 ); assert( p2nField ); - aType = pC->aType; - aOffset = aType + pC->nField; + aOffset = pC->aOffset; #ifndef SQLITE_OMIT_VIRTUALTABLE assert( pC->pVtabCursor==0 ); /* OP_Column never called on virtual table */ #endif @@ -2338,7 +2291,7 @@ case OP_Column: { /* If the cursor cache is stale, bring it up-to-date */ rc = sqlite3VdbeCursorMoveto(pC); if( rc ) goto abort_due_to_error; - if( pC->cacheStatus!=p->cacheCtr || (pOp->p5&OPFLAG_CLEARCACHE)!=0 ){ + if( pC->cacheStatus!=p->cacheCtr ){ if( pC->nullRow ){ if( pCrsr==0 ){ assert( pC->pseudoTableReg>0 ); @@ -2348,7 +2301,7 @@ case OP_Column: { pC->payloadSize = pC->szRow = avail = pReg->n; pC->aRow = (u8*)pReg->z; }else{ - MemSetTypeFlag(pDest, MEM_Null); + sqlite3VdbeMemSetNull(pDest); goto op_column_out; } }else{ @@ -2383,14 +2336,6 @@ case OP_Column: { pC->iHdrOffset = getVarint32(pC->aRow, offset); pC->nHdrParsed = 0; aOffset[0] = offset; - if( availaRow does not have to hold the entire row, but it does at least - ** need to cover the header of the record. If pC->aRow does not contain - ** the complete header, then set it to zero, forcing the header to be - ** dynamically allocated. */ - pC->aRow = 0; - pC->szRow = 0; - } /* Make sure a corrupt database has not given us an oversize header. ** Do this now to avoid an oversize memory allocation. @@ -2405,15 +2350,32 @@ case OP_Column: { rc = SQLITE_CORRUPT_BKPT; goto op_column_error; } + + if( availaRow does not have to hold the entire row, but it does at least + ** need to cover the header of the record. If pC->aRow does not contain + ** the complete header, then set it to zero, forcing the header to be + ** dynamically allocated. */ + pC->aRow = 0; + pC->szRow = 0; + } + + /* The following goto is an optimization. It can be omitted and + ** everything will still work. But OP_Column is measurably faster + ** by skipping the subsequent conditional, which is always true. + */ + assert( pC->nHdrParsed<=p2 ); /* Conditional skipped */ + goto op_column_read_header; } /* Make sure at least the first p2+1 entries of the header have been - ** parsed and valid information is in aOffset[] and aType[]. + ** parsed and valid information is in aOffset[] and pC->aType[]. */ if( pC->nHdrParsed<=p2 ){ /* If there is more header available for parsing in the record, try ** to extract additional fields up through the p2+1-th field */ + op_column_read_header: if( pC->iHdrOffsetaRow==0 ){ @@ -2428,7 +2390,7 @@ case OP_Column: { zData = pC->aRow; } - /* Fill in aType[i] and aOffset[i] values through the p2-th field. */ + /* Fill in pC->aType[i] and aOffset[i] values through the p2-th field. */ i = pC->nHdrParsed; offset = aOffset[i]; zHdr = zData + pC->iHdrOffset; @@ -2441,7 +2403,7 @@ case OP_Column: { }else{ zHdr += sqlite3GetVarint32(zHdr, &t); } - aType[i] = t; + pC->aType[i] = t; szField = sqlite3VdbeSerialTypeLen(t); offset += szField; if( offsetzEndHdr) + ** (2) the entire header was used but not all data was used + ** (zHdr==zEndHdr && offset!=pC->payloadSize) + ** (3) the end of the data extends beyond the end of the record. + ** (offset > pC->payloadSize) */ - if( (zHdr > zEndHdr) + if( (zHdr>=zEndHdr && (zHdr>zEndHdr || offset!=pC->payloadSize)) || (offset > pC->payloadSize) - || (zHdr==zEndHdr && offset!=pC->payloadSize) ){ rc = SQLITE_CORRUPT_BKPT; goto op_column_error; @@ -2481,68 +2444,68 @@ case OP_Column: { if( pOp->p4type==P4_MEM ){ sqlite3VdbeMemShallowCopy(pDest, pOp->p4.pMem, MEM_Static); }else{ - MemSetTypeFlag(pDest, MEM_Null); + sqlite3VdbeMemSetNull(pDest); } goto op_column_out; } } /* Extract the content for the p2+1-th column. Control can only - ** reach this point if aOffset[p2], aOffset[p2+1], and aType[p2] are + ** reach this point if aOffset[p2], aOffset[p2+1], and pC->aType[p2] are ** all valid. */ assert( p2nHdrParsed ); assert( rc==SQLITE_OK ); assert( sqlite3VdbeCheckMemInvariants(pDest) ); + if( VdbeMemDynamic(pDest) ) sqlite3VdbeMemSetNull(pDest); + t = pC->aType[p2]; if( pC->szRow>=aOffset[p2+1] ){ /* This is the common case where the desired content fits on the original ** page - where the content is not on an overflow page */ - VdbeMemRelease(pDest); - sqlite3VdbeSerialGet(pC->aRow+aOffset[p2], aType[p2], pDest); + sqlite3VdbeSerialGet(pC->aRow+aOffset[p2], t, pDest); }else{ /* This branch happens only when content is on overflow pages */ - t = aType[p2]; if( ((pOp->p5 & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG))!=0 && ((t>=12 && (t&1)==0) || (pOp->p5 & OPFLAG_TYPEOFARG)!=0)) || (len = sqlite3VdbeSerialTypeLen(t))==0 ){ - /* Content is irrelevant for the typeof() function and for - ** the length(X) function if X is a blob. So we might as well use - ** bogus content rather than reading content from disk. NULL works - ** for text and blob and whatever is in the payloadSize64 variable - ** will work for everything else. Content is also irrelevant if - ** the content length is 0. */ - zData = t<=13 ? (u8*)&payloadSize64 : 0; - sMem.zMalloc = 0; + /* Content is irrelevant for + ** 1. the typeof() function, + ** 2. the length(X) function if X is a blob, and + ** 3. if the content length is zero. + ** So we might as well use bogus content rather than reading + ** content from disk. NULL will work for the value for strings + ** and blobs and whatever is in the payloadSize64 variable + ** will work for everything else. */ + sqlite3VdbeSerialGet(t<=13 ? (u8*)&payloadSize64 : 0, t, pDest); }else{ - memset(&sMem, 0, sizeof(sMem)); - sqlite3VdbeMemMove(&sMem, pDest); rc = sqlite3VdbeMemFromBtree(pCrsr, aOffset[p2], len, !pC->isTable, - &sMem); + pDest); if( rc!=SQLITE_OK ){ goto op_column_error; } - zData = (u8*)sMem.z; - } - sqlite3VdbeSerialGet(zData, t, pDest); - /* If we dynamically allocated space to hold the data (in the - ** sqlite3VdbeMemFromBtree() call above) then transfer control of that - ** dynamically allocated space over to the pDest structure. - ** This prevents a memory copy. */ - if( sMem.zMalloc ){ - assert( sMem.z==sMem.zMalloc ); - assert( VdbeMemDynamic(pDest)==0 ); - assert( (pDest->flags & (MEM_Blob|MEM_Str))==0 || pDest->z==sMem.z ); - pDest->flags &= ~(MEM_Ephem|MEM_Static); - pDest->flags |= MEM_Term; - pDest->z = sMem.z; - pDest->zMalloc = sMem.zMalloc; + sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest); + pDest->flags &= ~MEM_Ephem; } } pDest->enc = encoding; op_column_out: - Deephemeralize(pDest); + /* If the column value is an ephemeral string, go ahead and persist + ** that string in case the cursor moves before the column value is + ** used. The following code does the equivalent of Deephemeralize() + ** but does it faster. */ + if( (pDest->flags & MEM_Ephem)!=0 && pDest->z ){ + fx = pDest->flags & (MEM_Str|MEM_Blob); + assert( fx!=0 ); + zData = (const u8*)pDest->z; + len = pDest->n; + if( sqlite3VdbeMemClearAndResize(pDest, len+2) ) goto no_mem; + memcpy(pDest->z, zData, len); + pDest->z[len] = 0; + pDest->z[len+1] = 0; + pDest->flags = fx|MEM_Term; + } op_column_error: UPDATE_MAX_BLOBSIZE(pDest); REGISTER_TRACE(pOp->p3, pDest); @@ -2617,7 +2580,7 @@ case OP_MakeRecord: { ** ------------------------------------------------------------------------ ** ** Data(0) is taken from register P1. Data(1) comes from register P1+1 - ** and so froth. + ** and so forth. ** ** Each type field is a varint representing the serial type of the ** corresponding data element (see sqlite3VdbeSerialType()). The @@ -2657,7 +2620,7 @@ case OP_MakeRecord: { pRec = pLast; do{ assert( memIsValid(pRec) ); - serial_type = sqlite3VdbeSerialType(pRec, file_format); + pRec->uTemp = serial_type = sqlite3VdbeSerialType(pRec, file_format); len = sqlite3VdbeSerialTypeLen(serial_type); if( pRec->flags & MEM_Zero ){ if( nData ){ @@ -2673,7 +2636,10 @@ case OP_MakeRecord: { nHdr += serial_type<=127 ? 1 : sqlite3VarintLen(serial_type); }while( (--pRec)>=pData0 ); - /* Add the initial header varint and total the size */ + /* EVIDENCE-OF: R-22564-11647 The header begins with a single varint + ** which determines the total number of bytes in the header. The varint + ** value is the size of the header in bytes including the size varint + ** itself. */ testcase( nHdr==126 ); testcase( nHdr==127 ); if( nHdr<=126 ){ @@ -2693,9 +2659,9 @@ case OP_MakeRecord: { /* Make sure the output register has a buffer large enough to store ** the new record. The output register (pOp->p3) is not allowed to ** be one of the input registers (because the following call to - ** sqlite3VdbeMemGrow() could clobber the value before it is used). + ** sqlite3VdbeMemClearAndResize() could clobber the value before it is used). */ - if( sqlite3VdbeMemGrow(pOut, (int)nByte, 0) ){ + if( sqlite3VdbeMemClearAndResize(pOut, (int)nByte) ){ goto no_mem; } zNewRecord = (u8 *)pOut->z; @@ -2706,8 +2672,12 @@ case OP_MakeRecord: { assert( pData0<=pLast ); pRec = pData0; do{ - serial_type = sqlite3VdbeSerialType(pRec, file_format); + serial_type = pRec->uTemp; + /* EVIDENCE-OF: R-06529-47362 Following the size varint are one or more + ** additional varints, one per column. */ i += putVarint32(&zNewRecord[i], serial_type); /* serial type */ + /* EVIDENCE-OF: R-64536-51728 The values for each column in the record + ** immediately follow the header. */ j += sqlite3VdbeSerialPut(&zNewRecord[j], pRec, serial_type); /* content */ }while( (++pRec)<=pLast ); assert( i==nHdr ); @@ -2716,7 +2686,6 @@ case OP_MakeRecord: { assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); pOut->n = (int)nByte; pOut->flags = MEM_Blob; - pOut->xDel = 0; if( nZero ){ pOut->u.nZero = nZero; pOut->flags |= MEM_Zero; @@ -2863,11 +2832,18 @@ case OP_Savepoint: { db->isTransactionSavepoint = 0; rc = p->rc; }else{ + int isSchemaChange; iSavepoint = db->nSavepoint - iSavepoint - 1; if( p1==SAVEPOINT_ROLLBACK ){ + isSchemaChange = (db->flags & SQLITE_InternChanges)!=0; for(ii=0; iinDb; ii++){ - sqlite3BtreeTripAllCursors(db->aDb[ii].pBt, SQLITE_ABORT); + rc = sqlite3BtreeTripAllCursors(db->aDb[ii].pBt, + SQLITE_ABORT_ROLLBACK, + isSchemaChange==0); + if( rc!=SQLITE_OK ) goto abort_due_to_error; } + }else{ + isSchemaChange = 0; } for(ii=0; iinDb; ii++){ rc = sqlite3BtreeSavepoint(db->aDb[ii].pBt, p1, iSavepoint); @@ -2875,7 +2851,7 @@ case OP_Savepoint: { goto abort_due_to_error; } } - if( p1==SAVEPOINT_ROLLBACK && (db->flags&SQLITE_InternChanges)!=0 ){ + if( isSchemaChange ){ sqlite3ExpirePreparedStatements(db); sqlite3ResetAllSchemasOfConnection(db); db->flags = (db->flags | SQLITE_InternChanges); @@ -3034,7 +3010,7 @@ case OP_Transaction: { assert( p->bIsReader ); assert( p->readOnly==0 || pOp->p2==0 ); assert( pOp->p1>=0 && pOp->p1nDb ); - assert( (p->btreeMask & (((yDbMask)1)<p1))!=0 ); + assert( DbMaskTest(p->btreeMask, pOp->p1) ); if( pOp->p2 && (db->flags & SQLITE_QueryOnly)!=0 ){ rc = SQLITE_READONLY; goto abort_due_to_error; @@ -3129,7 +3105,7 @@ case OP_ReadCookie: { /* out2-prerelease */ assert( pOp->p3=0 && iDbnDb ); assert( db->aDb[iDb].pBt!=0 ); - assert( (p->btreeMask & (((yDbMask)1)<btreeMask, iDb) ); sqlite3BtreeGetMeta(db->aDb[iDb].pBt, iCookie, (u32 *)&iMeta); pOut->u.i = iMeta; @@ -3150,7 +3126,7 @@ case OP_SetCookie: { /* in3 */ Db *pDb; assert( pOp->p2p1>=0 && pOp->p1nDb ); - assert( (p->btreeMask & (((yDbMask)1)<p1))!=0 ); + assert( DbMaskTest(p->btreeMask, pOp->p1) ); assert( p->readOnly==0 ); pDb = &db->aDb[pOp->p1]; assert( pDb->pBt!=0 ); @@ -3205,7 +3181,21 @@ case OP_SetCookie: { /* in3 */ ** sequence of the index being opened. Otherwise, if P4 is an integer ** value, it is set to the number of columns in the table. ** -** See also OpenWrite. +** See also: OpenWrite, ReopenIdx +*/ +/* Opcode: ReopenIdx P1 P2 P3 P4 P5 +** Synopsis: root=P2 iDb=P3 +** +** The ReopenIdx opcode works exactly like ReadOpen except that it first +** checks to see if the cursor on P1 is already open with a root page +** number of P2 and if it is this opcode becomes a no-op. In other words, +** if the cursor is already open, do not reopen it. +** +** The ReopenIdx opcode may only be used with P5==0 and with P4 being +** a P4_KEYINFO object. Furthermore, the P3 value must be the same as +** every other ReopenIdx or OpenRead for the same cursor number. +** +** See the OpenRead opcode documentation for additional information. */ /* Opcode: OpenWrite P1 P2 P3 P4 P5 ** Synopsis: root=P2 iDb=P3 @@ -3227,6 +3217,19 @@ case OP_SetCookie: { /* in3 */ ** ** See also OpenRead. */ +case OP_ReopenIdx: { + VdbeCursor *pCur; + + assert( pOp->p5==0 ); + assert( pOp->p4type==P4_KEYINFO ); + pCur = p->apCsr[pOp->p1]; + if( pCur && pCur->pgnoRoot==(u32)pOp->p2 ){ + assert( pCur->iDb==pOp->p3 ); /* Guaranteed by the code generator */ + break; + } + /* If the cursor is not currently open or is open on a different + ** index, then fall through into OP_OpenRead to force a reopen */ +} case OP_OpenRead: case OP_OpenWrite: { int nField; @@ -3241,10 +3244,11 @@ case OP_OpenWrite: { assert( (pOp->p5&(OPFLAG_P2ISREG|OPFLAG_BULKCSR))==pOp->p5 ); assert( pOp->opcode==OP_OpenWrite || pOp->p5==0 ); assert( p->bIsReader ); - assert( pOp->opcode==OP_OpenRead || p->readOnly==0 ); + assert( pOp->opcode==OP_OpenRead || pOp->opcode==OP_ReopenIdx + || p->readOnly==0 ); if( p->expired ){ - rc = SQLITE_ABORT; + rc = SQLITE_ABORT_ROLLBACK; break; } @@ -3253,7 +3257,7 @@ case OP_OpenWrite: { p2 = pOp->p2; iDb = pOp->p3; assert( iDb>=0 && iDbnDb ); - assert( (p->btreeMask & (((yDbMask)1)<btreeMask, iDb) ); pDb = &db->aDb[iDb]; pX = pDb->pBt; assert( pX!=0 ); @@ -3298,15 +3302,12 @@ case OP_OpenWrite: { if( pCur==0 ) goto no_mem; pCur->nullRow = 1; pCur->isOrdered = 1; + pCur->pgnoRoot = p2; rc = sqlite3BtreeCursor(pX, p2, wrFlag, pKeyInfo, pCur->pCursor); pCur->pKeyInfo = pKeyInfo; assert( OPFLAG_BULKCSR==BTREE_BULKLOAD ); sqlite3BtreeCursorHints(pCur->pCursor, (pOp->p5 & OPFLAG_BULKCSR)); - /* Since it performs no memory allocation or IO, the only value that - ** sqlite3BtreeCursor() may return is SQLITE_OK. */ - assert( rc==SQLITE_OK ); - /* Set the VdbeCursor.isTable variable. Previous versions of ** SQLite used to check if the root-page flags were sane at this point ** and report database corruption if they were not, but this check has @@ -3390,11 +3391,15 @@ case OP_OpenEphemeral: { break; } -/* Opcode: SorterOpen P1 P2 * P4 * +/* Opcode: SorterOpen P1 P2 P3 P4 * ** ** This opcode works like OP_OpenEphemeral except that it opens ** a transient index that is specifically designed to sort large ** tables using an external merge-sort algorithm. +** +** If argument P3 is non-zero, then it indicates that the sorter may +** assume that a stable sort considering the first P3 fields of each +** key is sufficient to produce the required results. */ case OP_SorterOpen: { VdbeCursor *pCx; @@ -3406,7 +3411,25 @@ case OP_SorterOpen: { pCx->pKeyInfo = pOp->p4.pKeyInfo; assert( pCx->pKeyInfo->db==db ); assert( pCx->pKeyInfo->enc==ENC(db) ); - rc = sqlite3VdbeSorterInit(db, pCx); + rc = sqlite3VdbeSorterInit(db, pOp->p3, pCx); + break; +} + +/* Opcode: SequenceTest P1 P2 * * * +** Synopsis: if( cursor[P1].ctr++ ) pc = P2 +** +** P1 is a sorter cursor. If the sequence counter is currently zero, jump +** to P2. Regardless of whether or not the jump is taken, increment the +** the sequence value. +*/ +case OP_SequenceTest: { + VdbeCursor *pC; + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC->pSorter ); + if( (pC->seqCount++)==0 ){ + pc = pOp->p2 - 1; + } break; } @@ -3452,7 +3475,7 @@ case OP_Close: { break; } -/* Opcode: SeekGe P1 P2 P3 P4 * +/* Opcode: SeekGE P1 P2 P3 P4 * ** Synopsis: key=r[P3@P4] ** ** If cursor P1 refers to an SQL table (B-Tree that uses integer keys), @@ -3464,9 +3487,13 @@ case OP_Close: { ** is greater than or equal to the key value. If there are no records ** greater than or equal to the key and P2 is not zero, then jump to P2. ** +** This opcode leaves the cursor configured to move in forward order, +** from the beginning toward the end. In other words, the cursor is +** configured to use Next, not Prev. +** ** See also: Found, NotFound, SeekLt, SeekGt, SeekLe */ -/* Opcode: SeekGt P1 P2 P3 P4 * +/* Opcode: SeekGT P1 P2 P3 P4 * ** Synopsis: key=r[P3@P4] ** ** If cursor P1 refers to an SQL table (B-Tree that uses integer keys), @@ -3478,9 +3505,13 @@ case OP_Close: { ** is greater than the key value. If there are no records greater than ** the key and P2 is not zero, then jump to P2. ** +** This opcode leaves the cursor configured to move in forward order, +** from the beginning toward the end. In other words, the cursor is +** configured to use Next, not Prev. +** ** See also: Found, NotFound, SeekLt, SeekGe, SeekLe */ -/* Opcode: SeekLt P1 P2 P3 P4 * +/* Opcode: SeekLT P1 P2 P3 P4 * ** Synopsis: key=r[P3@P4] ** ** If cursor P1 refers to an SQL table (B-Tree that uses integer keys), @@ -3492,9 +3523,13 @@ case OP_Close: { ** is less than the key value. If there are no records less than ** the key and P2 is not zero, then jump to P2. ** +** This opcode leaves the cursor configured to move in reverse order, +** from the end toward the beginning. In other words, the cursor is +** configured to use Prev, not Next. +** ** See also: Found, NotFound, SeekGt, SeekGe, SeekLe */ -/* Opcode: SeekLe P1 P2 P3 P4 * +/* Opcode: SeekLE P1 P2 P3 P4 * ** Synopsis: key=r[P3@P4] ** ** If cursor P1 refers to an SQL table (B-Tree that uses integer keys), @@ -3506,6 +3541,10 @@ case OP_Close: { ** is less than or equal to the key value. If there are no records ** less than or equal to the key and P2 is not zero, then jump to P2. ** +** This opcode leaves the cursor configured to move in reverse order, +** from the end toward the beginning. In other words, the cursor is +** configured to use Prev, not Next. +** ** See also: Found, NotFound, SeekGt, SeekGe, SeekLt */ case OP_SeekLT: /* jump, in3 */ @@ -3531,14 +3570,18 @@ case OP_SeekGT: { /* jump, in3 */ assert( pC->pCursor!=0 ); oc = pOp->opcode; pC->nullRow = 0; +#ifdef SQLITE_DEBUG + pC->seekOp = pOp->opcode; +#endif if( pC->isTable ){ /* The input value in P3 might be of any type: integer, real, string, ** blob, or NULL. But it needs to be an integer before we can do - ** the seek, so covert it. */ + ** the seek, so convert it. */ pIn3 = &aMem[pOp->p3]; - applyNumericAffinity(pIn3); + if( (pIn3->flags & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){ + applyNumericAffinity(pIn3, 0); + } iKey = sqlite3VdbeIntValue(pIn3); - pC->rowidIsValid = 0; /* If the P3 value could not be converted into an integer without ** loss of information, then special processing is required... */ @@ -3557,7 +3600,7 @@ case OP_SeekGT: { /* jump, in3 */ ** (x > 4.9) -> (x >= 5) ** (x <= 4.9) -> (x < 5) */ - if( pIn3->r<(double)iKey ){ + if( pIn3->u.r<(double)iKey ){ assert( OP_SeekGE==(OP_SeekGT-1) ); assert( OP_SeekLT==(OP_SeekLE-1) ); assert( (OP_SeekLE & 0x0001)==(OP_SeekGT & 0x0001) ); @@ -3566,7 +3609,7 @@ case OP_SeekGT: { /* jump, in3 */ /* If the approximation iKey is smaller than the actual real search ** term, substitute <= for < and > for >=. */ - else if( pIn3->r>(double)iKey ){ + else if( pIn3->u.r>(double)iKey ){ assert( OP_SeekLE==(OP_SeekLT+1) ); assert( OP_SeekGT==(OP_SeekGE+1) ); assert( (OP_SeekLT & 0x0001)==(OP_SeekGE & 0x0001) ); @@ -3574,13 +3617,10 @@ case OP_SeekGT: { /* jump, in3 */ } } rc = sqlite3BtreeMovetoUnpacked(pC->pCursor, 0, (u64)iKey, 0, &res); + pC->movetoTarget = iKey; /* Used by OP_Delete */ if( rc!=SQLITE_OK ){ goto abort_due_to_error; } - if( res==0 ){ - pC->rowidIsValid = 1; - pC->lastRowid = iKey; - } }else{ nField = pOp->p4.i; assert( pOp->p4type==P4_INT32 ); @@ -3610,7 +3650,6 @@ case OP_SeekGT: { /* jump, in3 */ if( rc!=SQLITE_OK ){ goto abort_due_to_error; } - pC->rowidIsValid = 0; } pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; @@ -3622,7 +3661,6 @@ case OP_SeekGT: { /* jump, in3 */ res = 0; rc = sqlite3BtreeNext(pC->pCursor, &res); if( rc!=SQLITE_OK ) goto abort_due_to_error; - pC->rowidIsValid = 0; }else{ res = 0; } @@ -3632,7 +3670,6 @@ case OP_SeekGT: { /* jump, in3 */ res = 0; rc = sqlite3BtreePrevious(pC->pCursor, &res); if( rc!=SQLITE_OK ) goto abort_due_to_error; - pC->rowidIsValid = 0; }else{ /* res might be negative because the table is empty. Check to ** see if this is the case. @@ -3669,7 +3706,6 @@ case OP_Seek: { /* in2 */ pC->nullRow = 0; pIn2 = &aMem[pOp->p2]; pC->movetoTarget = sqlite3VdbeIntValue(pIn2); - pC->rowidIsValid = 0; pC->deferredMoveto = 1; break; } @@ -3686,6 +3722,10 @@ case OP_Seek: { /* in2 */ ** is a prefix of any entry in P1 then a jump is made to P2 and ** P1 is left pointing at the matching entry. ** +** This operation leaves the cursor in a state where it can be +** advanced in the forward direction. The Next instruction will work, +** but not the Prev instruction. +** ** See also: NotFound, NoConflict, NotExists. SeekGe */ /* Opcode: NotFound P1 P2 P3 P4 * @@ -3701,6 +3741,10 @@ case OP_Seek: { /* in2 */ ** falls through to the next instruction and P1 is left pointing at the ** matching entry. ** +** This operation leaves the cursor in a state where it cannot be +** advanced in either direction. In other words, the Next and Prev +** opcodes do not work after this operation. +** ** See also: Found, NotExists, NoConflict */ /* Opcode: NoConflict P1 P2 P3 P4 * @@ -3720,6 +3764,10 @@ case OP_Seek: { /* in2 */ ** This opcode is similar to OP_NotFound with the exceptions that the ** branch is always taken if any part of the search key input is NULL. ** +** This operation leaves the cursor in a state where it cannot be +** advanced in either direction. In other words, the Next and Prev +** opcodes do not work after this operation. +** ** See also: NotFound, Found, NotExists */ case OP_NoConflict: /* jump, in3 */ @@ -3742,6 +3790,9 @@ case OP_Found: { /* jump, in3 */ assert( pOp->p4type==P4_INT32 ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); +#ifdef SQLITE_DEBUG + pC->seekOp = pOp->opcode; +#endif pIn3 = &aMem[pOp->p3]; assert( pC->pCursor!=0 ); assert( pC->isTable==0 ); @@ -3761,10 +3812,10 @@ case OP_Found: { /* jump, in3 */ }else{ pIdxKey = sqlite3VdbeAllocUnpackedRecord( pC->pKeyInfo, aTempRec, sizeof(aTempRec), &pFree - ); + ); if( pIdxKey==0 ) goto no_mem; assert( pIn3->flags & MEM_Blob ); - assert( (pIn3->flags & MEM_Zero)==0 ); /* zeroblobs already expanded */ + ExpandBlob(pIn3); sqlite3VdbeRecordUnpack(pC->pKeyInfo, pIn3->n, pIn3->z, pIdxKey); } pIdxKey->default_rc = 0; @@ -3813,6 +3864,10 @@ case OP_Found: { /* jump, in3 */ ** The OP_NotFound opcode performs the same operation on index btrees ** (with arbitrary multi-value keys). ** +** This opcode leaves the cursor in a state where it cannot be advanced +** in either direction. In other words, the Next and Prev opcodes will +** not work following this opcode. +** ** See also: Found, NotFound, NoConflict */ case OP_NotExists: { /* jump, in3 */ @@ -3826,6 +3881,9 @@ case OP_NotExists: { /* jump, in3 */ assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); +#ifdef SQLITE_DEBUG + pC->seekOp = 0; +#endif assert( pC->isTable ); assert( pC->pseudoTableReg==0 ); pCrsr = pC->pCursor; @@ -3833,15 +3891,13 @@ case OP_NotExists: { /* jump, in3 */ res = 0; iKey = pIn3->u.i; rc = sqlite3BtreeMovetoUnpacked(pCrsr, 0, iKey, 0, &res); - pC->lastRowid = pIn3->u.i; - pC->rowidIsValid = res==0 ?1:0; + pC->movetoTarget = iKey; /* Used by OP_Delete */ pC->nullRow = 0; pC->cacheStatus = CACHE_STALE; pC->deferredMoveto = 0; VdbeBranchTaken(res!=0,2); if( res!=0 ){ pc = pOp->p2 - 1; - assert( pC->rowidIsValid==0 ); } pC->seekResult = res; break; @@ -3975,32 +4031,20 @@ case OP_NewRowid: { /* out2-prerelease */ ** it finds one that is not previously used. */ assert( pOp->p3==0 ); /* We cannot be in random rowid mode if this is ** an AUTOINCREMENT table. */ - /* on the first attempt, simply do one more than previous */ - v = lastRowid; - v &= (MAX_ROWID>>1); /* ensure doesn't go negative */ - v++; /* ensure non-zero */ cnt = 0; - while( ((rc = sqlite3BtreeMovetoUnpacked(pC->pCursor, 0, (u64)v, + do{ + sqlite3_randomness(sizeof(v), &v); + v &= (MAX_ROWID>>1); v++; /* Ensure that v is greater than zero */ + }while( ((rc = sqlite3BtreeMovetoUnpacked(pC->pCursor, 0, (u64)v, 0, &res))==SQLITE_OK) && (res==0) - && (++cnt<100)){ - /* collision - try another random rowid */ - sqlite3_randomness(sizeof(v), &v); - if( cnt<5 ){ - /* try "small" random rowids for the initial attempts */ - v &= 0xffffff; - }else{ - v &= (MAX_ROWID>>1); /* ensure doesn't go negative */ - } - v++; /* ensure non-zero */ - } + && (++cnt<100)); if( rc==SQLITE_OK && res==0 ){ rc = SQLITE_FULL; /* IMP: R-38219-53002 */ goto abort_due_to_error; } assert( v>0 ); /* EV: R-40812-03570 */ } - pC->rowidIsValid = 0; pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; } @@ -4105,7 +4149,6 @@ case OP_InsertInt: { pData->z, pData->n, nZero, (pOp->p5 & OPFLAG_APPEND)!=0, seekResult ); - pC->rowidIsValid = 0; pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; @@ -4128,7 +4171,7 @@ case OP_InsertInt: { ** The cursor will be left pointing at either the next or the previous ** record in the table. If it is left pointing at the next record, then ** the next Next instruction will be a no-op. Hence it is OK to delete -** a record from within an Next loop. +** a record from within a Next loop. ** ** If the OPFLAG_NCHANGE flag of P2 is set, then the row change count is ** incremented (otherwise not). @@ -4142,33 +4185,32 @@ case OP_InsertInt: { ** using OP_NotFound prior to invoking this opcode. */ case OP_Delete: { - i64 iKey; VdbeCursor *pC; assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); assert( pC->pCursor!=0 ); /* Only valid for real tables, no pseudotables */ - iKey = pC->lastRowid; /* Only used for the update hook */ - - /* The OP_Delete opcode always follows an OP_NotExists or OP_Last or - ** OP_Column on the same table without any intervening operations that - ** might move or invalidate the cursor. Hence cursor pC is always pointing - ** to the row to be deleted and the sqlite3VdbeCursorMoveto() operation - ** below is always a no-op and cannot fail. We will run it anyhow, though, - ** to guard against future changes to the code generator. - **/ assert( pC->deferredMoveto==0 ); - rc = sqlite3VdbeCursorMoveto(pC); - if( NEVER(rc!=SQLITE_OK) ) goto abort_due_to_error; +#ifdef SQLITE_DEBUG + /* The seek operation that positioned the cursor prior to OP_Delete will + ** have also set the pC->movetoTarget field to the rowid of the row that + ** is being deleted */ + if( pOp->p4.z && pC->isTable ){ + i64 iKey = 0; + sqlite3BtreeKeySize(pC->pCursor, &iKey); + assert( pC->movetoTarget==iKey ); + } +#endif + rc = sqlite3BtreeDelete(pC->pCursor); pC->cacheStatus = CACHE_STALE; /* Invoke the update-hook if required. */ if( rc==SQLITE_OK && db->xUpdateCallback && pOp->p4.z && pC->isTable ){ db->xUpdateCallback(db->pUpdateArg, SQLITE_DELETE, - db->aDb[pC->iDb].zName, pOp->p4.z, iKey); + db->aDb[pC->iDb].zName, pOp->p4.z, pC->movetoTarget); assert( pC->iDb>=0 ); } if( pOp->p2 & OPFLAG_NCHANGE ) p->nChange++; @@ -4188,12 +4230,12 @@ case OP_ResetCount: { } /* Opcode: SorterCompare P1 P2 P3 P4 -** Synopsis: if key(P1)!=rtrim(r[P3],P4) goto P2 +** Synopsis: if key(P1)!=trim(r[P3],P4) goto P2 ** ** P1 is a sorter cursor. This instruction compares a prefix of the -** the record blob in register P3 against a prefix of the entry that -** the sorter cursor currently points to. The final P4 fields of both -** the P3 and sorter record are ignored. +** record blob in register P3 against a prefix of the entry that +** the sorter cursor currently points to. Only the first P4 fields +** of r[P3] and the sorter record are compared. ** ** If either P3 or the sorter contains a NULL in one of their significant ** fields (not counting the P4 fields at the end which are ignored) then @@ -4205,14 +4247,15 @@ case OP_ResetCount: { case OP_SorterCompare: { VdbeCursor *pC; int res; - int nIgnore; + int nKeyCol; pC = p->apCsr[pOp->p1]; assert( isSorter(pC) ); assert( pOp->p4type==P4_INT32 ); pIn3 = &aMem[pOp->p3]; - nIgnore = pOp->p4.i; - rc = sqlite3VdbeSorterCompare(pC, pIn3, nIgnore, &res); + nKeyCol = pOp->p4.i; + res = 0; + rc = sqlite3VdbeSorterCompare(pC, pIn3, nKeyCol, &res); VdbeBranchTaken(res!=0,2); if( res ){ pc = pOp->p2-1; @@ -4220,10 +4263,17 @@ case OP_SorterCompare: { break; }; -/* Opcode: SorterData P1 P2 * * * +/* Opcode: SorterData P1 P2 P3 * * ** Synopsis: r[P2]=data ** ** Write into register P2 the current sorter data for sorter cursor P1. +** Then clear the column header cache on cursor P3. +** +** This opcode is normally use to move a record out of the sorter and into +** a register that is the source for a pseudo-table cursor created using +** OpenPseudo. That pseudo-table cursor is the one that is identified by +** parameter P3. Clearing the P3 column cache as part of this opcode saves +** us from having to issue a separate NullRow instruction to clear that cache. */ case OP_SorterData: { VdbeCursor *pC; @@ -4233,6 +4283,8 @@ case OP_SorterData: { assert( isSorter(pC) ); rc = sqlite3VdbeSorterRowkey(pC, pOut); assert( rc!=SQLITE_OK || (pOut->flags & MEM_Blob) ); + assert( pOp->p1>=0 && pOp->p1nCursor ); + p->apCsr[pOp->p3]->cacheStatus = CACHE_STALE; break; } @@ -4279,16 +4331,20 @@ case OP_RowData: { assert( pC->pseudoTableReg==0 ); assert( pC->pCursor!=0 ); pCrsr = pC->pCursor; - assert( sqlite3BtreeCursorIsValid(pCrsr) ); /* The OP_RowKey and OP_RowData opcodes always follow OP_NotExists or ** OP_Rewind/Op_Next with no intervening instructions that might invalidate - ** the cursor. Hence the following sqlite3VdbeCursorMoveto() call is always - ** a no-op and can never fail. But we leave it in place as a safety. + ** the cursor. If this where not the case, on of the following assert()s + ** would fail. Should this ever change (because of changes in the code + ** generator) then the fix would be to insert a call to + ** sqlite3VdbeCursorMoveto(). */ assert( pC->deferredMoveto==0 ); + assert( sqlite3BtreeCursorIsValid(pCrsr) ); +#if 0 /* Not required due to the previous to assert() statements */ rc = sqlite3VdbeCursorMoveto(pC); - if( NEVER(rc!=SQLITE_OK) ) goto abort_due_to_error; + if( rc!=SQLITE_OK ) goto abort_due_to_error; +#endif if( pC->isTable==0 ){ assert( !pC->isTable ); @@ -4305,7 +4361,8 @@ case OP_RowData: { goto too_big; } } - if( sqlite3VdbeMemGrow(pOut, n, 0) ){ + testcase( n==0 ); + if( sqlite3VdbeMemClearAndResize(pOut, MAX(n,32)) ){ goto no_mem; } pOut->n = n; @@ -4356,14 +4413,14 @@ case OP_Rowid: { /* out2-prerelease */ #endif /* SQLITE_OMIT_VIRTUALTABLE */ }else{ assert( pC->pCursor!=0 ); - rc = sqlite3VdbeCursorMoveto(pC); + rc = sqlite3VdbeCursorRestore(pC); if( rc ) goto abort_due_to_error; - if( pC->rowidIsValid ){ - v = pC->lastRowid; - }else{ - rc = sqlite3BtreeKeySize(pC->pCursor, &v); - assert( rc==SQLITE_OK ); /* Always so because of CursorMoveto() above */ + if( pC->nullRow ){ + pOut->flags = MEM_Null; + break; } + rc = sqlite3BtreeKeySize(pC->pCursor, &v); + assert( rc==SQLITE_OK ); /* Always so because of CursorRestore() above */ } pOut->u.i = v; break; @@ -4382,7 +4439,6 @@ case OP_NullRow: { pC = p->apCsr[pOp->p1]; assert( pC!=0 ); pC->nullRow = 1; - pC->rowidIsValid = 0; pC->cacheStatus = CACHE_STALE; if( pC->pCursor ){ sqlite3BtreeClearCursor(pC->pCursor); @@ -4392,11 +4448,15 @@ case OP_NullRow: { /* Opcode: Last P1 P2 * * * ** -** The next use of the Rowid or Column or Next instruction for P1 +** The next use of the Rowid or Column or Prev instruction for P1 ** will refer to the last entry in the database table or index. ** If the table or index is empty and P2>0, then jump immediately to P2. ** If P2 is 0 or if the table or index is not empty, fall through ** to the following instruction. +** +** This opcode leaves the cursor configured to move in reverse order, +** from the end toward the beginning. In other words, the cursor is +** configured to use Prev, not Next. */ case OP_Last: { /* jump */ VdbeCursor *pC; @@ -4412,8 +4472,10 @@ case OP_Last: { /* jump */ rc = sqlite3BtreeLast(pCrsr, &res); pC->nullRow = (u8)res; pC->deferredMoveto = 0; - pC->rowidIsValid = 0; pC->cacheStatus = CACHE_STALE; +#ifdef SQLITE_DEBUG + pC->seekOp = OP_Last; +#endif if( pOp->p2>0 ){ VdbeBranchTaken(res!=0,2); if( res ) pc = pOp->p2 - 1; @@ -4447,9 +4509,13 @@ case OP_Sort: { /* jump */ ** ** The next use of the Rowid or Column or Next instruction for P1 ** will refer to the first entry in the database table or index. -** If the table or index is empty and P2>0, then jump immediately to P2. -** If P2 is 0 or if the table or index is not empty, fall through -** to the following instruction. +** If the table or index is empty, jump immediately to P2. +** If the table or index is not empty, fall through to the following +** instruction. +** +** This opcode leaves the cursor configured to move in forward order, +** from the beginning toward the end. In other words, the cursor is +** configured to use Next, not Prev. */ case OP_Rewind: { /* jump */ VdbeCursor *pC; @@ -4461,15 +4527,17 @@ case OP_Rewind: { /* jump */ assert( pC!=0 ); assert( isSorter(pC)==(pOp->opcode==OP_SorterSort) ); res = 1; +#ifdef SQLITE_DEBUG + pC->seekOp = OP_Rewind; +#endif if( isSorter(pC) ){ - rc = sqlite3VdbeSorterRewind(db, pC, &res); + rc = sqlite3VdbeSorterRewind(pC, &res); }else{ pCrsr = pC->pCursor; assert( pCrsr ); rc = sqlite3BtreeFirst(pCrsr, &res); pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; - pC->rowidIsValid = 0; } pC->nullRow = (u8)res; assert( pOp->p2>0 && pOp->p2nOp ); @@ -4487,6 +4555,10 @@ case OP_Rewind: { /* jump */ ** to the following instruction. But if the cursor advance was successful, ** jump immediately to P2. ** +** The Next opcode is only valid following an SeekGT, SeekGE, or +** OP_Rewind opcode used to position the cursor. Next is not allowed +** to follow SeekLT, SeekLE, or OP_Last. +** ** The P1 cursor must be for a real table, not a pseudo-table. P1 must have ** been opened prior to this opcode or the program will segfault. ** @@ -4505,7 +4577,7 @@ case OP_Rewind: { /* jump */ */ /* Opcode: NextIfOpen P1 P2 P3 P4 P5 ** -** This opcode works just like OP_Next except that if cursor P1 is not +** This opcode works just like Next except that if cursor P1 is not ** open it behaves a no-op. */ /* Opcode: Prev P1 P2 P3 P4 P5 @@ -4515,6 +4587,11 @@ case OP_Rewind: { /* jump */ ** to the following instruction. But if the cursor backup was successful, ** jump immediately to P2. ** +** +** The Prev opcode is only valid following an SeekLT, SeekLE, or +** OP_Last opcode used to position the cursor. Prev is not allowed +** to follow SeekGT, SeekGE, or OP_Rewind. +** ** The P1 cursor must be for a real table, not a pseudo-table. If P1 is ** not open then the behavior is undefined. ** @@ -4531,7 +4608,7 @@ case OP_Rewind: { /* jump */ */ /* Opcode: PrevIfOpen P1 P2 P3 P4 P5 ** -** This opcode works just like OP_Prev except that if cursor P1 is not +** This opcode works just like Prev except that if cursor P1 is not ** open it behaves a no-op. */ case OP_SorterNext: { /* jump */ @@ -4562,6 +4639,16 @@ case OP_Next: /* jump */ assert( pOp->opcode!=OP_Prev || pOp->p4.xAdvance==sqlite3BtreePrevious ); assert( pOp->opcode!=OP_NextIfOpen || pOp->p4.xAdvance==sqlite3BtreeNext ); assert( pOp->opcode!=OP_PrevIfOpen || pOp->p4.xAdvance==sqlite3BtreePrevious); + + /* The Next opcode is only used after SeekGT, SeekGE, and Rewind. + ** The Prev opcode is only used after SeekLT, SeekLE, and Last. */ + assert( pOp->opcode!=OP_Next || pOp->opcode!=OP_NextIfOpen + || pC->seekOp==OP_SeekGT || pC->seekOp==OP_SeekGE + || pC->seekOp==OP_Rewind || pC->seekOp==OP_Found); + assert( pOp->opcode!=OP_Prev || pOp->opcode!=OP_PrevIfOpen + || pC->seekOp==OP_SeekLT || pC->seekOp==OP_SeekLE + || pC->seekOp==OP_Last ); + rc = pOp->p4.xAdvance(pC->pCursor, &res); next_tail: pC->cacheStatus = CACHE_STALE; @@ -4576,7 +4663,6 @@ next_tail: }else{ pC->nullRow = 1; } - pC->rowidIsValid = 0; goto check_for_interrupt; } @@ -4621,7 +4707,7 @@ case OP_IdxInsert: { /* in2 */ rc = ExpandBlob(pIn2); if( rc==SQLITE_OK ){ if( isSorter(pC) ){ - rc = sqlite3VdbeSorterWrite(db, pC, pIn2); + rc = sqlite3VdbeSorterWrite(pC, pIn2); }else{ nKey = pIn2->n; zKey = pIn2->z; @@ -4692,10 +4778,16 @@ case OP_IdxRowid: { /* out2-prerelease */ pCrsr = pC->pCursor; assert( pCrsr!=0 ); pOut->flags = MEM_Null; - rc = sqlite3VdbeCursorMoveto(pC); - if( NEVER(rc) ) goto abort_due_to_error; - assert( pC->deferredMoveto==0 ); assert( pC->isTable==0 ); + assert( pC->deferredMoveto==0 ); + + /* sqlite3VbeCursorRestore() can only fail if the record has been deleted + ** out from under the cursor. That will never happend for an IdxRowid + ** opcode, hence the NEVER() arround the check of the return value. + */ + rc = sqlite3VdbeCursorRestore(pC); + if( NEVER(rc!=SQLITE_OK) ) goto abort_due_to_error; + if( !pC->nullRow ){ rowid = 0; /* Not needed. Only used to silence a warning. */ rc = sqlite3VdbeIdxRowid(db, pCrsr, &rowid); @@ -4782,7 +4874,7 @@ case OP_IdxGE: { /* jump */ { int i; for(i=0; iopcode&1)==(OP_IdxLT&1) ){ assert( pOp->opcode==OP_IdxLE || pOp->opcode==OP_IdxLT ); @@ -4844,7 +4936,7 @@ case OP_Destroy: { /* out2-prerelease */ }else{ iDb = pOp->p3; assert( iCnt==1 ); - assert( (p->btreeMask & (((yDbMask)1)<btreeMask, iDb) ); iMoved = 0; /* Not needed. Only to silence a warning. */ rc = sqlite3BtreeDropTable(db->aDb[iDb].pBt, pOp->p1, &iMoved); pOut->flags = MEM_Int; @@ -4884,7 +4976,7 @@ case OP_Clear: { nChange = 0; assert( p->readOnly==0 ); - assert( (p->btreeMask & (((yDbMask)1)<p2))!=0 ); + assert( DbMaskTest(p->btreeMask, pOp->p2) ); rc = sqlite3BtreeClearTable( db->aDb[pOp->p2].pBt, pOp->p1, (pOp->p3 ? &nChange : 0) ); @@ -4954,7 +5046,7 @@ case OP_CreateTable: { /* out2-prerelease */ pgno = 0; assert( pOp->p1>=0 && pOp->p1nDb ); - assert( (p->btreeMask & (((yDbMask)1)<p1))!=0 ); + assert( DbMaskTest(p->btreeMask, pOp->p1) ); assert( p->readOnly==0 ); pDb = &db->aDb[pOp->p1]; assert( pDb->pBt!=0 ); @@ -5042,7 +5134,8 @@ case OP_LoadAnalysis: { ** ** Remove the internal (in-memory) data structures that describe ** the table named P4 in database P1. This is called after a table -** is dropped in order to keep the internal representation of the +** is dropped from disk (using the Destroy opcode) in order to keep +** the internal representation of the ** schema consistent with what is on disk. */ case OP_DropTable: { @@ -5054,7 +5147,8 @@ case OP_DropTable: { ** ** Remove the internal (in-memory) data structures that describe ** the index named P4 in database P1. This is called after an index -** is dropped in order to keep the internal representation of the +** is dropped from disk (using the Destroy opcode) +** in order to keep the internal representation of the ** schema consistent with what is on disk. */ case OP_DropIndex: { @@ -5066,7 +5160,8 @@ case OP_DropIndex: { ** ** Remove the internal (in-memory) data structures that describe ** the trigger named P4 in database P1. This is called after a trigger -** is dropped in order to keep the internal representation of the +** is dropped from disk (using the Destroy opcode) in order to keep +** the internal representation of the ** schema consistent with what is on disk. */ case OP_DropTrigger: { @@ -5119,7 +5214,7 @@ case OP_IntegrityCk: { } aRoot[j] = 0; assert( pOp->p5nDb ); - assert( (p->btreeMask & (((yDbMask)1)<p5))!=0 ); + assert( DbMaskTest(p->btreeMask, pOp->p5) ); z = sqlite3BtreeIntegrityCheck(db->aDb[pOp->p5].pBt, aRoot, nRoot, (int)pnErr->u.i, &nErr); sqlite3DbFree(db, aRoot); @@ -5332,6 +5427,9 @@ case OP_Program: { /* jump */ pFrame->token = pProgram->token; pFrame->aOnceFlag = p->aOnceFlag; pFrame->nOnceFlag = p->nOnceFlag; +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + pFrame->anExec = p->anExec; +#endif pEnd = &VdbeFrameMem(pFrame)[pFrame->nChildMem]; for(pMem=VdbeFrameMem(pFrame); pMem!=pEnd; pMem++){ @@ -5349,6 +5447,7 @@ case OP_Program: { /* jump */ pFrame->pParent = p->pFrame; pFrame->lastRowid = lastRowid; pFrame->nChange = p->nChange; + pFrame->nDbChange = p->db->nChange; p->nChange = 0; p->pFrame = pFrame; p->aMem = aMem = &VdbeFrameMem(pFrame)[-1]; @@ -5359,6 +5458,9 @@ case OP_Program: { /* jump */ p->nOp = pProgram->nOp; p->aOnceFlag = (u8 *)&p->apCsr[p->nCursor]; p->nOnceFlag = pProgram->nOnce; +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + p->anExec = 0; +#endif pc = -1; memset(p->aOnceFlag, 0, p->nOnceFlag); @@ -5481,17 +5583,16 @@ case OP_IfPos: { /* jump, in1 */ break; } -/* Opcode: IfNeg P1 P2 * * * -** Synopsis: if r[P1]<0 goto P2 +/* Opcode: IfNeg P1 P2 P3 * * +** Synopsis: r[P1]+=P3, if r[P1]<0 goto P2 ** -** If the value of register P1 is less than zero, jump to P2. -** -** It is illegal to use this instruction on a register that does -** not contain an integer. An assertion fault will result if you try. +** Register P1 must contain an integer. Add literal P3 to the value in +** register P1 then if the value of register P1 is less than zero, jump to P2. */ case OP_IfNeg: { /* jump, in1 */ pIn1 = &aMem[pOp->p1]; assert( pIn1->flags&MEM_Int ); + pIn1->u.i += pOp->p3; VdbeBranchTaken(pIn1->u.i<0, 2); if( pIn1->u.i<0 ){ pc = pOp->p2 - 1; @@ -5504,9 +5605,6 @@ case OP_IfNeg: { /* jump, in1 */ ** ** The register P1 must contain an integer. Add literal P3 to the ** value in register P1. If the result is exactly 0, jump to P2. -** -** It is illegal to use this instruction on a register that does -** not contain an integer. An assertion fault will result if you try. */ case OP_IfZero: { /* jump, in1 */ pIn1 = &aMem[pOp->p1]; @@ -5535,6 +5633,7 @@ case OP_AggStep: { int i; Mem *pMem; Mem *pRec; + Mem t; sqlite3_context ctx; sqlite3_value **apVal; @@ -5552,23 +5651,15 @@ case OP_AggStep: { assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); ctx.pMem = pMem = &aMem[pOp->p3]; pMem->n++; - ctx.s.flags = MEM_Null; - ctx.s.z = 0; - ctx.s.zMalloc = 0; - ctx.s.xDel = 0; - ctx.s.db = db; + sqlite3VdbeMemInit(&t, db, MEM_Null); + ctx.pOut = &t; ctx.isError = 0; - ctx.pColl = 0; + ctx.pVdbe = p; + ctx.iOp = pc; ctx.skipFlag = 0; - if( ctx.pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){ - assert( pOp>p->aOp ); - assert( pOp[-1].p4type==P4_COLLSEQ ); - assert( pOp[-1].opcode==OP_CollSeq ); - ctx.pColl = pOp[-1].p4.pColl; - } (ctx.pFunc->xStep)(&ctx, n, apVal); /* IMP: R-24505-23230 */ if( ctx.isError ){ - sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&ctx.s)); + sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&t)); rc = ctx.isError; } if( ctx.skipFlag ){ @@ -5576,9 +5667,7 @@ case OP_AggStep: { i = pOp[-1].p1; if( i ) sqlite3VdbeMemSetInt64(&aMem[i], 1); } - - sqlite3VdbeMemRelease(&ctx.s); - + sqlite3VdbeMemRelease(&t); break; } @@ -5616,8 +5705,8 @@ case OP_AggFinal: { /* Opcode: Checkpoint P1 P2 P3 * * ** ** Checkpoint database P1. This is a no-op if P1 is not currently in -** WAL mode. Parameter P2 is one of SQLITE_CHECKPOINT_PASSIVE, FULL -** or RESTART. Write 1 or 0 into mem[P3] if the checkpoint returns +** WAL mode. Parameter P2 is one of SQLITE_CHECKPOINT_PASSIVE, FULL, +** RESTART, or TRUNCATE. Write 1 or 0 into mem[P3] if the checkpoint returns ** SQLITE_BUSY or not, respectively. Write the number of pages in the ** WAL after the checkpoint into mem[P3+1] and the number of pages ** in the WAL that have been checkpointed after the checkpoint @@ -5635,6 +5724,7 @@ case OP_Checkpoint: { assert( pOp->p2==SQLITE_CHECKPOINT_PASSIVE || pOp->p2==SQLITE_CHECKPOINT_FULL || pOp->p2==SQLITE_CHECKPOINT_RESTART + || pOp->p2==SQLITE_CHECKPOINT_TRUNCATE ); rc = sqlite3Checkpoint(db, pOp->p1, pOp->p2, &aRes[1], &aRes[2]); if( rc==SQLITE_BUSY ){ @@ -5779,7 +5869,7 @@ case OP_IncrVacuum: { /* jump */ Btree *pBt; assert( pOp->p1>=0 && pOp->p1nDb ); - assert( (p->btreeMask & (((yDbMask)1)<p1))!=0 ); + assert( DbMaskTest(p->btreeMask, pOp->p1) ); assert( p->readOnly==0 ); pBt = db->aDb[pOp->p1].pBt; rc = sqlite3BtreeIncrVacuum(pBt); @@ -5794,12 +5884,13 @@ case OP_IncrVacuum: { /* jump */ /* Opcode: Expire P1 * * * * ** -** Cause precompiled statements to become expired. An expired statement -** fails with an error code of SQLITE_SCHEMA if it is ever executed -** (via sqlite3_step()). +** Cause precompiled statements to expire. When an expired statement +** is executed using sqlite3_step() it will either automatically +** reprepare itself (if it was originally created using sqlite3_prepare_v2()) +** or it will fail with SQLITE_SCHEMA. ** ** If P1 is 0, then all SQL statements become expired. If P1 is non-zero, -** then only the currently executing statement is affected. +** then only the currently executing statement is expired. */ case OP_Expire: { if( !pOp->p1 ){ @@ -5831,7 +5922,7 @@ case OP_TableLock: { if( isWriteLock || 0==(db->flags&SQLITE_ReadUncommitted) ){ int p1 = pOp->p1; assert( p1>=0 && p1nDb ); - assert( (p->btreeMask & (((yDbMask)1)<btreeMask, p1) ); assert( isWriteLock==0 || isWriteLock==1 ); rc = sqlite3BtreeLockTable(db->aDb[p1].pBt, pOp->p2, isWriteLock); if( (rc&0xFF)==SQLITE_LOCKED ){ @@ -6027,27 +6118,14 @@ case OP_VColumn: { pModule = pVtab->pModule; assert( pModule->xColumn ); memset(&sContext, 0, sizeof(sContext)); - - /* The output cell may already have a buffer allocated. Move - ** the current contents to sContext.s so in case the user-function - ** can use the already allocated buffer instead of allocating a - ** new one. - */ - sqlite3VdbeMemMove(&sContext.s, pDest); - MemSetTypeFlag(&sContext.s, MEM_Null); - + sContext.pOut = pDest; + MemSetTypeFlag(pDest, MEM_Null); rc = pModule->xColumn(pCur->pVtabCursor, &sContext, pOp->p2); sqlite3VtabImportErrmsg(p, pVtab); if( sContext.isError ){ rc = sContext.isError; } - - /* Copy the result of the function to the P3 register. We - ** do this regardless of whether or not an error occurred to ensure any - ** dynamic allocation in sContext.s (a Mem struct) is released. - */ - sqlite3VdbeChangeEncoding(&sContext.s, encoding); - sqlite3VdbeMemMove(pDest, &sContext.s); + sqlite3VdbeChangeEncoding(pDest, encoding); REGISTER_TRACE(pOp->p3, pDest); UPDATE_MAX_BLOBSIZE(pDest); @@ -6281,7 +6359,7 @@ case OP_Init: { /* jump */ if( zTrace ){ int i; for(i=0; inDb; i++){ - if( (MASKBIT(i) & p->btreeMask)==0 ) continue; + if( DbMaskTest(p->btreeMask, i)==0 ) continue; sqlite3_file_control(db, db->aDb[i].zName, SQLITE_FCNTL_TRACE, zTrace); } } diff --git a/src/vdbe.h b/src/vdbe.h index 10a4140ee1..b715241b41 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -209,12 +209,13 @@ void sqlite3VdbeSetVarmask(Vdbe*, int); #ifndef SQLITE_OMIT_TRACE char *sqlite3VdbeExpandSql(Vdbe*, const char*); #endif +int sqlite3MemCompare(const Mem*, const Mem*, const CollSeq*); void sqlite3VdbeRecordUnpack(KeyInfo*,int,const void*,UnpackedRecord*); -int sqlite3VdbeRecordCompare(int,const void*,UnpackedRecord*,int); +int sqlite3VdbeRecordCompare(int,const void*,UnpackedRecord*); UnpackedRecord *sqlite3VdbeAllocUnpackedRecord(KeyInfo *, char *, int, char **); -typedef int (*RecordCompare)(int,const void*,UnpackedRecord*,int); +typedef int (*RecordCompare)(int,const void*,UnpackedRecord*); RecordCompare sqlite3VdbeFindCompare(UnpackedRecord*); #ifndef SQLITE_OMIT_TRIGGER @@ -281,4 +282,10 @@ void sqlite3VdbeLinkSubProgram(Vdbe *, SubProgram *); # define VDBE_OFFSET_LINENO(x) 0 #endif +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +void sqlite3VdbeScanStatus(Vdbe*, int, int, int, LogEst, const char*); +#else +# define sqlite3VdbeScanStatus(a,b,c,d,e) +#endif + #endif diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 043b56da59..1a7297e946 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -68,18 +68,20 @@ struct VdbeCursor { int pseudoTableReg; /* Register holding pseudotable content. */ i16 nField; /* Number of fields in the header */ u16 nHdrParsed; /* Number of header fields parsed so far */ +#ifdef SQLITE_DEBUG + u8 seekOp; /* Most recent seek operation on this cursor */ +#endif i8 iDb; /* Index of cursor database in db->aDb[] (or -1) */ u8 nullRow; /* True if pointing to a row with no data */ - u8 rowidIsValid; /* True if lastRowid is valid */ u8 deferredMoveto; /* A call to sqlite3BtreeMoveto() is needed */ Bool isEphemeral:1; /* True for an ephemeral table */ Bool useRandomRowid:1;/* Generate new record numbers semi-randomly */ Bool isTable:1; /* True if a table requiring integer keys */ Bool isOrdered:1; /* True if the underlying table is BTREE_UNORDERED */ + Pgno pgnoRoot; /* Root page of the open btree cursor */ sqlite3_vtab_cursor *pVtabCursor; /* The cursor for a virtual table */ i64 seqCount; /* Sequence counter */ i64 movetoTarget; /* Argument to the deferred sqlite3BtreeMoveto() */ - i64 lastRowid; /* Rowid being deleted by OP_Delete */ VdbeSorter *pSorter; /* Sorter object for OP_SorterOpen cursors */ /* Cached information about the header for the data record that the @@ -96,6 +98,7 @@ struct VdbeCursor { u32 szRow; /* Byte available in aRow */ u32 iHdrOffset; /* Offset to next unparsed byte of the header */ const u8 *aRow; /* Data for the current row, if all on one page */ + u32 *aOffset; /* Pointer to aType[nField] */ u32 aType[1]; /* Type values for all entries in the record */ /* 2*nField extra array elements allocated for aType[], beyond the one ** static element declared in the structure. nField total array slots for @@ -129,6 +132,7 @@ struct VdbeFrame { Vdbe *v; /* VM this frame belongs to */ VdbeFrame *pParent; /* Parent of this frame, or NULL if parent is main */ Op *aOp; /* Program instructions for parent frame */ + i64 *anExec; /* Event counters from parent frame */ Mem *aMem; /* Array of memory cells for parent frame */ u8 *aOnceFlag; /* Array of OP_Once flags for parent frame */ VdbeCursor **apCsr; /* Array of Vdbe cursors for parent frame */ @@ -141,7 +145,8 @@ struct VdbeFrame { int nOnceFlag; /* Number of entries in aOnceFlag */ int nChildMem; /* Number of memory cells for child frame */ int nChildCsr; /* Number of cursors for child frame */ - int nChange; /* Statement changes (Vdbe.nChanges) */ + int nChange; /* Statement changes (Vdbe.nChange) */ + int nDbChange; /* Value of db->nChange */ }; #define VdbeFrameMem(p) ((Mem *)&((u8 *)p)[ROUND8(sizeof(VdbeFrame))]) @@ -157,25 +162,28 @@ struct VdbeFrame { ** integer etc.) of the same value. */ struct Mem { - sqlite3 *db; /* The associated database connection */ - char *z; /* String or BLOB value */ - double r; /* Real value */ - union { + union MemValue { + double r; /* Real value used when MEM_Real is set in flags */ i64 i; /* Integer value used when MEM_Int is set in flags */ int nZero; /* Used when bit MEM_Zero is set in flags */ FuncDef *pDef; /* Used only when flags==MEM_Agg */ RowSet *pRowSet; /* Used only when flags==MEM_RowSet */ VdbeFrame *pFrame; /* Used when flags==MEM_Frame */ } u; - int n; /* Number of characters in string value, excluding '\0' */ u16 flags; /* Some combination of MEM_Null, MEM_Str, MEM_Dyn, etc. */ u8 enc; /* SQLITE_UTF8, SQLITE_UTF16BE, SQLITE_UTF16LE */ + int n; /* Number of characters in string value, excluding '\0' */ + char *z; /* String or BLOB value */ + /* ShallowCopy only needs to copy the information above */ + char *zMalloc; /* Space to hold MEM_Str or MEM_Blob if szMalloc>0 */ + int szMalloc; /* Size of the zMalloc allocation */ + u32 uTemp; /* Transient storage for serial_type in OP_MakeRecord */ + sqlite3 *db; /* The associated database connection */ + void (*xDel)(void*);/* Destructor for Mem.z - only valid if MEM_Dyn */ #ifdef SQLITE_DEBUG Mem *pScopyFrom; /* This Mem is a shallow copy of pScopyFrom */ void *pFiller; /* So that sizeof(Mem) is a multiple of 8 */ #endif - void (*xDel)(void *); /* If not null, call this function to delete Mem.z */ - char *zMalloc; /* Dynamic buffer allocated by sqlite3_malloc() */ }; /* One or more of the following flags are set to indicate the validOK @@ -234,7 +242,7 @@ struct Mem { #endif /* -** Each auxilliary data pointer stored by a user defined function +** Each auxiliary data pointer stored by a user defined function ** implementation calling sqlite3_set_auxdata() is stored in an instance ** of this structure. All such structures associated with a single VM ** are stored in a linked list headed at Vdbe.pAuxData. All are destroyed @@ -249,7 +257,7 @@ struct AuxData { }; /* -** The "context" argument for a installable function. A pointer to an +** The "context" argument for an installable function. A pointer to an ** instance of this structure is the first argument to the routines used ** implement the SQL functions. ** @@ -262,14 +270,13 @@ struct AuxData { ** (Mem) which are only defined there. */ struct sqlite3_context { - FuncDef *pFunc; /* Pointer to function information. MUST BE FIRST */ - Mem s; /* The return value is stored here */ + Mem *pOut; /* The return value is stored here */ + FuncDef *pFunc; /* Pointer to function information */ Mem *pMem; /* Memory cell used to store aggregate context */ - CollSeq *pColl; /* Collating sequence */ Vdbe *pVdbe; /* The VM that owns this context */ int iOp; /* Instruction number of OP_Function */ int isError; /* Error code returned by the function. */ - u8 skipFlag; /* Skip skip accumulator loading if true */ + u8 skipFlag; /* Skip accumulator loading if true */ u8 fErrorOrAux; /* isError!=0 or pVdbe->pAuxData modified */ }; @@ -290,6 +297,16 @@ struct Explain { */ typedef unsigned bft; /* Bit Field Type */ +typedef struct ScanStatus ScanStatus; +struct ScanStatus { + int addrExplain; /* OP_Explain for loop */ + int addrLoop; /* Address of "loops" counter */ + int addrVisit; /* Address of "rows visited" counter */ + int iSelectID; /* The "Select-ID" for this loop */ + LogEst nEst; /* Estimated output rows per loop */ + char *zName; /* Name of table or index */ +}; + /* ** An instance of the virtual machine. This structure contains the complete ** state of the virtual machine. @@ -354,10 +371,6 @@ struct Vdbe { i64 nStmtDefImmCons; /* Number of def. imm constraints when stmt started */ char *zSql; /* Text of the SQL statement that generated this */ void *pFree; /* Free this when deleting the vdbe */ -#ifdef SQLITE_ENABLE_TREE_EXPLAIN - Explain *pExplain; /* The explainer */ - char *zExplain; /* Explanation of data structures */ -#endif VdbeFrame *pFrame; /* Parent frame */ VdbeFrame *pDelFrame; /* List of frame objects to free on VM reset */ int nFrame; /* Number of frames in pFrame list */ @@ -366,6 +379,11 @@ struct Vdbe { int nOnceFlag; /* Size of array aOnceFlag[] */ u8 *aOnceFlag; /* Flags for OP_Once */ AuxData *pAuxData; /* Linked list of auxdata allocations */ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + i64 *anExec; /* Number of times each op has been executed */ + int nScan; /* Entries in aScan[] */ + ScanStatus *aScan; /* Scan definitions for sqlite3_stmt_scanstatus() */ +#endif }; /* @@ -382,6 +400,7 @@ struct Vdbe { void sqlite3VdbeFreeCursor(Vdbe *, VdbeCursor*); void sqliteVdbePopStack(Vdbe*,int); int sqlite3VdbeCursorMoveto(VdbeCursor*); +int sqlite3VdbeCursorRestore(VdbeCursor*); #if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) void sqlite3VdbePrintOp(FILE*, int, Op*); #endif @@ -392,9 +411,8 @@ u32 sqlite3VdbeSerialGet(const unsigned char*, u32, Mem*); void sqlite3VdbeDeleteAuxData(Vdbe*, int, int); int sqlite2BtreeKeyCompare(BtCursor *, const void *, int, int, int *); -int sqlite3VdbeIdxKeyCompare(VdbeCursor*,UnpackedRecord*,int*); -int sqlite3VdbeIdxRowid(sqlite3*, BtCursor *, i64 *); -int sqlite3MemCompare(const Mem*, const Mem*, const CollSeq*); +int sqlite3VdbeIdxKeyCompare(sqlite3*,VdbeCursor*,UnpackedRecord*,int*); +int sqlite3VdbeIdxRowid(sqlite3*, BtCursor*, i64*); int sqlite3VdbeExec(Vdbe*); int sqlite3VdbeList(Vdbe*); int sqlite3VdbeHalt(Vdbe*); @@ -411,39 +429,39 @@ void sqlite3VdbeMemSetInt64(Mem*, i64); #else void sqlite3VdbeMemSetDouble(Mem*, double); #endif +void sqlite3VdbeMemInit(Mem*,sqlite3*,u16); void sqlite3VdbeMemSetNull(Mem*); void sqlite3VdbeMemSetZeroBlob(Mem*,int); void sqlite3VdbeMemSetRowSet(Mem*); int sqlite3VdbeMemMakeWriteable(Mem*); -int sqlite3VdbeMemStringify(Mem*, int); +int sqlite3VdbeMemStringify(Mem*, u8, u8); i64 sqlite3VdbeIntValue(Mem*); int sqlite3VdbeMemIntegerify(Mem*); double sqlite3VdbeRealValue(Mem*); void sqlite3VdbeIntegerAffinity(Mem*); int sqlite3VdbeMemRealify(Mem*); int sqlite3VdbeMemNumerify(Mem*); +void sqlite3VdbeMemCast(Mem*,u8,u8); int sqlite3VdbeMemFromBtree(BtCursor*,u32,u32,int,Mem*); void sqlite3VdbeMemRelease(Mem *p); -void sqlite3VdbeMemReleaseExternal(Mem *p); #define VdbeMemDynamic(X) \ (((X)->flags&(MEM_Agg|MEM_Dyn|MEM_RowSet|MEM_Frame))!=0) -#define VdbeMemRelease(X) \ - if( VdbeMemDynamic(X) ) sqlite3VdbeMemReleaseExternal(X); int sqlite3VdbeMemFinalize(Mem*, FuncDef*); const char *sqlite3OpcodeName(int); int sqlite3VdbeMemGrow(Mem *pMem, int n, int preserve); +int sqlite3VdbeMemClearAndResize(Mem *pMem, int n); int sqlite3VdbeCloseStatement(Vdbe *, int); void sqlite3VdbeFrameDelete(VdbeFrame*); int sqlite3VdbeFrameRestore(VdbeFrame *); int sqlite3VdbeTransferError(Vdbe *p); -int sqlite3VdbeSorterInit(sqlite3 *, VdbeCursor *); +int sqlite3VdbeSorterInit(sqlite3 *, int, VdbeCursor *); void sqlite3VdbeSorterReset(sqlite3 *, VdbeSorter *); void sqlite3VdbeSorterClose(sqlite3 *, VdbeCursor *); int sqlite3VdbeSorterRowkey(const VdbeCursor *, Mem *); int sqlite3VdbeSorterNext(sqlite3 *, const VdbeCursor *, int *); -int sqlite3VdbeSorterRewind(sqlite3 *, const VdbeCursor *, int *); -int sqlite3VdbeSorterWrite(sqlite3 *, const VdbeCursor *, Mem *); +int sqlite3VdbeSorterRewind(const VdbeCursor *, int *); +int sqlite3VdbeSorterWrite(const VdbeCursor *, Mem *); int sqlite3VdbeSorterCompare(const VdbeCursor *, Mem *, int, int *); #if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE>0 diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 5d7a0b0608..21c537d776 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -212,9 +212,12 @@ int sqlite3_value_type(sqlite3_value* pVal){ ** The following routines are used by user-defined functions to specify ** the function result. ** -** The setStrOrError() funtion calls sqlite3VdbeMemSetStr() to store the +** The setStrOrError() function calls sqlite3VdbeMemSetStr() to store the ** result as a string or blob but if the string or blob is too large, it ** then sets the error code to SQLITE_TOOBIG +** +** The invokeValueDestructor(P,X) routine invokes destructor function X() +** on value P is not going to be used and need to be destroyed. */ static void setResultStrOrError( sqlite3_context *pCtx, /* Function context */ @@ -223,10 +226,26 @@ static void setResultStrOrError( u8 enc, /* Encoding of z. 0 for BLOBs */ void (*xDel)(void*) /* Destructor function */ ){ - if( sqlite3VdbeMemSetStr(&pCtx->s, z, n, enc, xDel)==SQLITE_TOOBIG ){ + if( sqlite3VdbeMemSetStr(pCtx->pOut, z, n, enc, xDel)==SQLITE_TOOBIG ){ sqlite3_result_error_toobig(pCtx); } } +static int invokeValueDestructor( + const void *p, /* Value to destroy */ + void (*xDel)(void*), /* The destructor */ + sqlite3_context *pCtx /* Set a SQLITE_TOOBIG error if no NULL */ +){ + assert( xDel!=SQLITE_DYNAMIC ); + if( xDel==0 ){ + /* noop */ + }else if( xDel==SQLITE_TRANSIENT ){ + /* noop */ + }else{ + xDel((void*)p); + } + if( pCtx ) sqlite3_result_error_toobig(pCtx); + return SQLITE_TOOBIG; +} void sqlite3_result_blob( sqlite3_context *pCtx, const void *z, @@ -234,38 +253,52 @@ void sqlite3_result_blob( void (*xDel)(void *) ){ assert( n>=0 ); - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); setResultStrOrError(pCtx, z, n, 0, xDel); } +void sqlite3_result_blob64( + sqlite3_context *pCtx, + const void *z, + sqlite3_uint64 n, + void (*xDel)(void *) +){ + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + assert( xDel!=SQLITE_DYNAMIC ); + if( n>0x7fffffff ){ + (void)invokeValueDestructor(z, xDel, pCtx); + }else{ + setResultStrOrError(pCtx, z, (int)n, 0, xDel); + } +} void sqlite3_result_double(sqlite3_context *pCtx, double rVal){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); - sqlite3VdbeMemSetDouble(&pCtx->s, rVal); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + sqlite3VdbeMemSetDouble(pCtx->pOut, rVal); } void sqlite3_result_error(sqlite3_context *pCtx, const char *z, int n){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); pCtx->isError = SQLITE_ERROR; pCtx->fErrorOrAux = 1; - sqlite3VdbeMemSetStr(&pCtx->s, z, n, SQLITE_UTF8, SQLITE_TRANSIENT); + sqlite3VdbeMemSetStr(pCtx->pOut, z, n, SQLITE_UTF8, SQLITE_TRANSIENT); } #ifndef SQLITE_OMIT_UTF16 void sqlite3_result_error16(sqlite3_context *pCtx, const void *z, int n){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); pCtx->isError = SQLITE_ERROR; pCtx->fErrorOrAux = 1; - sqlite3VdbeMemSetStr(&pCtx->s, z, n, SQLITE_UTF16NATIVE, SQLITE_TRANSIENT); + sqlite3VdbeMemSetStr(pCtx->pOut, z, n, SQLITE_UTF16NATIVE, SQLITE_TRANSIENT); } #endif void sqlite3_result_int(sqlite3_context *pCtx, int iVal){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); - sqlite3VdbeMemSetInt64(&pCtx->s, (i64)iVal); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + sqlite3VdbeMemSetInt64(pCtx->pOut, (i64)iVal); } void sqlite3_result_int64(sqlite3_context *pCtx, i64 iVal){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); - sqlite3VdbeMemSetInt64(&pCtx->s, iVal); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + sqlite3VdbeMemSetInt64(pCtx->pOut, iVal); } void sqlite3_result_null(sqlite3_context *pCtx){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); - sqlite3VdbeMemSetNull(&pCtx->s); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + sqlite3VdbeMemSetNull(pCtx->pOut); } void sqlite3_result_text( sqlite3_context *pCtx, @@ -273,9 +306,25 @@ void sqlite3_result_text( int n, void (*xDel)(void *) ){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); setResultStrOrError(pCtx, z, n, SQLITE_UTF8, xDel); } +void sqlite3_result_text64( + sqlite3_context *pCtx, + const char *z, + sqlite3_uint64 n, + void (*xDel)(void *), + unsigned char enc +){ + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + assert( xDel!=SQLITE_DYNAMIC ); + if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE; + if( n>0x7fffffff ){ + (void)invokeValueDestructor(z, xDel, pCtx); + }else{ + setResultStrOrError(pCtx, z, (int)n, enc, xDel); + } +} #ifndef SQLITE_OMIT_UTF16 void sqlite3_result_text16( sqlite3_context *pCtx, @@ -283,7 +332,7 @@ void sqlite3_result_text16( int n, void (*xDel)(void *) ){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); setResultStrOrError(pCtx, z, n, SQLITE_UTF16NATIVE, xDel); } void sqlite3_result_text16be( @@ -292,7 +341,7 @@ void sqlite3_result_text16be( int n, void (*xDel)(void *) ){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); setResultStrOrError(pCtx, z, n, SQLITE_UTF16BE, xDel); } void sqlite3_result_text16le( @@ -301,43 +350,43 @@ void sqlite3_result_text16le( int n, void (*xDel)(void *) ){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); setResultStrOrError(pCtx, z, n, SQLITE_UTF16LE, xDel); } #endif /* SQLITE_OMIT_UTF16 */ void sqlite3_result_value(sqlite3_context *pCtx, sqlite3_value *pValue){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); - sqlite3VdbeMemCopy(&pCtx->s, pValue); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + sqlite3VdbeMemCopy(pCtx->pOut, pValue); } void sqlite3_result_zeroblob(sqlite3_context *pCtx, int n){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); - sqlite3VdbeMemSetZeroBlob(&pCtx->s, n); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + sqlite3VdbeMemSetZeroBlob(pCtx->pOut, n); } void sqlite3_result_error_code(sqlite3_context *pCtx, int errCode){ pCtx->isError = errCode; pCtx->fErrorOrAux = 1; - if( pCtx->s.flags & MEM_Null ){ - sqlite3VdbeMemSetStr(&pCtx->s, sqlite3ErrStr(errCode), -1, + if( pCtx->pOut->flags & MEM_Null ){ + sqlite3VdbeMemSetStr(pCtx->pOut, sqlite3ErrStr(errCode), -1, SQLITE_UTF8, SQLITE_STATIC); } } /* Force an SQLITE_TOOBIG error. */ void sqlite3_result_error_toobig(sqlite3_context *pCtx){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); pCtx->isError = SQLITE_TOOBIG; pCtx->fErrorOrAux = 1; - sqlite3VdbeMemSetStr(&pCtx->s, "string or blob too big", -1, + sqlite3VdbeMemSetStr(pCtx->pOut, "string or blob too big", -1, SQLITE_UTF8, SQLITE_STATIC); } /* An SQLITE_NOMEM error. */ void sqlite3_result_error_nomem(sqlite3_context *pCtx){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); - sqlite3VdbeMemSetNull(&pCtx->s); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + sqlite3VdbeMemSetNull(pCtx->pOut); pCtx->isError = SQLITE_NOMEM; pCtx->fErrorOrAux = 1; - pCtx->s.db->mallocFailed = 1; + pCtx->pOut->db->mallocFailed = 1; } /* @@ -351,7 +400,10 @@ static int doWalCallbacks(sqlite3 *db){ for(i=0; inDb; i++){ Btree *pBt = db->aDb[i].pBt; if( pBt ){ - int nEntry = sqlite3PagerWalCallback(sqlite3BtreePager(pBt)); + int nEntry; + sqlite3BtreeEnter(pBt); + nEntry = sqlite3PagerWalCallback(sqlite3BtreePager(pBt)); + sqlite3BtreeLeave(pBt); if( db->xWalCallback && nEntry>0 && rc==SQLITE_OK ){ rc = db->xWalCallback(db->pWalArg, db, db->aDb[i].zName, nEntry); } @@ -513,10 +565,12 @@ int sqlite3_step(sqlite3_stmt *pStmt){ sqlite3_mutex_enter(db->mutex); v->doingRerun = 0; while( (rc = sqlite3Step(v))==SQLITE_SCHEMA - && cnt++ < SQLITE_MAX_SCHEMA_RETRY - && (rc2 = rc = sqlite3Reprepare(v))==SQLITE_OK ){ + && cnt++ < SQLITE_MAX_SCHEMA_RETRY ){ + int savedPc = v->pc; + rc2 = rc = sqlite3Reprepare(v); + if( rc!=SQLITE_OK) break; sqlite3_reset(pStmt); - v->doingRerun = 1; + if( savedPc>=0 ) v->doingRerun = 1; assert( v->expired==0 ); } if( rc2!=SQLITE_OK ){ @@ -529,7 +583,6 @@ int sqlite3_step(sqlite3_stmt *pStmt){ ** sqlite3_errmsg() and sqlite3_errcode(). */ const char *zErr = (const char *)sqlite3_value_text(db->pErr); - assert( zErr!=0 || db->mallocFailed ); sqlite3DbFree(db, v->zErrMsg); if( !db->mallocFailed ){ v->zErrMsg = sqlite3DbStrDup(db, zErr); @@ -566,7 +619,7 @@ void *sqlite3_user_data(sqlite3_context *p){ */ sqlite3 *sqlite3_context_db_handle(sqlite3_context *p){ assert( p && p->pFunc ); - return p->s.db; + return p->pOut->db; } /* @@ -576,7 +629,7 @@ sqlite3_int64 sqlite3StmtCurrentTime(sqlite3_context *p){ Vdbe *v = p->pVdbe; int rc; if( v->iCurrentTime==0 ){ - rc = sqlite3OsCurrentTimeInt64(p->s.db->pVfs, &v->iCurrentTime); + rc = sqlite3OsCurrentTimeInt64(p->pOut->db->pVfs, &v->iCurrentTime); if( rc ) v->iCurrentTime = 0; } return v->iCurrentTime; @@ -605,41 +658,50 @@ void sqlite3InvalidFunction( } /* -** Allocate or return the aggregate context for a user function. A new -** context is allocated on the first call. Subsequent calls return the -** same context that was returned on prior calls. +** Create a new aggregate context for p and return a pointer to +** its pMem->z element. */ -void *sqlite3_aggregate_context(sqlite3_context *p, int nByte){ - Mem *pMem; - assert( p && p->pFunc && p->pFunc->xStep ); - assert( sqlite3_mutex_held(p->s.db->mutex) ); - pMem = p->pMem; - testcase( nByte<0 ); - if( (pMem->flags & MEM_Agg)==0 ){ - if( nByte<=0 ){ - sqlite3VdbeMemReleaseExternal(pMem); - pMem->flags = MEM_Null; - pMem->z = 0; - }else{ - sqlite3VdbeMemGrow(pMem, nByte, 0); - pMem->flags = MEM_Agg; - pMem->u.pDef = p->pFunc; - if( pMem->z ){ - memset(pMem->z, 0, nByte); - } +static SQLITE_NOINLINE void *createAggContext(sqlite3_context *p, int nByte){ + Mem *pMem = p->pMem; + assert( (pMem->flags & MEM_Agg)==0 ); + if( nByte<=0 ){ + sqlite3VdbeMemSetNull(pMem); + pMem->z = 0; + }else{ + sqlite3VdbeMemClearAndResize(pMem, nByte); + pMem->flags = MEM_Agg; + pMem->u.pDef = p->pFunc; + if( pMem->z ){ + memset(pMem->z, 0, nByte); } } return (void*)pMem->z; } /* -** Return the auxilary data pointer, if any, for the iArg'th argument to +** Allocate or return the aggregate context for a user function. A new +** context is allocated on the first call. Subsequent calls return the +** same context that was returned on prior calls. +*/ +void *sqlite3_aggregate_context(sqlite3_context *p, int nByte){ + assert( p && p->pFunc && p->pFunc->xStep ); + assert( sqlite3_mutex_held(p->pOut->db->mutex) ); + testcase( nByte<0 ); + if( (p->pMem->flags & MEM_Agg)==0 ){ + return createAggContext(p, nByte); + }else{ + return (void*)p->pMem->z; + } +} + +/* +** Return the auxiliary data pointer, if any, for the iArg'th argument to ** the user-function defined by pCtx. */ void *sqlite3_get_auxdata(sqlite3_context *pCtx, int iArg){ AuxData *pAuxData; - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); for(pAuxData=pCtx->pVdbe->pAuxData; pAuxData; pAuxData=pAuxData->pNext){ if( pAuxData->iOp==pCtx->iOp && pAuxData->iArg==iArg ) break; } @@ -648,7 +710,7 @@ void *sqlite3_get_auxdata(sqlite3_context *pCtx, int iArg){ } /* -** Set the auxilary data pointer and delete function, for the iArg'th +** Set the auxiliary data pointer and delete function, for the iArg'th ** argument to the user-function defined by pCtx. Any previous value is ** deleted by calling the delete function specified when it was set. */ @@ -661,7 +723,7 @@ void sqlite3_set_auxdata( AuxData *pAuxData; Vdbe *pVdbe = pCtx->pVdbe; - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); if( iArg<0 ) goto failed; for(pAuxData=pVdbe->pAuxData; pAuxData; pAuxData=pAuxData->pNext){ @@ -694,7 +756,7 @@ failed: #ifndef SQLITE_OMIT_DEPRECATED /* -** Return the number of times the Step function of a aggregate has been +** Return the number of times the Step function of an aggregate has been ** called. ** ** This function is deprecated. Do not use it for new code. It is @@ -743,11 +805,22 @@ static const Mem *columnNullValue(void){ #if defined(SQLITE_DEBUG) && defined(__GNUC__) __attribute__((aligned(8))) #endif - = {0, "", (double)0, {0}, 0, MEM_Null, 0, + = { + /* .u = */ {0}, + /* .flags = */ MEM_Null, + /* .enc = */ 0, + /* .n = */ 0, + /* .z = */ 0, + /* .zMalloc = */ 0, + /* .szMalloc = */ 0, + /* .iPadding1 = */ 0, + /* .db = */ 0, + /* .xDel = */ 0, #ifdef SQLITE_DEBUG - 0, 0, /* pScopyFrom, pFiller */ + /* .pScopyFrom = */ 0, + /* .pFiller = */ 0, #endif - 0, 0 }; + }; return &nullMem; } @@ -768,7 +841,7 @@ static Mem *columnMem(sqlite3_stmt *pStmt, int i){ }else{ if( pVm && ALWAYS(pVm->db) ){ sqlite3_mutex_enter(pVm->db->mutex); - sqlite3Error(pVm->db, SQLITE_RANGE, 0); + sqlite3Error(pVm->db, SQLITE_RANGE); } pOut = (Mem*)columnNullValue(); } @@ -895,11 +968,19 @@ static const void *columnName( const void *(*xFunc)(Mem*), int useType ){ - const void *ret = 0; - Vdbe *p = (Vdbe *)pStmt; + const void *ret; + Vdbe *p; int n; - sqlite3 *db = p->db; - + sqlite3 *db; +#ifdef SQLITE_ENABLE_API_ARMOR + if( pStmt==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + ret = 0; + p = (Vdbe *)pStmt; + db = p->db; assert( db!=0 ); n = sqlite3_column_count(pStmt); if( N=0 ){ @@ -964,7 +1045,7 @@ const void *sqlite3_column_decltype16(sqlite3_stmt *pStmt, int N){ /* ** Return the name of the database from which a result column derives. ** NULL is returned if the result column is an expression or constant or -** anything else which is not an unabiguous reference to a database column. +** anything else which is not an unambiguous reference to a database column. */ const char *sqlite3_column_database_name(sqlite3_stmt *pStmt, int N){ return columnName( @@ -980,7 +1061,7 @@ const void *sqlite3_column_database_name16(sqlite3_stmt *pStmt, int N){ /* ** Return the name of the table from which a result column derives. ** NULL is returned if the result column is an expression or constant or -** anything else which is not an unabiguous reference to a database column. +** anything else which is not an unambiguous reference to a database column. */ const char *sqlite3_column_table_name(sqlite3_stmt *pStmt, int N){ return columnName( @@ -996,7 +1077,7 @@ const void *sqlite3_column_table_name16(sqlite3_stmt *pStmt, int N){ /* ** Return the name of the table column from which a result column derives. ** NULL is returned if the result column is an expression or constant or -** anything else which is not an unabiguous reference to a database column. +** anything else which is not an unambiguous reference to a database column. */ const char *sqlite3_column_origin_name(sqlite3_stmt *pStmt, int N){ return columnName( @@ -1033,14 +1114,14 @@ static int vdbeUnbind(Vdbe *p, int i){ } sqlite3_mutex_enter(p->db->mutex); if( p->magic!=VDBE_MAGIC_RUN || p->pc>=0 ){ - sqlite3Error(p->db, SQLITE_MISUSE, 0); + sqlite3Error(p->db, SQLITE_MISUSE); sqlite3_mutex_leave(p->db->mutex); sqlite3_log(SQLITE_MISUSE, "bind on a busy prepared statement: [%s]", p->zSql); return SQLITE_MISUSE_BKPT; } if( i<1 || i>p->nVar ){ - sqlite3Error(p->db, SQLITE_RANGE, 0); + sqlite3Error(p->db, SQLITE_RANGE); sqlite3_mutex_leave(p->db->mutex); return SQLITE_RANGE; } @@ -1048,7 +1129,7 @@ static int vdbeUnbind(Vdbe *p, int i){ pVar = &p->aVar[i]; sqlite3VdbeMemRelease(pVar); pVar->flags = MEM_Null; - sqlite3Error(p->db, SQLITE_OK, 0); + sqlite3Error(p->db, SQLITE_OK); /* If the bit corresponding to this variable in Vdbe.expmask is set, then ** binding a new value to this variable invalidates the current query plan. @@ -1090,7 +1171,7 @@ static int bindText( if( rc==SQLITE_OK && encoding!=0 ){ rc = sqlite3VdbeChangeEncoding(pVar, ENC(p->db)); } - sqlite3Error(p->db, rc, 0); + sqlite3Error(p->db, rc); rc = sqlite3ApiExit(p->db, rc); } sqlite3_mutex_leave(p->db->mutex); @@ -1113,6 +1194,20 @@ int sqlite3_bind_blob( ){ return bindText(pStmt, i, zData, nData, xDel, 0); } +int sqlite3_bind_blob64( + sqlite3_stmt *pStmt, + int i, + const void *zData, + sqlite3_uint64 nData, + void (*xDel)(void*) +){ + assert( xDel!=SQLITE_DYNAMIC ); + if( nData>0x7fffffff ){ + return invokeValueDestructor(zData, xDel, 0); + }else{ + return bindText(pStmt, i, zData, (int)nData, xDel, 0); + } +} int sqlite3_bind_double(sqlite3_stmt *pStmt, int i, double rValue){ int rc; Vdbe *p = (Vdbe *)pStmt; @@ -1154,6 +1249,22 @@ int sqlite3_bind_text( ){ return bindText(pStmt, i, zData, nData, xDel, SQLITE_UTF8); } +int sqlite3_bind_text64( + sqlite3_stmt *pStmt, + int i, + const char *zData, + sqlite3_uint64 nData, + void (*xDel)(void*), + unsigned char enc +){ + assert( xDel!=SQLITE_DYNAMIC ); + if( nData>0x7fffffff ){ + return invokeValueDestructor(zData, xDel, 0); + }else{ + if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE; + return bindText(pStmt, i, zData, (int)nData, xDel, enc); + } +} #ifndef SQLITE_OMIT_UTF16 int sqlite3_bind_text16( sqlite3_stmt *pStmt, @@ -1173,7 +1284,7 @@ int sqlite3_bind_value(sqlite3_stmt *pStmt, int i, const sqlite3_value *pValue){ break; } case SQLITE_FLOAT: { - rc = sqlite3_bind_double(pStmt, i, pValue->r); + rc = sqlite3_bind_double(pStmt, i, pValue->u.r); break; } case SQLITE_BLOB: { @@ -1276,7 +1387,7 @@ int sqlite3TransferBindings(sqlite3_stmt *pFromStmt, sqlite3_stmt *pToStmt){ ** Deprecated external interface. Internal/core SQLite code ** should call sqlite3TransferBindings. ** -** Is is misuse to call this routine with statements from different +** It is misuse to call this routine with statements from different ** database connections. But as this is a deprecated interface, we ** will not bother to check for that condition. ** @@ -1323,7 +1434,7 @@ int sqlite3_stmt_readonly(sqlite3_stmt *pStmt){ */ int sqlite3_stmt_busy(sqlite3_stmt *pStmt){ Vdbe *v = (Vdbe*)pStmt; - return v!=0 && v->pc>0 && v->magic==VDBE_MAGIC_RUN; + return v!=0 && v->pc>=0 && v->magic==VDBE_MAGIC_RUN; } /* @@ -1334,6 +1445,12 @@ int sqlite3_stmt_busy(sqlite3_stmt *pStmt){ */ sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt){ sqlite3_stmt *pNext; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(pDb) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif sqlite3_mutex_enter(pDb->mutex); if( pStmt==0 ){ pNext = (sqlite3_stmt*)pDb->pVdbe; @@ -1349,7 +1466,83 @@ sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt){ */ int sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, int resetFlag){ Vdbe *pVdbe = (Vdbe*)pStmt; - u32 v = pVdbe->aCounter[op]; + u32 v; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !pStmt ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + v = pVdbe->aCounter[op]; if( resetFlag ) pVdbe->aCounter[op] = 0; return (int)v; } + +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +/* +** Return status data for a single loop within query pStmt. +*/ +int sqlite3_stmt_scanstatus( + sqlite3_stmt *pStmt, /* Prepared statement being queried */ + int idx, /* Index of loop to report on */ + int iScanStatusOp, /* Which metric to return */ + void *pOut /* OUT: Write the answer here */ +){ + Vdbe *p = (Vdbe*)pStmt; + ScanStatus *pScan; + if( idx<0 || idx>=p->nScan ) return 1; + pScan = &p->aScan[idx]; + switch( iScanStatusOp ){ + case SQLITE_SCANSTAT_NLOOP: { + *(sqlite3_int64*)pOut = p->anExec[pScan->addrLoop]; + break; + } + case SQLITE_SCANSTAT_NVISIT: { + *(sqlite3_int64*)pOut = p->anExec[pScan->addrVisit]; + break; + } + case SQLITE_SCANSTAT_EST: { + double r = 1.0; + LogEst x = pScan->nEst; + while( x<100 ){ + x += 10; + r *= 0.5; + } + *(double*)pOut = r*sqlite3LogEstToInt(x); + break; + } + case SQLITE_SCANSTAT_NAME: { + *(const char**)pOut = pScan->zName; + break; + } + case SQLITE_SCANSTAT_EXPLAIN: { + if( pScan->addrExplain ){ + *(const char**)pOut = p->aOp[ pScan->addrExplain ].p4.z; + }else{ + *(const char**)pOut = 0; + } + break; + } + case SQLITE_SCANSTAT_SELECTID: { + if( pScan->addrExplain ){ + *(int*)pOut = p->aOp[ pScan->addrExplain ].p1; + }else{ + *(int*)pOut = -1; + } + break; + } + default: { + return 1; + } + } + return 0; +} + +/* +** Zero all counters associated with the sqlite3_stmt_scanstatus() data. +*/ +void sqlite3_stmt_scanstatus_reset(sqlite3_stmt *pStmt){ + Vdbe *p = (Vdbe*)pStmt; + memset(p->anExec, 0, p->nOp * sizeof(i64)); +} +#endif /* SQLITE_ENABLE_STMT_SCANSTATUS */ diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 0a6b536720..03c229c994 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -10,9 +10,7 @@ ** ************************************************************************* ** This file contains code used for creating, destroying, and populating -** a VDBE (or an "sqlite3_stmt" as it is known to the outside world.) Prior -** to version 2.8.7, all this code was combined into the vdbe.c source file. -** But that file was getting too big so this subroutines were split out. +** a VDBE (or an "sqlite3_stmt" as it is known to the outside world.) */ #include "sqliteInt.h" #include "vdbeInt.h" @@ -84,18 +82,35 @@ void sqlite3VdbeSwap(Vdbe *pA, Vdbe *pB){ } /* -** Resize the Vdbe.aOp array so that it is at least one op larger than -** it was. +** Resize the Vdbe.aOp array so that it is at least nOp elements larger +** than its current size. nOp is guaranteed to be less than or equal +** to 1024/sizeof(Op). ** ** If an out-of-memory error occurs while resizing the array, return -** SQLITE_NOMEM. In this case Vdbe.aOp and Vdbe.nOpAlloc remain +** SQLITE_NOMEM. In this case Vdbe.aOp and Parse.nOpAlloc remain ** unchanged (this is so that any opcodes already allocated can be ** correctly deallocated along with the rest of the Vdbe). */ -static int growOpArray(Vdbe *v){ +static int growOpArray(Vdbe *v, int nOp){ VdbeOp *pNew; Parse *p = v->pParse; + + /* The SQLITE_TEST_REALLOC_STRESS compile-time option is designed to force + ** more frequent reallocs and hence provide more opportunities for + ** simulated OOM faults. SQLITE_TEST_REALLOC_STRESS is generally used + ** during testing only. With SQLITE_TEST_REALLOC_STRESS grow the op array + ** by the minimum* amount required until the size reaches 512. Normal + ** operation (without SQLITE_TEST_REALLOC_STRESS) is to double the current + ** size of the op array or add 1KB of space, whichever is smaller. */ +#ifdef SQLITE_TEST_REALLOC_STRESS + int nNew = (p->nOpAlloc>=512 ? p->nOpAlloc*2 : p->nOpAlloc+nOp); +#else int nNew = (p->nOpAlloc ? p->nOpAlloc*2 : (int)(1024/sizeof(Op))); + UNUSED_PARAMETER(nOp); +#endif + + assert( nOp<=(1024/sizeof(Op)) ); + assert( nNew>=(p->nOpAlloc+nOp) ); pNew = sqlite3DbRealloc(p->db, v->aOp, nNew*sizeof(Op)); if( pNew ){ p->nOpAlloc = sqlite3DbMallocSize(p->db, pNew)/sizeof(Op); @@ -139,7 +154,7 @@ int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){ assert( p->magic==VDBE_MAGIC_INIT ); assert( op>0 && op<0xff ); if( p->pParse->nOpAlloc<=i ){ - if( growOpArray(p) ){ + if( growOpArray(p, 1) ){ return 1; } } @@ -381,6 +396,7 @@ static Op *opIterNext(VdbeOpIter *p){ */ int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ int hasAbort = 0; + int hasFkCounter = 0; Op *pOp; VdbeOpIter sIter; memset(&sIter, 0, sizeof(sIter)); @@ -389,15 +405,17 @@ int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ while( (pOp = opIterNext(&sIter))!=0 ){ int opcode = pOp->opcode; if( opcode==OP_Destroy || opcode==OP_VUpdate || opcode==OP_VRename -#ifndef SQLITE_OMIT_FOREIGN_KEY - || (opcode==OP_FkCounter && pOp->p1==0 && pOp->p2==1) -#endif || ((opcode==OP_Halt || opcode==OP_HaltIfNull) && ((pOp->p1&0xff)==SQLITE_CONSTRAINT && pOp->p2==OE_Abort)) ){ hasAbort = 1; break; } +#ifndef SQLITE_OMIT_FOREIGN_KEY + if( opcode==OP_FkCounter && pOp->p1==0 && pOp->p2==1 ){ + hasFkCounter = 1; + } +#endif } sqlite3DbFree(v->db, sIter.apSub); @@ -406,7 +424,7 @@ int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ ** through all opcodes and hasAbort may be set incorrectly. Return ** true for this case to prevent the assert() in the callers frame ** from failing. */ - return ( v->db->mallocFailed || hasAbort==mayAbort ); + return ( v->db->mallocFailed || hasAbort==mayAbort || hasFkCounter ); } #endif /* SQLITE_DEBUG - the sqlite3AssertMayAbort() function */ @@ -499,7 +517,7 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ pParse->aLabel = 0; pParse->nLabel = 0; *pMaxFuncArgs = nMaxArgs; - assert( p->bIsReader!=0 || p->btreeMask==0 ); + assert( p->bIsReader!=0 || DbMaskAllZero(p->btreeMask) ); } /* @@ -526,7 +544,7 @@ VdbeOp *sqlite3VdbeTakeOpArray(Vdbe *p, int *pnOp, int *pnMaxArg){ assert( aOp && !p->db->mallocFailed ); /* Check that sqlite3VdbeUsesBtree() was not called on this VM */ - assert( p->btreeMask==0 ); + assert( DbMaskAllZero(p->btreeMask) ); resolveP2Values(p, pnMaxArg); *pnOp = p->nOp; @@ -541,7 +559,7 @@ VdbeOp *sqlite3VdbeTakeOpArray(Vdbe *p, int *pnOp, int *pnMaxArg){ int sqlite3VdbeAddOpList(Vdbe *p, int nOp, VdbeOpList const *aOp, int iLineno){ int addr; assert( p->magic==VDBE_MAGIC_INIT ); - if( p->nOp + nOp > p->pParse->nOpAlloc && growOpArray(p) ){ + if( p->nOp + nOp > p->pParse->nOpAlloc && growOpArray(p, nOp) ){ return 0; } addr = p->nOp; @@ -582,6 +600,34 @@ int sqlite3VdbeAddOpList(Vdbe *p, int nOp, VdbeOpList const *aOp, int iLineno){ return addr; } +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) +/* +** Add an entry to the array of counters managed by sqlite3_stmt_scanstatus(). +*/ +void sqlite3VdbeScanStatus( + Vdbe *p, /* VM to add scanstatus() to */ + int addrExplain, /* Address of OP_Explain (or 0) */ + int addrLoop, /* Address of loop counter */ + int addrVisit, /* Address of rows visited counter */ + LogEst nEst, /* Estimated number of output rows */ + const char *zName /* Name of table or index being scanned */ +){ + int nByte = (p->nScan+1) * sizeof(ScanStatus); + ScanStatus *aNew; + aNew = (ScanStatus*)sqlite3DbRealloc(p->db, p->aScan, nByte); + if( aNew ){ + ScanStatus *pNew = &aNew[p->nScan++]; + pNew->addrExplain = addrExplain; + pNew->addrLoop = addrLoop; + pNew->addrVisit = addrVisit; + pNew->nEst = nEst; + pNew->zName = sqlite3DbStrDup(p->db, zName); + p->aScan = aNew; + } +} +#endif + + /* ** Change the value of the P1 operand for a specific instruction. ** This routine is useful when a large program is loaded from a @@ -681,7 +727,7 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){ sqlite3ValueFree((sqlite3_value*)p4); }else{ Mem *p = (Mem*)p4; - sqlite3DbFree(db, p->zMalloc); + if( p->szMalloc ) sqlite3DbFree(db, p->zMalloc); sqlite3DbFree(db, p); } break; @@ -726,7 +772,7 @@ void sqlite3VdbeLinkSubProgram(Vdbe *pVdbe, SubProgram *p){ ** Change the opcode at addr into OP_Noop */ void sqlite3VdbeChangeToNoop(Vdbe *p, int addr){ - if( p->aOp ){ + if( addrnOp ){ VdbeOp *pOp = &p->aOp[addr]; sqlite3 *db = p->db; freeP4(db, pOp->p4type, pOp->p4.p); @@ -737,7 +783,8 @@ void sqlite3VdbeChangeToNoop(Vdbe *p, int addr){ } /* -** Remove the last opcode inserted +** If the last opcode is "op" and it is not a jump destination, +** then remove it. Return true if and only if an opcode was removed. */ int sqlite3VdbeDeletePriorOpcode(Vdbe *p, u8 op){ if( (p->nOp-1)>(p->pParse->iFixedOp) && p->aOp[p->nOp-1].opcode==op ){ @@ -878,7 +925,7 @@ void sqlite3VdbeSetLineNumber(Vdbe *v, int iLine){ ** routine, then a pointer to a dummy VdbeOp will be returned. That opcode ** is readable but not writable, though it is cast to a writable value. ** The return of a dummy opcode allows the call to continue functioning -** after a OOM fault without having to check to see if the return from +** after an OOM fault without having to check to see if the return from ** this routine is a valid pointer. But because the dummy.opcode is 0, ** dummy will never be written to. This is verified by code inspection and ** by running with Valgrind. @@ -1059,7 +1106,7 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ }else if( pMem->flags & MEM_Int ){ sqlite3_snprintf(nTemp, zTemp, "%lld", pMem->u.i); }else if( pMem->flags & MEM_Real ){ - sqlite3_snprintf(nTemp, zTemp, "%.16g", pMem->r); + sqlite3_snprintf(nTemp, zTemp, "%.16g", pMem->u.r); }else if( pMem->flags & MEM_Null ){ sqlite3_snprintf(nTemp, zTemp, "NULL"); }else{ @@ -1111,9 +1158,9 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ void sqlite3VdbeUsesBtree(Vdbe *p, int i){ assert( i>=0 && idb->nDb && i<(int)sizeof(yDbMask)*8 ); assert( i<(int)sizeof(p->btreeMask)*8 ); - p->btreeMask |= ((yDbMask)1)<btreeMask, i); if( i!=1 && sqlite3BtreeSharable(p->db->aDb[i].pBt) ){ - p->lockMask |= ((yDbMask)1)<lockMask, i); } } @@ -1141,16 +1188,15 @@ void sqlite3VdbeUsesBtree(Vdbe *p, int i){ */ void sqlite3VdbeEnter(Vdbe *p){ int i; - yDbMask mask; sqlite3 *db; Db *aDb; int nDb; - if( p->lockMask==0 ) return; /* The common case */ + if( DbMaskAllZero(p->lockMask) ) return; /* The common case */ db = p->db; aDb = db->aDb; nDb = db->nDb; - for(i=0, mask=1; ilockMask)!=0 && ALWAYS(aDb[i].pBt!=0) ){ + for(i=0; ilockMask,i) && ALWAYS(aDb[i].pBt!=0) ){ sqlite3BtreeEnter(aDb[i].pBt); } } @@ -1163,16 +1209,15 @@ void sqlite3VdbeEnter(Vdbe *p){ */ void sqlite3VdbeLeave(Vdbe *p){ int i; - yDbMask mask; sqlite3 *db; Db *aDb; int nDb; - if( p->lockMask==0 ) return; /* The common case */ + if( DbMaskAllZero(p->lockMask) ) return; /* The common case */ db = p->db; aDb = db->aDb; nDb = db->nDb; - for(i=0, mask=1; ilockMask)!=0 && ALWAYS(aDb[i].pBt!=0) ){ + for(i=0; ilockMask,i) && ALWAYS(aDb[i].pBt!=0) ){ sqlite3BtreeLeave(aDb[i].pBt); } } @@ -1211,16 +1256,16 @@ void sqlite3VdbePrintOp(FILE *pOut, int pc, Op *pOp){ */ static void releaseMemArray(Mem *p, int N){ if( p && N ){ - Mem *pEnd; + Mem *pEnd = &p[N]; sqlite3 *db = p->db; u8 malloc_failed = db->mallocFailed; if( db->pnBytesFreed ){ - for(pEnd=&p[N]; pzMalloc); - } + do{ + if( p->szMalloc ) sqlite3DbFree(db, p->zMalloc); + }while( (++p)flags & MEM_RowSet ); if( p->flags&(MEM_Agg|MEM_Dyn|MEM_Frame|MEM_RowSet) ){ sqlite3VdbeMemRelease(p); - }else if( p->zMalloc ){ + }else if( p->szMalloc ){ sqlite3DbFree(db, p->zMalloc); - p->zMalloc = 0; + p->szMalloc = 0; } p->flags = MEM_Undefined; - } + }while( (++p)mallocFailed = malloc_failed; } } @@ -1411,7 +1456,7 @@ int sqlite3VdbeList( pMem->u.i = pOp->p3; /* P3 */ pMem++; - if( sqlite3VdbeMemGrow(pMem, 32, 0) ){ /* P4 */ + if( sqlite3VdbeMemClearAndResize(pMem, 32) ){ /* P4 */ assert( p->db->mallocFailed ); return SQLITE_ERROR; } @@ -1427,7 +1472,7 @@ int sqlite3VdbeList( pMem++; if( p->explain==1 ){ - if( sqlite3VdbeMemGrow(pMem, 4, 0) ){ + if( sqlite3VdbeMemClearAndResize(pMem, 4) ){ assert( p->db->mallocFailed ); return SQLITE_ERROR; } @@ -1438,7 +1483,7 @@ int sqlite3VdbeList( pMem++; #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS - if( sqlite3VdbeMemGrow(pMem, 500, 0) ){ + if( sqlite3VdbeMemClearAndResize(pMem, 500) ){ assert( p->db->mallocFailed ); return SQLITE_ERROR; } @@ -1591,13 +1636,13 @@ void sqlite3VdbeRewind(Vdbe *p){ /* ** Prepare a virtual machine for execution for the first time after ** creating the virtual machine. This involves things such -** as allocating stack space and initializing the program counter. +** as allocating registers and initializing the program counter. ** After the VDBE has be prepped, it can be executed by one or more ** calls to sqlite3VdbeExec(). ** -** This function may be called exact once on a each virtual machine. +** This function may be called exactly once on each virtual machine. ** After this routine is called the VM has been "packaged" and is ready -** to run. After this routine is called, futher calls to +** to run. After this routine is called, further calls to ** sqlite3VdbeAddOp() functions are prohibited. This routine disconnects ** the Vdbe from the Parse object that helped generate it so that the ** the Vdbe becomes an independent entity and the Parse object can be @@ -1681,6 +1726,9 @@ void sqlite3VdbeMakeReady( p->apCsr = allocSpace(p->apCsr, nCursor*sizeof(VdbeCursor*), &zCsr, zEnd, &nByte); p->aOnceFlag = allocSpace(p->aOnceFlag, nOnce, &zCsr, zEnd, &nByte); +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + p->anExec = allocSpace(p->anExec, p->nOp*sizeof(i64), &zCsr, zEnd, &nByte); +#endif if( nByte ){ p->pFree = sqlite3DbMallocZero(db, nByte); } @@ -1697,7 +1745,7 @@ void sqlite3VdbeMakeReady( p->aVar[n].db = db; } } - if( p->azVar ){ + if( p->azVar && pParse->nzVar>0 ){ p->nzVar = pParse->nzVar; memcpy(p->azVar, pParse->azVar, p->nzVar*sizeof(p->azVar[0])); memset(pParse->azVar, 0, pParse->nzVar*sizeof(pParse->azVar[0])); @@ -1731,7 +1779,7 @@ void sqlite3VdbeFreeCursor(Vdbe *p, VdbeCursor *pCx){ sqlite3BtreeCloseCursor(pCx->pCursor); } #ifndef SQLITE_OMIT_VIRTUALTABLE - if( pCx->pVtabCursor ){ + else if( pCx->pVtabCursor ){ sqlite3_vtab_cursor *pVtabCursor = pCx->pVtabCursor; const sqlite3_module *pModule = pVtabCursor->pVtab->pModule; p->inVtabMethod = 1; @@ -1748,6 +1796,9 @@ void sqlite3VdbeFreeCursor(Vdbe *p, VdbeCursor *pCx){ */ int sqlite3VdbeFrameRestore(VdbeFrame *pFrame){ Vdbe *v = pFrame->v; +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + v->anExec = pFrame->anExec; +#endif v->aOnceFlag = pFrame->aOnceFlag; v->nOnceFlag = pFrame->nOnceFlag; v->aOp = pFrame->aOp; @@ -1758,6 +1809,7 @@ int sqlite3VdbeFrameRestore(VdbeFrame *pFrame){ v->nCursor = pFrame->nCursor; v->db->lastRowid = pFrame->lastRowid; v->nChange = pFrame->nChange; + v->db->nChange = pFrame->nDbChange; return pFrame->pc; } @@ -1774,9 +1826,10 @@ static void closeAllCursors(Vdbe *p){ VdbeFrame *pFrame; for(pFrame=p->pFrame; pFrame->pParent; pFrame=pFrame->pParent); sqlite3VdbeFrameRestore(pFrame); + p->pFrame = 0; + p->nFrame = 0; } - p->pFrame = 0; - p->nFrame = 0; + assert( p->nFrame==0 ); if( p->apCsr ){ int i; @@ -1798,16 +1851,12 @@ static void closeAllCursors(Vdbe *p){ } /* Delete any auxdata allocations made by the VM */ - sqlite3VdbeDeleteAuxData(p, -1, 0); + if( p->pAuxData ) sqlite3VdbeDeleteAuxData(p, -1, 0); assert( p->pAuxData==0 ); } /* -** Clean up the VM after execution. -** -** This routine will automatically close any cursors, lists, and/or -** sorters that were left open. It also deletes the values of -** variables in the aVar[] array. +** Clean up the VM after a single run. */ static void Cleanup(Vdbe *p){ sqlite3 *db = p->db; @@ -1975,7 +2024,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ /* The complex case - There is a multi-file write-transaction active. ** This requires a master journal file to ensure the transaction is - ** committed atomicly. + ** committed atomically. */ #ifndef SQLITE_OMIT_DISKIO else{ @@ -2143,7 +2192,7 @@ static void checkActiveVdbeCnt(sqlite3 *db){ int nRead = 0; p = db->pVdbe; while( p ){ - if( p->magic==VDBE_MAGIC_RUN && p->pc>=0 ){ + if( sqlite3_stmt_busy((sqlite3_stmt*)p) ){ cnt++; if( p->readOnly==0 ) nWrite++; if( p->bIsReader ) nRead++; @@ -2303,7 +2352,6 @@ int sqlite3VdbeHalt(Vdbe *p){ /* Check for one of the special errors */ mrc = p->rc & 0xff; - assert( p->rc!=SQLITE_IOERR_BLOCKED ); /* This error no longer exists */ isSpecialError = mrc==SQLITE_NOMEM || mrc==SQLITE_IOERR || mrc==SQLITE_INTERRUPT || mrc==SQLITE_FULL; if( isSpecialError ){ @@ -2329,6 +2377,7 @@ int sqlite3VdbeHalt(Vdbe *p){ sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); sqlite3CloseSavepoints(db); db->autoCommit = 1; + p->nChange = 0; } } } @@ -2369,6 +2418,7 @@ int sqlite3VdbeHalt(Vdbe *p){ }else if( rc!=SQLITE_OK ){ p->rc = rc; sqlite3RollbackAll(db, SQLITE_OK); + p->nChange = 0; }else{ db->nDeferredCons = 0; db->nDeferredImmCons = 0; @@ -2377,6 +2427,7 @@ int sqlite3VdbeHalt(Vdbe *p){ } }else{ sqlite3RollbackAll(db, SQLITE_OK); + p->nChange = 0; } db->nStatement = 0; }else if( eStatementOp==0 ){ @@ -2388,6 +2439,7 @@ int sqlite3VdbeHalt(Vdbe *p){ sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); sqlite3CloseSavepoints(db); db->autoCommit = 1; + p->nChange = 0; } } @@ -2408,6 +2460,7 @@ int sqlite3VdbeHalt(Vdbe *p){ sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); sqlite3CloseSavepoints(db); db->autoCommit = 1; + p->nChange = 0; } } @@ -2483,7 +2536,7 @@ int sqlite3VdbeTransferError(Vdbe *p){ db->mallocFailed = mallocFailed; db->errCode = rc; }else{ - sqlite3Error(db, rc, 0); + sqlite3Error(db, rc); } return rc; } @@ -2546,7 +2599,7 @@ int sqlite3VdbeReset(Vdbe *p){ ** to sqlite3_step(). For consistency (since sqlite3_step() was ** called), set the database error in this case as well. */ - sqlite3Error(db, p->rc, p->zErrMsg ? "%s" : 0, p->zErrMsg); + sqlite3ErrorWithMsg(db, p->rc, p->zErrMsg ? "%s" : 0, p->zErrMsg); sqlite3DbFree(db, p->zErrMsg); p->zErrMsg = 0; } @@ -2624,7 +2677,7 @@ int sqlite3VdbeFinalize(Vdbe *p){ ** from left to right), or ** ** * the corresponding bit in argument mask is clear (where the first -** function parameter corrsponds to bit 0 etc.). +** function parameter corresponds to bit 0 etc.). */ void sqlite3VdbeDeleteAuxData(Vdbe *pVdbe, int iOp, int mask){ AuxData **pp = &pVdbe->pAuxData; @@ -2669,9 +2722,11 @@ void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){ sqlite3DbFree(db, p->aColName); sqlite3DbFree(db, p->zSql); sqlite3DbFree(db, p->pFree); -#if defined(SQLITE_ENABLE_TREE_EXPLAIN) - sqlite3DbFree(db, p->zExplain); - sqlite3DbFree(db, p->pExplain); +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + for(i=0; inScan; i++){ + sqlite3DbFree(db, p->aScan[i].zName); + } + sqlite3DbFree(db, p->aScan); #endif } @@ -2699,6 +2754,57 @@ void sqlite3VdbeDelete(Vdbe *p){ sqlite3DbFree(db, p); } +/* +** The cursor "p" has a pending seek operation that has not yet been +** carried out. Seek the cursor now. If an error occurs, return +** the appropriate error code. +*/ +static int SQLITE_NOINLINE handleDeferredMoveto(VdbeCursor *p){ + int res, rc; +#ifdef SQLITE_TEST + extern int sqlite3_search_count; +#endif + assert( p->deferredMoveto ); + assert( p->isTable ); + rc = sqlite3BtreeMovetoUnpacked(p->pCursor, 0, p->movetoTarget, 0, &res); + if( rc ) return rc; + if( res!=0 ) return SQLITE_CORRUPT_BKPT; +#ifdef SQLITE_TEST + sqlite3_search_count++; +#endif + p->deferredMoveto = 0; + p->cacheStatus = CACHE_STALE; + return SQLITE_OK; +} + +/* +** Something has moved cursor "p" out of place. Maybe the row it was +** pointed to was deleted out from under it. Or maybe the btree was +** rebalanced. Whatever the cause, try to restore "p" to the place it +** is supposed to be pointing. If the row was deleted out from under the +** cursor, set the cursor to point to a NULL row. +*/ +static int SQLITE_NOINLINE handleMovedCursor(VdbeCursor *p){ + int isDifferentRow, rc; + assert( p->pCursor!=0 ); + assert( sqlite3BtreeCursorHasMoved(p->pCursor) ); + rc = sqlite3BtreeCursorRestore(p->pCursor, &isDifferentRow); + p->cacheStatus = CACHE_STALE; + if( isDifferentRow ) p->nullRow = 1; + return rc; +} + +/* +** Check to ensure that the cursor is valid. Restore the cursor +** if need be. Return any I/O error from the restore operation. +*/ +int sqlite3VdbeCursorRestore(VdbeCursor *p){ + if( sqlite3BtreeCursorHasMoved(p->pCursor) ){ + return handleMovedCursor(p); + } + return SQLITE_OK; +} + /* ** Make sure the cursor p is ready to read or write the row to which it ** was last positioned. Return an error code if an OOM fault or I/O error @@ -2714,29 +2820,10 @@ void sqlite3VdbeDelete(Vdbe *p){ */ int sqlite3VdbeCursorMoveto(VdbeCursor *p){ if( p->deferredMoveto ){ - int res, rc; -#ifdef SQLITE_TEST - extern int sqlite3_search_count; -#endif - assert( p->isTable ); - rc = sqlite3BtreeMovetoUnpacked(p->pCursor, 0, p->movetoTarget, 0, &res); - if( rc ) return rc; - p->lastRowid = p->movetoTarget; - if( res!=0 ) return SQLITE_CORRUPT_BKPT; - p->rowidIsValid = 1; -#ifdef SQLITE_TEST - sqlite3_search_count++; -#endif - p->deferredMoveto = 0; - p->cacheStatus = CACHE_STALE; - }else if( p->pCursor ){ - int hasMoved; - int rc = sqlite3BtreeCursorHasMoved(p->pCursor, &hasMoved); - if( rc ) return rc; - if( hasMoved ){ - p->cacheStatus = CACHE_STALE; - if( hasMoved==2 ) p->nullRow = 1; - } + return handleDeferredMoveto(p); + } + if( p->pCursor && sqlite3BtreeCursorHasMoved(p->pCursor) ){ + return handleMovedCursor(p); } return SQLITE_OK; } @@ -2788,7 +2875,7 @@ int sqlite3VdbeCursorMoveto(VdbeCursor *p){ */ u32 sqlite3VdbeSerialType(Mem *pMem, int file_format){ int flags = pMem->flags; - int n; + u32 n; if( flags&MEM_Null ){ return 0; @@ -2799,9 +2886,7 @@ u32 sqlite3VdbeSerialType(Mem *pMem, int file_format){ i64 i = pMem->u.i; u64 u; if( i<0 ){ - if( i<(-MAX_6BYTE) ) return 6; - /* Previous test prevents: u = -(-9223372036854775808) */ - u = -i; + u = ~i; }else{ u = i; } @@ -2818,11 +2903,11 @@ u32 sqlite3VdbeSerialType(Mem *pMem, int file_format){ return 7; } assert( pMem->db->mallocFailed || flags&(MEM_Str|MEM_Blob) ); - n = pMem->n; + assert( pMem->n>=0 ); + n = (u32)pMem->n; if( flags & MEM_Zero ){ n += pMem->u.nZero; } - assert( n>=0 ); return ((n*2) + 12 + ((flags&MEM_Str)!=0)); } @@ -2912,17 +2997,18 @@ u32 sqlite3VdbeSerialPut(u8 *buf, Mem *pMem, u32 serial_type){ u64 v; u32 i; if( serial_type==7 ){ - assert( sizeof(v)==sizeof(pMem->r) ); - memcpy(&v, &pMem->r, sizeof(v)); + assert( sizeof(v)==sizeof(pMem->u.r) ); + memcpy(&v, &pMem->u.r, sizeof(v)); swapMixedEndianFloat(v); }else{ v = pMem->u.i; } len = i = sqlite3VdbeSerialTypeLen(serial_type); - while( i-- ){ - buf[i] = (u8)(v&0xFF); + assert( i>0 ); + do{ + buf[--i] = (u8)(v&0xFF); v >>= 8; - } + }while( i ); return len; } @@ -2946,51 +3032,101 @@ u32 sqlite3VdbeSerialPut(u8 *buf, Mem *pMem, u32 serial_type){ #define TWO_BYTE_INT(x) (256*(i8)((x)[0])|(x)[1]) #define THREE_BYTE_INT(x) (65536*(i8)((x)[0])|((x)[1]<<8)|(x)[2]) #define FOUR_BYTE_UINT(x) (((u32)(x)[0]<<24)|((x)[1]<<16)|((x)[2]<<8)|(x)[3]) +#define FOUR_BYTE_INT(x) (16777216*(i8)((x)[0])|((x)[1]<<16)|((x)[2]<<8)|(x)[3]) /* ** Deserialize the data blob pointed to by buf as serial type serial_type ** and store the result in pMem. Return the number of bytes read. +** +** This function is implemented as two separate routines for performance. +** The few cases that require local variables are broken out into a separate +** routine so that in most cases the overhead of moving the stack pointer +** is avoided. */ +static u32 SQLITE_NOINLINE serialGet( + const unsigned char *buf, /* Buffer to deserialize from */ + u32 serial_type, /* Serial type to deserialize */ + Mem *pMem /* Memory cell to write value into */ +){ + u64 x = FOUR_BYTE_UINT(buf); + u32 y = FOUR_BYTE_UINT(buf+4); + x = (x<<32) + y; + if( serial_type==6 ){ + /* EVIDENCE-OF: R-29851-52272 Value is a big-endian 64-bit + ** twos-complement integer. */ + pMem->u.i = *(i64*)&x; + pMem->flags = MEM_Int; + testcase( pMem->u.i<0 ); + }else{ + /* EVIDENCE-OF: R-57343-49114 Value is a big-endian IEEE 754-2008 64-bit + ** floating point number. */ +#if !defined(NDEBUG) && !defined(SQLITE_OMIT_FLOATING_POINT) + /* Verify that integers and floating point values use the same + ** byte order. Or, that if SQLITE_MIXED_ENDIAN_64BIT_FLOAT is + ** defined that 64-bit floating point values really are mixed + ** endian. + */ + static const u64 t1 = ((u64)0x3ff00000)<<32; + static const double r1 = 1.0; + u64 t2 = t1; + swapMixedEndianFloat(t2); + assert( sizeof(r1)==sizeof(t2) && memcmp(&r1, &t2, sizeof(r1))==0 ); +#endif + assert( sizeof(x)==8 && sizeof(pMem->u.r)==8 ); + swapMixedEndianFloat(x); + memcpy(&pMem->u.r, &x, sizeof(x)); + pMem->flags = sqlite3IsNaN(pMem->u.r) ? MEM_Null : MEM_Real; + } + return 8; +} u32 sqlite3VdbeSerialGet( const unsigned char *buf, /* Buffer to deserialize from */ u32 serial_type, /* Serial type to deserialize */ Mem *pMem /* Memory cell to write value into */ ){ - u64 x; - u32 y; switch( serial_type ){ case 10: /* Reserved for future use */ case 11: /* Reserved for future use */ - case 0: { /* NULL */ + case 0: { /* Null */ + /* EVIDENCE-OF: R-24078-09375 Value is a NULL. */ pMem->flags = MEM_Null; break; } - case 1: { /* 1-byte signed integer */ + case 1: { + /* EVIDENCE-OF: R-44885-25196 Value is an 8-bit twos-complement + ** integer. */ pMem->u.i = ONE_BYTE_INT(buf); pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); return 1; } case 2: { /* 2-byte signed integer */ + /* EVIDENCE-OF: R-49794-35026 Value is a big-endian 16-bit + ** twos-complement integer. */ pMem->u.i = TWO_BYTE_INT(buf); pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); return 2; } case 3: { /* 3-byte signed integer */ + /* EVIDENCE-OF: R-37839-54301 Value is a big-endian 24-bit + ** twos-complement integer. */ pMem->u.i = THREE_BYTE_INT(buf); pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); return 3; } case 4: { /* 4-byte signed integer */ - y = FOUR_BYTE_UINT(buf); - pMem->u.i = (i64)*(int*)&y; + /* EVIDENCE-OF: R-01849-26079 Value is a big-endian 32-bit + ** twos-complement integer. */ + pMem->u.i = FOUR_BYTE_INT(buf); pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); return 4; } case 5: { /* 6-byte signed integer */ + /* EVIDENCE-OF: R-50385-09674 Value is a big-endian 48-bit + ** twos-complement integer. */ pMem->u.i = FOUR_BYTE_UINT(buf+2) + (((i64)1)<<32)*TWO_BYTE_INT(buf); pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); @@ -2998,52 +3134,32 @@ u32 sqlite3VdbeSerialGet( } case 6: /* 8-byte signed integer */ case 7: { /* IEEE floating point */ -#if !defined(NDEBUG) && !defined(SQLITE_OMIT_FLOATING_POINT) - /* Verify that integers and floating point values use the same - ** byte order. Or, that if SQLITE_MIXED_ENDIAN_64BIT_FLOAT is - ** defined that 64-bit floating point values really are mixed - ** endian. - */ - static const u64 t1 = ((u64)0x3ff00000)<<32; - static const double r1 = 1.0; - u64 t2 = t1; - swapMixedEndianFloat(t2); - assert( sizeof(r1)==sizeof(t2) && memcmp(&r1, &t2, sizeof(r1))==0 ); -#endif - x = FOUR_BYTE_UINT(buf); - y = FOUR_BYTE_UINT(buf+4); - x = (x<<32) | y; - if( serial_type==6 ){ - pMem->u.i = *(i64*)&x; - pMem->flags = MEM_Int; - testcase( pMem->u.i<0 ); - }else{ - assert( sizeof(x)==8 && sizeof(pMem->r)==8 ); - swapMixedEndianFloat(x); - memcpy(&pMem->r, &x, sizeof(x)); - pMem->flags = sqlite3IsNaN(pMem->r) ? MEM_Null : MEM_Real; - } - return 8; + /* These use local variables, so do them in a separate routine + ** to avoid having to move the frame pointer in the common case */ + return serialGet(buf,serial_type,pMem); } case 8: /* Integer 0 */ case 9: { /* Integer 1 */ + /* EVIDENCE-OF: R-12976-22893 Value is the integer 0. */ + /* EVIDENCE-OF: R-18143-12121 Value is the integer 1. */ pMem->u.i = serial_type-8; pMem->flags = MEM_Int; return 0; } default: { + /* EVIDENCE-OF: R-14606-31564 Value is a BLOB that is (N-12)/2 bytes in + ** length. + ** EVIDENCE-OF: R-28401-00140 Value is a string in the text encoding and + ** (N-13)/2 bytes in length. */ static const u16 aFlag[] = { MEM_Blob|MEM_Ephem, MEM_Str|MEM_Ephem }; - u32 len = (serial_type-12)/2; pMem->z = (char *)buf; - pMem->n = len; - pMem->xDel = 0; + pMem->n = (serial_type-12)/2; pMem->flags = aFlag[serial_type&1]; - return len; + return pMem->n; } } return 0; } - /* ** This routine is used to allocate sufficient space for an UnpackedRecord ** structure large enough to be used with sqlite3VdbeRecordUnpack() if @@ -3113,17 +3229,17 @@ void sqlite3VdbeRecordUnpack( idx = getVarint32(aKey, szHdr); d = szHdr; u = 0; - while( idxnField && d<=nKey ){ + while( idxenc = pKeyInfo->enc; pMem->db = pKeyInfo->db; /* pMem->flags = 0; // sqlite3VdbeSerialGet() will set this for us */ - pMem->zMalloc = 0; + pMem->szMalloc = 0; d += sqlite3VdbeSerialGet(&aKey[d], serial_type, pMem); pMem++; - u++; + if( (++u)>=p->nField ) break; } assert( u<=pKeyInfo->nField + 1 ); p->nField = u; @@ -3137,10 +3253,14 @@ void sqlite3VdbeRecordUnpack( ** sqlite3VdbeSerialGet() and sqlite3MemCompare() functions. It is used ** in assert() statements to ensure that the optimized code in ** sqlite3VdbeRecordCompare() returns results with these two primitives. +** +** Return true if the result of comparison is equivalent to desiredResult. +** Return false if there is a disagreement. */ static int vdbeRecordCompareDebug( int nKey1, const void *pKey1, /* Left key */ - const UnpackedRecord *pPKey2 /* Right key */ + const UnpackedRecord *pPKey2, /* Right key */ + int desiredResult /* Correct answer */ ){ u32 d1; /* Offset into aKey[] of next data element */ u32 idx1; /* Offset into aKey[] of next header element */ @@ -3152,10 +3272,11 @@ static int vdbeRecordCompareDebug( Mem mem1; pKeyInfo = pPKey2->pKeyInfo; + if( pKeyInfo->db==0 ) return 1; mem1.enc = pKeyInfo->enc; mem1.db = pKeyInfo->db; /* mem1.flags = 0; // Will be initialized by sqlite3VdbeSerialGet() */ - VVA_ONLY( mem1.zMalloc = 0; ) /* Only needed by assert() statements */ + VVA_ONLY( mem1.szMalloc = 0; ) /* Only needed by assert() statements */ /* Compilers may complain that mem1.u.i is potentially uninitialized. ** We could initialize it, as shown here, to silence those complaints. @@ -3198,11 +3319,11 @@ static int vdbeRecordCompareDebug( */ rc = sqlite3MemCompare(&mem1, &pPKey2->aMem[i], pKeyInfo->aColl[i]); if( rc!=0 ){ - assert( mem1.zMalloc==0 ); /* See comment below */ + assert( mem1.szMalloc==0 ); /* See comment below */ if( pKeyInfo->aSortOrder[i] ){ rc = -rc; /* Invert the result for DESC sort order. */ } - return rc; + goto debugCompareEnd; } i++; }while( idx1nField ); @@ -3211,12 +3332,20 @@ static int vdbeRecordCompareDebug( ** the following assert(). If the assert() fails, it indicates a ** memory leak and a need to call sqlite3VdbeMemRelease(&mem1). */ - assert( mem1.zMalloc==0 ); + assert( mem1.szMalloc==0 ); /* rc==0 here means that one of the keys ran out of fields and - ** all the fields up to that point were equal. Return the the default_rc + ** all the fields up to that point were equal. Return the default_rc ** value. */ - return pPKey2->default_rc; + rc = pPKey2->default_rc; + +debugCompareEnd: + if( desiredResult==0 && rc==0 ) return 1; + if( desiredResult<0 && rc<0 ) return 1; + if( desiredResult>0 && rc>0 ) return 1; + if( CORRUPT_DB ) return 1; + if( pKeyInfo->db->mallocFailed ) return 1; + return 0; } #endif @@ -3229,7 +3358,8 @@ static int vdbeRecordCompareDebug( static int vdbeCompareMemString( const Mem *pMem1, const Mem *pMem2, - const CollSeq *pColl + const CollSeq *pColl, + u8 *prcErr /* If an OOM occurs, set to SQLITE_NOMEM */ ){ if( pMem1->enc==pColl->enc ){ /* The strings are already in the correct encoding. Call the @@ -3241,8 +3371,8 @@ static int vdbeCompareMemString( int n1, n2; Mem c1; Mem c2; - memset(&c1, 0, sizeof(c1)); - memset(&c2, 0, sizeof(c2)); + sqlite3VdbeMemInit(&c1, pMem1->db, MEM_Null); + sqlite3VdbeMemInit(&c2, pMem1->db, MEM_Null); sqlite3VdbeMemShallowCopy(&c1, pMem1, MEM_Ephem); sqlite3VdbeMemShallowCopy(&c2, pMem2, MEM_Ephem); v1 = sqlite3ValueText((sqlite3_value*)&c1, pColl->enc); @@ -3252,10 +3382,23 @@ static int vdbeCompareMemString( rc = pColl->xCmp(pColl->pUser, n1, v1, n2, v2); sqlite3VdbeMemRelease(&c1); sqlite3VdbeMemRelease(&c2); + if( (v1==0 || v2==0) && prcErr ) *prcErr = SQLITE_NOMEM; return rc; } } +/* +** Compare two blobs. Return negative, zero, or positive if the first +** is less than, equal to, or greater than the second, respectively. +** If one blob is a prefix of the other, then the shorter is the lessor. +*/ +static SQLITE_NOINLINE int sqlite3BlobCompare(const Mem *pB1, const Mem *pB2){ + int c = memcmp(pB1->z, pB2->z, pB1->n>pB2->n ? pB2->n : pB1->n); + if( c ) return c; + return pB1->n - pB2->n; +} + + /* ** Compare the values contained by the two memory cells, returning ** negative, zero or positive if pMem1 is less than, equal to, or greater @@ -3266,7 +3409,6 @@ static int vdbeCompareMemString( ** Two NULL values are considered equal by this function. */ int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const CollSeq *pColl){ - int rc; int f1, f2; int combined_flags; @@ -3294,14 +3436,14 @@ int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const CollSeq *pColl){ return 0; } if( (f1&MEM_Real)!=0 ){ - r1 = pMem1->r; + r1 = pMem1->u.r; }else if( (f1&MEM_Int)!=0 ){ r1 = (double)pMem1->u.i; }else{ return 1; } if( (f2&MEM_Real)!=0 ){ - r2 = pMem2->r; + r2 = pMem2->u.r; }else if( (f2&MEM_Int)!=0 ){ r2 = (double)pMem2->u.i; }else{ @@ -3334,18 +3476,14 @@ int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const CollSeq *pColl){ assert( !pColl || pColl->xCmp ); if( pColl ){ - return vdbeCompareMemString(pMem1, pMem2, pColl); + return vdbeCompareMemString(pMem1, pMem2, pColl, 0); } /* If a NULL pointer was passed as the collate function, fall through ** to the blob case and use memcmp(). */ } /* Both values must be blobs. Compare using memcmp(). */ - rc = memcmp(pMem1->z, pMem2->z, (pMem1->n>pMem2->n)?pMem2->n:pMem1->n); - if( rc==0 ){ - rc = pMem1->n - pMem2->n; - } - return rc; + return sqlite3BlobCompare(pMem1, pMem2); } @@ -3395,7 +3533,7 @@ static i64 vdbeRecordDecodeInt(u32 serial_type, const u8 *aKey){ ** specified by {nKey1, pKey1} and pPKey2. It returns a negative, zero ** or positive integer if key1 is less than, equal to or ** greater than key2. The {nKey1, pKey1} key must be a blob -** created by th OP_MakeRecord opcode of the VDBE. The pPKey2 +** created by the OP_MakeRecord opcode of the VDBE. The pPKey2 ** key must be a parsed key such as obtained from ** sqlite3VdbeParseRecord. ** @@ -3406,10 +3544,12 @@ static i64 vdbeRecordDecodeInt(u32 serial_type, const u8 *aKey){ ** fields that appear in both keys are equal, then pPKey2->default_rc is ** returned. ** -** If database corruption is discovered, set pPKey2->isCorrupt to non-zero -** and return 0. +** If database corruption is discovered, set pPKey2->errCode to +** SQLITE_CORRUPT and return 0. If an OOM error is encountered, +** pPKey2->errCode is set to SQLITE_NOMEM and, if it is not NULL, the +** malloc-failed flag set on database handle (pPKey2->pKeyInfo->db). */ -int sqlite3VdbeRecordCompare( +static int vdbeRecordCompareWithSkip( int nKey1, const void *pKey1, /* Left key */ UnpackedRecord *pPKey2, /* Right key */ int bSkip /* If true, skip the first field */ @@ -3438,13 +3578,13 @@ int sqlite3VdbeRecordCompare( idx1 = getVarint32(aKey1, szHdr1); d1 = szHdr1; if( d1>(unsigned)nKey1 ){ - pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT; + pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; return 0; /* Corruption */ } i = 0; } - VVA_ONLY( mem1.zMalloc = 0; ) /* Only needed by assert() statements */ + VVA_ONLY( mem1.szMalloc = 0; ) /* Only needed by assert() statements */ assert( pPKey2->pKeyInfo->nField+pPKey2->pKeyInfo->nXField>=pPKey2->nField || CORRUPT_DB ); assert( pPKey2->pKeyInfo->aSortOrder!=0 ); @@ -3464,9 +3604,9 @@ int sqlite3VdbeRecordCompare( }else if( serial_type==7 ){ double rhs = (double)pRhs->u.i; sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1); - if( mem1.rrhs ){ + }else if( mem1.u.r>rhs ){ rc = +1; } }else{ @@ -3488,11 +3628,11 @@ int sqlite3VdbeRecordCompare( }else if( serial_type==0 ){ rc = -1; }else{ - double rhs = pRhs->r; + double rhs = pRhs->u.r; double lhs; sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1); if( serial_type==7 ){ - lhs = mem1.r; + lhs = mem1.u.r; }else{ lhs = (double)mem1.u.i; } @@ -3517,14 +3657,16 @@ int sqlite3VdbeRecordCompare( testcase( (d1+mem1.n)==(unsigned)nKey1 ); testcase( (d1+mem1.n+1)==(unsigned)nKey1 ); if( (d1+mem1.n) > (unsigned)nKey1 ){ - pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT; + pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; return 0; /* Corruption */ }else if( pKeyInfo->aColl[i] ){ mem1.enc = pKeyInfo->enc; mem1.db = pKeyInfo->db; mem1.flags = MEM_Str; mem1.z = (char*)&aKey1[d1]; - rc = vdbeCompareMemString(&mem1, pRhs, pKeyInfo->aColl[i]); + rc = vdbeCompareMemString( + &mem1, pRhs, pKeyInfo->aColl[i], &pPKey2->errCode + ); }else{ int nCmp = MIN(mem1.n, pRhs->n); rc = memcmp(&aKey1[d1], pRhs->z, nCmp); @@ -3544,7 +3686,7 @@ int sqlite3VdbeRecordCompare( testcase( (d1+nStr)==(unsigned)nKey1 ); testcase( (d1+nStr+1)==(unsigned)nKey1 ); if( (d1+nStr) > (unsigned)nKey1 ){ - pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT; + pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; return 0; /* Corruption */ }else{ int nCmp = MIN(nStr, pRhs->n); @@ -3564,12 +3706,8 @@ int sqlite3VdbeRecordCompare( if( pKeyInfo->aSortOrder[i] ){ rc = -rc; } - assert( CORRUPT_DB - || (rc<0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)<0) - || (rc>0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)>0) - || pKeyInfo->db->mallocFailed - ); - assert( mem1.zMalloc==0 ); /* See comment below */ + assert( vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, rc) ); + assert( mem1.szMalloc==0 ); /* See comment below */ return rc; } @@ -3582,16 +3720,24 @@ int sqlite3VdbeRecordCompare( /* No memory allocation is ever used on mem1. Prove this using ** the following assert(). If the assert() fails, it indicates a ** memory leak and a need to call sqlite3VdbeMemRelease(&mem1). */ - assert( mem1.zMalloc==0 ); + assert( mem1.szMalloc==0 ); /* rc==0 here means that one or both of the keys ran out of fields and - ** all the fields up to that point were equal. Return the the default_rc + ** all the fields up to that point were equal. Return the default_rc ** value. */ assert( CORRUPT_DB - || pPKey2->default_rc==vdbeRecordCompareDebug(nKey1, pKey1, pPKey2) + || vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, pPKey2->default_rc) + || pKeyInfo->db->mallocFailed ); return pPKey2->default_rc; } +int sqlite3VdbeRecordCompare( + int nKey1, const void *pKey1, /* Left key */ + UnpackedRecord *pPKey2 /* Right key */ +){ + return vdbeRecordCompareWithSkip(nKey1, pKey1, pPKey2, 0); +} + /* ** This function is an optimized version of sqlite3VdbeRecordCompare() @@ -3604,8 +3750,7 @@ int sqlite3VdbeRecordCompare( */ static int vdbeRecordCompareInt( int nKey1, const void *pKey1, /* Left key */ - UnpackedRecord *pPKey2, /* Right key */ - int bSkip /* Ignored */ + UnpackedRecord *pPKey2 /* Right key */ ){ const u8 *aKey = &((const u8*)pKey1)[*(const u8*)pKey1 & 0x3F]; int serial_type = ((const u8*)pKey1)[1]; @@ -3614,9 +3759,7 @@ static int vdbeRecordCompareInt( u64 x; i64 v = pPKey2->aMem[0].u.i; i64 lhs; - UNUSED_PARAMETER(bSkip); - assert( bSkip==0 ); assert( (*(u8*)pKey1)<=0x3F || CORRUPT_DB ); switch( serial_type ){ case 1: { /* 1-byte signed integer */ @@ -3666,10 +3809,10 @@ static int vdbeRecordCompareInt( ** (as gcc is clever enough to combine the two like cases). Other ** compilers might be similar. */ case 0: case 7: - return sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2, 0); + return sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2); default: - return sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2, 0); + return sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2); } if( v>lhs ){ @@ -3679,18 +3822,14 @@ static int vdbeRecordCompareInt( }else if( pPKey2->nField>1 ){ /* The first fields of the two keys are equal. Compare the trailing ** fields. */ - res = sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2, 1); + res = vdbeRecordCompareWithSkip(nKey1, pKey1, pPKey2, 1); }else{ /* The first fields of the two keys are equal and there are no trailing ** fields. Return pPKey2->default_rc in this case. */ res = pPKey2->default_rc; } - assert( (res==0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)==0) - || (res<0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)<0) - || (res>0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)>0) - || CORRUPT_DB - ); + assert( vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, res) ); return res; } @@ -3702,17 +3841,13 @@ static int vdbeRecordCompareInt( */ static int vdbeRecordCompareString( int nKey1, const void *pKey1, /* Left key */ - UnpackedRecord *pPKey2, /* Right key */ - int bSkip + UnpackedRecord *pPKey2 /* Right key */ ){ const u8 *aKey1 = (const u8*)pKey1; int serial_type; int res; - UNUSED_PARAMETER(bSkip); - assert( bSkip==0 ); getVarint32(&aKey1[1], serial_type); - if( serial_type<12 ){ res = pPKey2->r1; /* (pKey1/nKey1) is a number or a null */ }else if( !(serial_type & 0x01) ){ @@ -3724,7 +3859,7 @@ static int vdbeRecordCompareString( nStr = (serial_type-12) / 2; if( (szHdr + nStr) > nKey1 ){ - pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT; + pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; return 0; /* Corruption */ } nCmp = MIN( pPKey2->aMem[0].n, nStr ); @@ -3734,7 +3869,7 @@ static int vdbeRecordCompareString( res = nStr - pPKey2->aMem[0].n; if( res==0 ){ if( pPKey2->nField>1 ){ - res = sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2, 1); + res = vdbeRecordCompareWithSkip(nKey1, pKey1, pPKey2, 1); }else{ res = pPKey2->default_rc; } @@ -3750,10 +3885,9 @@ static int vdbeRecordCompareString( } } - assert( (res==0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)==0) - || (res<0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)<0) - || (res>0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)>0) + assert( vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, res) || CORRUPT_DB + || pPKey2->pKeyInfo->db->mallocFailed ); return res; } @@ -3817,8 +3951,6 @@ int sqlite3VdbeIdxRowid(sqlite3 *db, BtCursor *pCur, i64 *rowid){ u32 lenRowid; /* Size of the rowid */ Mem m, v; - UNUSED_PARAMETER(db); - /* Get the size of the index entry. Only indices entries of less ** than 2GiB are support - anything large must be database corruption. ** Any corruption is detected in sqlite3BtreeParseCellPtr(), though, so @@ -3830,7 +3962,7 @@ int sqlite3VdbeIdxRowid(sqlite3 *db, BtCursor *pCur, i64 *rowid){ assert( (nCellKey & SQLITE_MAX_U32)==(u64)nCellKey ); /* Read in the complete content of the index entry */ - memset(&m, 0, sizeof(m)); + sqlite3VdbeMemInit(&m, db, 0); rc = sqlite3VdbeMemFromBtree(pCur, 0, (u32)nCellKey, 1, &m); if( rc ){ return rc; @@ -3873,7 +4005,7 @@ int sqlite3VdbeIdxRowid(sqlite3 *db, BtCursor *pCur, i64 *rowid){ /* Jump here if database corruption is detected after m has been ** allocated. Free the m object and return SQLITE_CORRUPT. */ idx_rowid_corruption: - testcase( m.zMalloc!=0 ); + testcase( m.szMalloc!=0 ); sqlite3VdbeMemRelease(&m); return SQLITE_CORRUPT_BKPT; } @@ -3890,6 +4022,7 @@ idx_rowid_corruption: ** of the keys prior to the final rowid, not the entire key. */ int sqlite3VdbeIdxKeyCompare( + sqlite3 *db, /* Database connection */ VdbeCursor *pC, /* The cursor to compare against */ UnpackedRecord *pUnpacked, /* Unpacked version of key */ int *res /* Write the comparison result here */ @@ -3908,12 +4041,12 @@ int sqlite3VdbeIdxKeyCompare( *res = 0; return SQLITE_CORRUPT_BKPT; } - memset(&m, 0, sizeof(m)); + sqlite3VdbeMemInit(&m, db, 0); rc = sqlite3VdbeMemFromBtree(pC->pCursor, 0, (u32)nCellKey, 1, &m); if( rc ){ return rc; } - *res = sqlite3VdbeRecordCompare(m.n, m.z, pUnpacked, 0); + *res = sqlite3VdbeRecordCompare(m.n, m.z, pUnpacked); sqlite3VdbeMemRelease(&m); return SQLITE_OK; } diff --git a/src/vdbeblob.c b/src/vdbeblob.c index 083f3f413c..cf1eb59054 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -153,6 +153,11 @@ int sqlite3_blob_open( Parse *pParse = 0; Incrblob *pBlob = 0; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || ppBlob==0 || zTable==0 ){ + return SQLITE_MISUSE_BKPT; + } +#endif flags = !!flags; /* flags = (flags ? 1 : 0); */ *ppBlob = 0; @@ -318,7 +323,7 @@ blob_open_out: if( pBlob && pBlob->pStmt ) sqlite3VdbeFinalize((Vdbe *)pBlob->pStmt); sqlite3DbFree(db, pBlob); } - sqlite3Error(db, rc, (zErr ? "%s" : 0), zErr); + sqlite3ErrorWithMsg(db, rc, (zErr ? "%s" : 0), zErr); sqlite3DbFree(db, zErr); sqlite3ParserReset(pParse); sqlite3StackFree(db, pParse); @@ -371,7 +376,6 @@ static int blobReadWrite( if( n<0 || iOffset<0 || (iOffset+n)>p->nByte ){ /* Request is out of range. Return a transient error. */ rc = SQLITE_ERROR; - sqlite3Error(db, SQLITE_ERROR, 0); }else if( v==0 ){ /* If there is no statement handle, then the blob-handle has ** already been invalidated. Return SQLITE_ABORT in this case. @@ -389,10 +393,10 @@ static int blobReadWrite( sqlite3VdbeFinalize(v); p->pStmt = 0; }else{ - db->errCode = rc; v->rc = rc; } } + sqlite3Error(db, rc); rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); return rc; @@ -451,7 +455,7 @@ int sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){ char *zErr; rc = blobSeekToRow(p, iRow, &zErr); if( rc!=SQLITE_OK ){ - sqlite3Error(db, rc, (zErr ? "%s" : 0), zErr); + sqlite3ErrorWithMsg(db, rc, (zErr ? "%s" : 0), zErr); sqlite3DbFree(db, zErr); } assert( rc!=SQLITE_SCHEMA ); diff --git a/src/vdbemem.c b/src/vdbemem.c index 2c4aa4ad7f..870fb5bd89 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -26,29 +26,40 @@ ** this: assert( sqlite3VdbeCheckMemInvariants(pMem) ); */ int sqlite3VdbeCheckMemInvariants(Mem *p){ - /* The MEM_Dyn bit is set if and only if Mem.xDel is a non-NULL destructor - ** function for Mem.z + /* If MEM_Dyn is set then Mem.xDel!=0. + ** Mem.xDel is might not be initialized if MEM_Dyn is clear. */ assert( (p->flags & MEM_Dyn)==0 || p->xDel!=0 ); - assert( (p->flags & MEM_Dyn)!=0 || p->xDel==0 ); + + /* MEM_Dyn may only be set if Mem.szMalloc==0. In this way we + ** ensure that if Mem.szMalloc>0 then it is safe to do + ** Mem.z = Mem.zMalloc without having to check Mem.flags&MEM_Dyn. + ** That saves a few cycles in inner loops. */ + assert( (p->flags & MEM_Dyn)==0 || p->szMalloc==0 ); + + /* Cannot be both MEM_Int and MEM_Real at the same time */ + assert( (p->flags & (MEM_Int|MEM_Real))!=(MEM_Int|MEM_Real) ); + + /* The szMalloc field holds the correct memory allocation size */ + assert( p->szMalloc==0 + || p->szMalloc==sqlite3DbMallocSize(p->db,p->zMalloc) ); /* If p holds a string or blob, the Mem.z must point to exactly ** one of the following: ** ** (1) Memory in Mem.zMalloc and managed by the Mem object ** (2) Memory to be freed using Mem.xDel - ** (3) An ephermal string or blob + ** (3) An ephemeral string or blob ** (4) A static string or blob */ - if( (p->flags & (MEM_Str|MEM_Blob)) && p->z!=0 ){ + if( (p->flags & (MEM_Str|MEM_Blob)) && p->n>0 ){ assert( - ((p->z==p->zMalloc)? 1 : 0) + + ((p->szMalloc>0 && p->z==p->zMalloc)? 1 : 0) + ((p->flags&MEM_Dyn)!=0 ? 1 : 0) + ((p->flags&MEM_Ephem)!=0 ? 1 : 0) + ((p->flags&MEM_Static)!=0 ? 1 : 0) == 1 ); } - return 1; } #endif @@ -102,7 +113,7 @@ int sqlite3VdbeChangeEncoding(Mem *pMem, int desiredEnc){ ** blob if bPreserve is true. If bPreserve is false, any prior content ** in pMem->z is discarded. */ -int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){ +SQLITE_NOINLINE int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){ assert( sqlite3VdbeCheckMemInvariants(pMem) ); assert( (pMem->flags&MEM_RowSet)==0 ); @@ -111,24 +122,28 @@ int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){ assert( bPreserve==0 || pMem->flags&(MEM_Blob|MEM_Str) ); testcase( bPreserve && pMem->z==0 ); - if( pMem->zMalloc==0 || sqlite3DbMallocSize(pMem->db, pMem->zMalloc)szMalloc==0 + || pMem->szMalloc==sqlite3DbMallocSize(pMem->db, pMem->zMalloc) ); + if( pMem->szMallocz==pMem->zMalloc ){ + if( bPreserve && pMem->szMalloc>0 && pMem->z==pMem->zMalloc ){ pMem->z = pMem->zMalloc = sqlite3DbReallocOrFree(pMem->db, pMem->z, n); bPreserve = 0; }else{ - sqlite3DbFree(pMem->db, pMem->zMalloc); + if( pMem->szMalloc>0 ) sqlite3DbFree(pMem->db, pMem->zMalloc); pMem->zMalloc = sqlite3DbMallocRaw(pMem->db, n); } if( pMem->zMalloc==0 ){ - VdbeMemRelease(pMem); + sqlite3VdbeMemSetNull(pMem); pMem->z = 0; - pMem->flags = MEM_Null; + pMem->szMalloc = 0; return SQLITE_NOMEM; + }else{ + pMem->szMalloc = sqlite3DbMallocSize(pMem->db, pMem->zMalloc); } } - if( pMem->z && bPreserve && pMem->z!=pMem->zMalloc ){ + if( bPreserve && pMem->z && pMem->z!=pMem->zMalloc ){ memcpy(pMem->zMalloc, pMem->z, pMem->n); } if( (pMem->flags&MEM_Dyn)!=0 ){ @@ -138,15 +153,37 @@ int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){ pMem->z = pMem->zMalloc; pMem->flags &= ~(MEM_Dyn|MEM_Ephem|MEM_Static); - pMem->xDel = 0; return SQLITE_OK; } /* -** Make the given Mem object MEM_Dyn. In other words, make it so -** that any TEXT or BLOB content is stored in memory obtained from -** malloc(). In this way, we know that the memory is safe to be -** overwritten or altered. +** Change the pMem->zMalloc allocation to be at least szNew bytes. +** If pMem->zMalloc already meets or exceeds the requested size, this +** routine is a no-op. +** +** Any prior string or blob content in the pMem object may be discarded. +** The pMem->xDel destructor is called, if it exists. Though MEM_Str +** and MEM_Blob values may be discarded, MEM_Int, MEM_Real, and MEM_Null +** values are preserved. +** +** Return SQLITE_OK on success or an error code (probably SQLITE_NOMEM) +** if unable to complete the resizing. +*/ +int sqlite3VdbeMemClearAndResize(Mem *pMem, int szNew){ + assert( szNew>0 ); + assert( (pMem->flags & MEM_Dyn)==0 || pMem->szMalloc==0 ); + if( pMem->szMallocflags & MEM_Dyn)==0 ); + pMem->z = pMem->zMalloc; + pMem->flags &= (MEM_Null|MEM_Int|MEM_Real); + return SQLITE_OK; +} + +/* +** Change pMem so that its MEM_Str or MEM_Blob value is stored in +** MEM.zMalloc, where it can be safely written. ** ** Return SQLITE_OK on success or SQLITE_NOMEM if malloc fails. */ @@ -156,7 +193,7 @@ int sqlite3VdbeMemMakeWriteable(Mem *pMem){ assert( (pMem->flags&MEM_RowSet)==0 ); ExpandBlob(pMem); f = pMem->flags; - if( (f&(MEM_Str|MEM_Blob)) && pMem->z!=pMem->zMalloc ){ + if( (f&(MEM_Str|MEM_Blob)) && (pMem->szMalloc==0 || pMem->z!=pMem->zMalloc) ){ if( sqlite3VdbeMemGrow(pMem, pMem->n + 2, 1) ){ return SQLITE_NOMEM; } @@ -200,15 +237,11 @@ int sqlite3VdbeMemExpandBlob(Mem *pMem){ } #endif - /* -** Make sure the given Mem is \u0000 terminated. +** It is already known that pMem contains an unterminated string. +** Add the zero terminator. */ -int sqlite3VdbeMemNulTerminate(Mem *pMem){ - assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - if( (pMem->flags & MEM_Term)!=0 || (pMem->flags & MEM_Str)==0 ){ - return SQLITE_OK; /* Nothing to do */ - } +static SQLITE_NOINLINE int vdbeMemAddTerminator(Mem *pMem){ if( sqlite3VdbeMemGrow(pMem, pMem->n+2, 1) ){ return SQLITE_NOMEM; } @@ -218,21 +251,35 @@ int sqlite3VdbeMemNulTerminate(Mem *pMem){ return SQLITE_OK; } +/* +** Make sure the given Mem is \u0000 terminated. +*/ +int sqlite3VdbeMemNulTerminate(Mem *pMem){ + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + testcase( (pMem->flags & (MEM_Term|MEM_Str))==(MEM_Term|MEM_Str) ); + testcase( (pMem->flags & (MEM_Term|MEM_Str))==0 ); + if( (pMem->flags & (MEM_Term|MEM_Str))!=MEM_Str ){ + return SQLITE_OK; /* Nothing to do */ + }else{ + return vdbeMemAddTerminator(pMem); + } +} + /* ** Add MEM_Str to the set of representations for the given Mem. Numbers ** are converted using sqlite3_snprintf(). Converting a BLOB to a string ** is a no-op. ** -** Existing representations MEM_Int and MEM_Real are *not* invalidated. +** Existing representations MEM_Int and MEM_Real are invalidated if +** bForce is true but are retained if bForce is false. ** ** A MEM_Null value will never be passed to this function. This function is ** used for converting values to text for returning to the user (i.e. via ** sqlite3_value_text()), or for ensuring that values to be used as btree ** keys are strings. In the former case a NULL pointer is returned the -** user and the later is an internal programming error. +** user and the latter is an internal programming error. */ -int sqlite3VdbeMemStringify(Mem *pMem, int enc){ - int rc = SQLITE_OK; +int sqlite3VdbeMemStringify(Mem *pMem, u8 enc, u8 bForce){ int fg = pMem->flags; const int nByte = 32; @@ -244,11 +291,11 @@ int sqlite3VdbeMemStringify(Mem *pMem, int enc){ assert( EIGHT_BYTE_ALIGNMENT(pMem) ); - if( sqlite3VdbeMemGrow(pMem, nByte, 0) ){ + if( sqlite3VdbeMemClearAndResize(pMem, nByte) ){ return SQLITE_NOMEM; } - /* For a Real or Integer, use sqlite3_mprintf() to produce the UTF-8 + /* For a Real or Integer, use sqlite3_snprintf() to produce the UTF-8 ** string representation of the value. Then, if the required encoding ** is UTF-16le or UTF-16be do a translation. ** @@ -258,13 +305,14 @@ int sqlite3VdbeMemStringify(Mem *pMem, int enc){ sqlite3_snprintf(nByte, pMem->z, "%lld", pMem->u.i); }else{ assert( fg & MEM_Real ); - sqlite3_snprintf(nByte, pMem->z, "%!.15g", pMem->r); + sqlite3_snprintf(nByte, pMem->z, "%!.15g", pMem->u.r); } pMem->n = sqlite3Strlen30(pMem->z); pMem->enc = SQLITE_UTF8; pMem->flags |= MEM_Str|MEM_Term; + if( bForce ) pMem->flags &= ~(MEM_Int|MEM_Real); sqlite3VdbeChangeEncoding(pMem, enc); - return rc; + return SQLITE_OK; } /* @@ -279,59 +327,90 @@ int sqlite3VdbeMemFinalize(Mem *pMem, FuncDef *pFunc){ int rc = SQLITE_OK; if( ALWAYS(pFunc && pFunc->xFinalize) ){ sqlite3_context ctx; + Mem t; assert( (pMem->flags & MEM_Null)!=0 || pFunc==pMem->u.pDef ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); memset(&ctx, 0, sizeof(ctx)); - ctx.s.flags = MEM_Null; - ctx.s.db = pMem->db; + memset(&t, 0, sizeof(t)); + t.flags = MEM_Null; + t.db = pMem->db; + ctx.pOut = &t; ctx.pMem = pMem; ctx.pFunc = pFunc; pFunc->xFinalize(&ctx); /* IMP: R-24505-23230 */ - assert( 0==(pMem->flags&MEM_Dyn) && !pMem->xDel ); - sqlite3DbFree(pMem->db, pMem->zMalloc); - memcpy(pMem, &ctx.s, sizeof(ctx.s)); + assert( (pMem->flags & MEM_Dyn)==0 ); + if( pMem->szMalloc>0 ) sqlite3DbFree(pMem->db, pMem->zMalloc); + memcpy(pMem, &t, sizeof(t)); rc = ctx.isError; } return rc; } /* -** If the memory cell contains a string value that must be freed by -** invoking an external callback, free it now. Calling this function -** does not free any Mem.zMalloc buffer. +** If the memory cell contains a value that must be freed by +** invoking the external callback in Mem.xDel, then this routine +** will free that value. It also sets Mem.flags to MEM_Null. +** +** This is a helper routine for sqlite3VdbeMemSetNull() and +** for sqlite3VdbeMemRelease(). Use those other routines as the +** entry point for releasing Mem resources. */ -void sqlite3VdbeMemReleaseExternal(Mem *p){ +static SQLITE_NOINLINE void vdbeMemClearExternAndSetNull(Mem *p){ assert( p->db==0 || sqlite3_mutex_held(p->db->mutex) ); + assert( VdbeMemDynamic(p) ); if( p->flags&MEM_Agg ){ sqlite3VdbeMemFinalize(p, p->u.pDef); assert( (p->flags & MEM_Agg)==0 ); - sqlite3VdbeMemRelease(p); - }else if( p->flags&MEM_Dyn ){ + testcase( p->flags & MEM_Dyn ); + } + if( p->flags&MEM_Dyn ){ assert( (p->flags&MEM_RowSet)==0 ); assert( p->xDel!=SQLITE_DYNAMIC && p->xDel!=0 ); p->xDel((void *)p->z); - p->xDel = 0; }else if( p->flags&MEM_RowSet ){ sqlite3RowSetClear(p->u.pRowSet); }else if( p->flags&MEM_Frame ){ - sqlite3VdbeMemSetNull(p); + VdbeFrame *pFrame = p->u.pFrame; + pFrame->pParent = pFrame->v->pDelFrame; + pFrame->v->pDelFrame = pFrame; } + p->flags = MEM_Null; } /* -** Release any memory held by the Mem. This may leave the Mem in an -** inconsistent state, for example with (Mem.z==0) and -** (Mem.flags==MEM_Str). +** Release memory held by the Mem p, both external memory cleared +** by p->xDel and memory in p->zMalloc. +** +** This is a helper routine invoked by sqlite3VdbeMemRelease() in +** the unusual case where there really is memory in p that needs +** to be freed. +*/ +static SQLITE_NOINLINE void vdbeMemClear(Mem *p){ + if( VdbeMemDynamic(p) ){ + vdbeMemClearExternAndSetNull(p); + } + if( p->szMalloc ){ + sqlite3DbFree(p->db, p->zMalloc); + p->szMalloc = 0; + } + p->z = 0; +} + +/* +** Release any memory resources held by the Mem. Both the memory that is +** free by Mem.xDel and the Mem.zMalloc allocation are freed. +** +** Use this routine prior to clean up prior to abandoning a Mem, or to +** reset a Mem back to its minimum memory utilization. +** +** Use sqlite3VdbeMemSetNull() to release just the Mem.xDel space +** prior to inserting new content into the Mem. */ void sqlite3VdbeMemRelease(Mem *p){ assert( sqlite3VdbeCheckMemInvariants(p) ); - VdbeMemRelease(p); - if( p->zMalloc ){ - sqlite3DbFree(p->db, p->zMalloc); - p->zMalloc = 0; + if( VdbeMemDynamic(p) || p->szMalloc ){ + vdbeMemClear(p); } - p->z = 0; - assert( p->xDel==0 ); /* Zeroed by VdbeMemRelease() above */ } /* @@ -370,7 +449,7 @@ static i64 doubleToInt64(double r){ ** If pMem is an integer, then the value is exact. If pMem is ** a floating-point then the value returned is the integer part. ** If pMem is a string or blob, then we make an attempt to convert -** it into a integer and return that. If pMem represents an +** it into an integer and return that. If pMem represents an ** an SQL-NULL value, return 0. ** ** If pMem represents a string value, its encoding might be changed. @@ -383,11 +462,10 @@ i64 sqlite3VdbeIntValue(Mem *pMem){ if( flags & MEM_Int ){ return pMem->u.i; }else if( flags & MEM_Real ){ - return doubleToInt64(pMem->r); + return doubleToInt64(pMem->u.r); }else if( flags & (MEM_Str|MEM_Blob) ){ i64 value = 0; assert( pMem->z || pMem->n==0 ); - testcase( pMem->z==0 ); sqlite3Atoi64(pMem->z, &value, pMem->n, pMem->enc); return value; }else{ @@ -405,7 +483,7 @@ double sqlite3VdbeRealValue(Mem *pMem){ assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); assert( EIGHT_BYTE_ALIGNMENT(pMem) ); if( pMem->flags & MEM_Real ){ - return pMem->r; + return pMem->u.r; }else if( pMem->flags & MEM_Int ){ return (double)pMem->u.i; }else if( pMem->flags & (MEM_Str|MEM_Blob) ){ @@ -424,12 +502,13 @@ double sqlite3VdbeRealValue(Mem *pMem){ ** MEM_Int if we can. */ void sqlite3VdbeIntegerAffinity(Mem *pMem){ + i64 ix; assert( pMem->flags & MEM_Real ); assert( (pMem->flags & MEM_RowSet)==0 ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); assert( EIGHT_BYTE_ALIGNMENT(pMem) ); - pMem->u.i = doubleToInt64(pMem->r); + ix = doubleToInt64(pMem->u.r); /* Only mark the value as an integer if ** @@ -441,11 +520,9 @@ void sqlite3VdbeIntegerAffinity(Mem *pMem){ ** the second condition under the assumption that addition overflow causes ** values to wrap around. */ - if( pMem->r==(double)pMem->u.i - && pMem->u.i>SMALLEST_INT64 - && pMem->u.iflags |= MEM_Int; + if( pMem->u.r==ix && ix>SMALLEST_INT64 && ixu.i = ix; + MemSetTypeFlag(pMem, MEM_Int); } } @@ -470,7 +547,7 @@ int sqlite3VdbeMemRealify(Mem *pMem){ assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); assert( EIGHT_BYTE_ALIGNMENT(pMem) ); - pMem->r = sqlite3VdbeRealValue(pMem); + pMem->u.r = sqlite3VdbeRealValue(pMem); MemSetTypeFlag(pMem, MEM_Real); return SQLITE_OK; } @@ -490,7 +567,7 @@ int sqlite3VdbeMemNumerify(Mem *pMem){ if( 0==sqlite3Atoi64(pMem->z, &pMem->u.i, pMem->n, pMem->enc) ){ MemSetTypeFlag(pMem, MEM_Int); }else{ - pMem->r = sqlite3VdbeRealValue(pMem); + pMem->u.r = sqlite3VdbeRealValue(pMem); MemSetTypeFlag(pMem, MEM_Real); sqlite3VdbeIntegerAffinity(pMem); } @@ -500,19 +577,81 @@ int sqlite3VdbeMemNumerify(Mem *pMem){ return SQLITE_OK; } +/* +** Cast the datatype of the value in pMem according to the affinity +** "aff". Casting is different from applying affinity in that a cast +** is forced. In other words, the value is converted into the desired +** affinity even if that results in loss of data. This routine is +** used (for example) to implement the SQL "cast()" operator. +*/ +void sqlite3VdbeMemCast(Mem *pMem, u8 aff, u8 encoding){ + if( pMem->flags & MEM_Null ) return; + switch( aff ){ + case SQLITE_AFF_NONE: { /* Really a cast to BLOB */ + if( (pMem->flags & MEM_Blob)==0 ){ + sqlite3ValueApplyAffinity(pMem, SQLITE_AFF_TEXT, encoding); + assert( pMem->flags & MEM_Str || pMem->db->mallocFailed ); + MemSetTypeFlag(pMem, MEM_Blob); + }else{ + pMem->flags &= ~(MEM_TypeMask&~MEM_Blob); + } + break; + } + case SQLITE_AFF_NUMERIC: { + sqlite3VdbeMemNumerify(pMem); + break; + } + case SQLITE_AFF_INTEGER: { + sqlite3VdbeMemIntegerify(pMem); + break; + } + case SQLITE_AFF_REAL: { + sqlite3VdbeMemRealify(pMem); + break; + } + default: { + assert( aff==SQLITE_AFF_TEXT ); + assert( MEM_Str==(MEM_Blob>>3) ); + pMem->flags |= (pMem->flags&MEM_Blob)>>3; + sqlite3ValueApplyAffinity(pMem, SQLITE_AFF_TEXT, encoding); + assert( pMem->flags & MEM_Str || pMem->db->mallocFailed ); + pMem->flags &= ~(MEM_Int|MEM_Real|MEM_Blob|MEM_Zero); + break; + } + } +} + +/* +** Initialize bulk memory to be a consistent Mem object. +** +** The minimum amount of initialization feasible is performed. +*/ +void sqlite3VdbeMemInit(Mem *pMem, sqlite3 *db, u16 flags){ + assert( (flags & ~MEM_TypeMask)==0 ); + pMem->flags = flags; + pMem->db = db; + pMem->szMalloc = 0; +} + + /* ** Delete any previous value and set the value stored in *pMem to NULL. +** +** This routine calls the Mem.xDel destructor to dispose of values that +** require the destructor. But it preserves the Mem.zMalloc memory allocation. +** To free all resources, use sqlite3VdbeMemRelease(), which both calls this +** routine to invoke the destructor and deallocates Mem.zMalloc. +** +** Use this routine to reset the Mem prior to insert a new value. +** +** Use sqlite3VdbeMemRelease() to complete erase the Mem prior to abandoning it. */ void sqlite3VdbeMemSetNull(Mem *pMem){ - if( pMem->flags & MEM_Frame ){ - VdbeFrame *pFrame = pMem->u.pFrame; - pFrame->pParent = pFrame->v->pDelFrame; - pFrame->v->pDelFrame = pFrame; + if( VdbeMemDynamic(pMem) ){ + vdbeMemClearExternAndSetNull(pMem); + }else{ + pMem->flags = MEM_Null; } - if( pMem->flags & MEM_RowSet ){ - sqlite3RowSetClear(pMem->u.pRowSet); - } - MemSetTypeFlag(pMem, MEM_Null); } void sqlite3ValueSetNull(sqlite3_value *p){ sqlite3VdbeMemSetNull((Mem*)p); @@ -529,14 +668,18 @@ void sqlite3VdbeMemSetZeroBlob(Mem *pMem, int n){ if( n<0 ) n = 0; pMem->u.nZero = n; pMem->enc = SQLITE_UTF8; + pMem->z = 0; +} -#ifdef SQLITE_OMIT_INCRBLOB - sqlite3VdbeMemGrow(pMem, n, 0); - if( pMem->z ){ - pMem->n = n; - memset(pMem->z, 0, n); - } -#endif +/* +** The pMem is known to contain content that needs to be destroyed prior +** to a value change. So invoke the destructor, then set the value to +** a 64-bit integer. +*/ +static SQLITE_NOINLINE void vdbeReleaseAndSetInt64(Mem *pMem, i64 val){ + sqlite3VdbeMemSetNull(pMem); + pMem->u.i = val; + pMem->flags = MEM_Int; } /* @@ -544,9 +687,12 @@ void sqlite3VdbeMemSetZeroBlob(Mem *pMem, int n){ ** manifest type INTEGER. */ void sqlite3VdbeMemSetInt64(Mem *pMem, i64 val){ - sqlite3VdbeMemRelease(pMem); - pMem->u.i = val; - pMem->flags = MEM_Int; + if( VdbeMemDynamic(pMem) ){ + vdbeReleaseAndSetInt64(pMem, val); + }else{ + pMem->u.i = val; + pMem->flags = MEM_Int; + } } #ifndef SQLITE_OMIT_FLOATING_POINT @@ -555,11 +701,9 @@ void sqlite3VdbeMemSetInt64(Mem *pMem, i64 val){ ** manifest type REAL. */ void sqlite3VdbeMemSetDouble(Mem *pMem, double val){ - if( sqlite3IsNaN(val) ){ - sqlite3VdbeMemSetNull(pMem); - }else{ - sqlite3VdbeMemRelease(pMem); - pMem->r = val; + sqlite3VdbeMemSetNull(pMem); + if( !sqlite3IsNaN(val) ){ + pMem->u.r = val; pMem->flags = MEM_Real; } } @@ -577,10 +721,11 @@ void sqlite3VdbeMemSetRowSet(Mem *pMem){ pMem->zMalloc = sqlite3DbMallocRaw(db, 64); if( db->mallocFailed ){ pMem->flags = MEM_Null; + pMem->szMalloc = 0; }else{ assert( pMem->zMalloc ); - pMem->u.pRowSet = sqlite3RowSetInit(db, pMem->zMalloc, - sqlite3DbMallocSize(db, pMem->zMalloc)); + pMem->szMalloc = sqlite3DbMallocSize(db, pMem->zMalloc); + pMem->u.pRowSet = sqlite3RowSetInit(db, pMem->zMalloc, pMem->szMalloc); assert( pMem->u.pRowSet!=0 ); pMem->flags = MEM_RowSet; } @@ -604,7 +749,7 @@ int sqlite3VdbeMemTooBig(Mem *p){ #ifdef SQLITE_DEBUG /* -** This routine prepares a memory cell for modication by breaking +** This routine prepares a memory cell for modification by breaking ** its link to a shallow copy and by marking any current shallow ** copies of this cell as invalid. ** @@ -637,9 +782,9 @@ void sqlite3VdbeMemAboutToChange(Vdbe *pVdbe, Mem *pMem){ */ void sqlite3VdbeMemShallowCopy(Mem *pTo, const Mem *pFrom, int srcType){ assert( (pFrom->flags & MEM_RowSet)==0 ); - VdbeMemRelease(pTo); + assert( pTo->db==pFrom->db ); + if( VdbeMemDynamic(pTo) ) vdbeMemClearExternAndSetNull(pTo); memcpy(pTo, pFrom, MEMCELLSIZE); - pTo->xDel = 0; if( (pFrom->flags&MEM_Static)==0 ){ pTo->flags &= ~(MEM_Dyn|MEM_Static|MEM_Ephem); assert( srcType==MEM_Ephem || srcType==MEM_Static ); @@ -654,12 +799,11 @@ void sqlite3VdbeMemShallowCopy(Mem *pTo, const Mem *pFrom, int srcType){ int sqlite3VdbeMemCopy(Mem *pTo, const Mem *pFrom){ int rc = SQLITE_OK; + assert( pTo->db==pFrom->db ); assert( (pFrom->flags & MEM_RowSet)==0 ); - VdbeMemRelease(pTo); + if( VdbeMemDynamic(pTo) ) vdbeMemClearExternAndSetNull(pTo); memcpy(pTo, pFrom, MEMCELLSIZE); pTo->flags &= ~MEM_Dyn; - pTo->xDel = 0; - if( pTo->flags&(MEM_Str|MEM_Blob) ){ if( 0==(pFrom->flags&MEM_Static) ){ pTo->flags |= MEM_Ephem; @@ -684,8 +828,7 @@ void sqlite3VdbeMemMove(Mem *pTo, Mem *pFrom){ sqlite3VdbeMemRelease(pTo); memcpy(pTo, pFrom, sizeof(Mem)); pFrom->flags = MEM_Null; - pFrom->xDel = 0; - pFrom->zMalloc = 0; + pFrom->szMalloc = 0; } /* @@ -732,7 +875,8 @@ int sqlite3VdbeMemSetStr( if( nByte<0 ){ assert( enc!=0 ); if( enc==SQLITE_UTF8 ){ - for(nByte=0; nByte<=iLimit && z[nByte]; nByte++){} + nByte = sqlite3Strlen30(z); + if( nByte>iLimit ) nByte = iLimit+1; }else{ for(nByte=0; nByte<=iLimit && (z[nByte] | z[nByte+1]); nByte+=2){} } @@ -751,14 +895,17 @@ int sqlite3VdbeMemSetStr( if( nByte>iLimit ){ return SQLITE_TOOBIG; } - if( sqlite3VdbeMemGrow(pMem, nAlloc, 0) ){ + testcase( nAlloc==0 ); + testcase( nAlloc==31 ); + testcase( nAlloc==32 ); + if( sqlite3VdbeMemClearAndResize(pMem, MAX(nAlloc,32)) ){ return SQLITE_NOMEM; } memcpy(pMem->z, z, nAlloc); }else if( xDel==SQLITE_DYNAMIC ){ sqlite3VdbeMemRelease(pMem); pMem->zMalloc = pMem->z = (char *)z; - pMem->xDel = 0; + pMem->szMalloc = sqlite3DbMallocSize(pMem->db, pMem->zMalloc); }else{ sqlite3VdbeMemRelease(pMem); pMem->z = (char *)z; @@ -790,8 +937,11 @@ int sqlite3VdbeMemSetStr( ** key is true to get the key or false to get data. The result is written ** into the pMem element. ** -** The pMem structure is assumed to be uninitialized. Any prior content -** is overwritten without being freed. +** The pMem object must have been initialized. This routine will use +** pMem->zMalloc to hold the content from the btree, if possible. New +** pMem->zMalloc space will be allocated if necessary. The calling routine +** is responsible for making sure that the pMem object is eventually +** destroyed. ** ** If this routine fails for any reason (malloc returns NULL or unable ** to read from the disk) then the pMem is left in an inconsistent state. @@ -808,6 +958,7 @@ int sqlite3VdbeMemFromBtree( int rc = SQLITE_OK; /* Return code */ assert( sqlite3BtreeCursorIsValid(pCur) ); + assert( !VdbeMemDynamic(pMem) ); /* Note: the calls to BtreeKeyFetch() and DataFetch() below assert() ** that both the BtShared and database handle mutexes are held. */ @@ -820,29 +971,70 @@ int sqlite3VdbeMemFromBtree( assert( zData!=0 ); if( offset+amt<=available ){ - sqlite3VdbeMemRelease(pMem); pMem->z = &zData[offset]; pMem->flags = MEM_Blob|MEM_Ephem; pMem->n = (int)amt; - }else if( SQLITE_OK==(rc = sqlite3VdbeMemGrow(pMem, amt+2, 0)) ){ - if( key ){ - rc = sqlite3BtreeKey(pCur, offset, amt, pMem->z); - }else{ - rc = sqlite3BtreeData(pCur, offset, amt, pMem->z); - } - if( rc==SQLITE_OK ){ - pMem->z[amt] = 0; - pMem->z[amt+1] = 0; - pMem->flags = MEM_Blob|MEM_Term; - pMem->n = (int)amt; - }else{ - sqlite3VdbeMemRelease(pMem); + }else{ + pMem->flags = MEM_Null; + if( SQLITE_OK==(rc = sqlite3VdbeMemClearAndResize(pMem, amt+2)) ){ + if( key ){ + rc = sqlite3BtreeKey(pCur, offset, amt, pMem->z); + }else{ + rc = sqlite3BtreeData(pCur, offset, amt, pMem->z); + } + if( rc==SQLITE_OK ){ + pMem->z[amt] = 0; + pMem->z[amt+1] = 0; + pMem->flags = MEM_Blob|MEM_Term; + pMem->n = (int)amt; + }else{ + sqlite3VdbeMemRelease(pMem); + } } } return rc; } +/* +** The pVal argument is known to be a value other than NULL. +** Convert it into a string with encoding enc and return a pointer +** to a zero-terminated version of that string. +*/ +static SQLITE_NOINLINE const void *valueToText(sqlite3_value* pVal, u8 enc){ + assert( pVal!=0 ); + assert( pVal->db==0 || sqlite3_mutex_held(pVal->db->mutex) ); + assert( (enc&3)==(enc&~SQLITE_UTF16_ALIGNED) ); + assert( (pVal->flags & MEM_RowSet)==0 ); + assert( (pVal->flags & (MEM_Null))==0 ); + if( pVal->flags & (MEM_Blob|MEM_Str) ){ + pVal->flags |= MEM_Str; + if( pVal->flags & MEM_Zero ){ + sqlite3VdbeMemExpandBlob(pVal); + } + if( pVal->enc != (enc & ~SQLITE_UTF16_ALIGNED) ){ + sqlite3VdbeChangeEncoding(pVal, enc & ~SQLITE_UTF16_ALIGNED); + } + if( (enc & SQLITE_UTF16_ALIGNED)!=0 && 1==(1&SQLITE_PTR_TO_INT(pVal->z)) ){ + assert( (pVal->flags & (MEM_Ephem|MEM_Static))!=0 ); + if( sqlite3VdbeMemMakeWriteable(pVal)!=SQLITE_OK ){ + return 0; + } + } + sqlite3VdbeMemNulTerminate(pVal); /* IMP: R-31275-44060 */ + }else{ + sqlite3VdbeMemStringify(pVal, enc, 0); + assert( 0==(1&SQLITE_PTR_TO_INT(pVal->z)) ); + } + assert(pVal->enc==(enc & ~SQLITE_UTF16_ALIGNED) || pVal->db==0 + || pVal->db->mallocFailed ); + if( pVal->enc==(enc & ~SQLITE_UTF16_ALIGNED) ){ + return pVal->z; + }else{ + return 0; + } +} + /* This function is only available internally, it is not part of the ** external API. It works in a similar way to sqlite3_value_text(), ** except the data returned is in the encoding specified by the second @@ -855,38 +1047,16 @@ int sqlite3VdbeMemFromBtree( */ const void *sqlite3ValueText(sqlite3_value* pVal, u8 enc){ if( !pVal ) return 0; - assert( pVal->db==0 || sqlite3_mutex_held(pVal->db->mutex) ); assert( (enc&3)==(enc&~SQLITE_UTF16_ALIGNED) ); assert( (pVal->flags & MEM_RowSet)==0 ); - + if( (pVal->flags&(MEM_Str|MEM_Term))==(MEM_Str|MEM_Term) && pVal->enc==enc ){ + return pVal->z; + } if( pVal->flags&MEM_Null ){ return 0; } - assert( (MEM_Blob>>3) == MEM_Str ); - pVal->flags |= (pVal->flags & MEM_Blob)>>3; - ExpandBlob(pVal); - if( pVal->flags&MEM_Str ){ - sqlite3VdbeChangeEncoding(pVal, enc & ~SQLITE_UTF16_ALIGNED); - if( (enc & SQLITE_UTF16_ALIGNED)!=0 && 1==(1&SQLITE_PTR_TO_INT(pVal->z)) ){ - assert( (pVal->flags & (MEM_Ephem|MEM_Static))!=0 ); - if( sqlite3VdbeMemMakeWriteable(pVal)!=SQLITE_OK ){ - return 0; - } - } - sqlite3VdbeMemNulTerminate(pVal); /* IMP: R-31275-44060 */ - }else{ - assert( (pVal->flags&MEM_Blob)==0 ); - sqlite3VdbeMemStringify(pVal, enc); - assert( 0==(1&SQLITE_PTR_TO_INT(pVal->z)) ); - } - assert(pVal->enc==(enc & ~SQLITE_UTF16_ALIGNED) || pVal->db==0 - || pVal->db->mallocFailed ); - if( pVal->enc==(enc & ~SQLITE_UTF16_ALIGNED) ){ - return pVal->z; - }else{ - return 0; - } + return valueToText(pVal, enc); } /* @@ -993,9 +1163,20 @@ static int valueFromExpr( *ppVal = 0; return SQLITE_OK; } - op = pExpr->op; + while( (op = pExpr->op)==TK_UPLUS ) pExpr = pExpr->pLeft; if( NEVER(op==TK_REGISTER) ) op = pExpr->op2; + if( op==TK_CAST ){ + u8 aff = sqlite3AffinityType(pExpr->u.zToken,0); + rc = valueFromExpr(db, pExpr->pLeft, enc, aff, ppVal, pCtx); + testcase( rc!=SQLITE_OK ); + if( *ppVal ){ + sqlite3VdbeMemCast(*ppVal, aff, SQLITE_UTF8); + sqlite3ValueApplyAffinity(*ppVal, affinity, SQLITE_UTF8); + } + return rc; + } + /* Handle negative integers in a single step. This is needed in the ** case when the value is -9223372036854775808. */ @@ -1032,14 +1213,14 @@ static int valueFromExpr( && pVal!=0 ){ sqlite3VdbeMemNumerify(pVal); - if( pVal->u.i==SMALLEST_INT64 ){ - pVal->flags &= ~MEM_Int; - pVal->flags |= MEM_Real; - pVal->r = (double)SMALLEST_INT64; + if( pVal->flags & MEM_Real ){ + pVal->u.r = -pVal->u.r; + }else if( pVal->u.i==SMALLEST_INT64 ){ + pVal->u.r = -(double)SMALLEST_INT64; + MemSetTypeFlag(pVal, MEM_Real); }else{ pVal->u.i = -pVal->u.i; } - pVal->r = -pVal->r; sqlite3ValueApplyAffinity(pVal, affinity, enc); } }else if( op==TK_NULL ){ @@ -1130,7 +1311,7 @@ static void recordFunc( sqlite3_result_error_nomem(context); }else{ aRet[0] = nSerial+1; - sqlite3PutVarint(&aRet[1], iSerial); + putVarint32(&aRet[1], iSerial); sqlite3VdbeSerialPut(&aRet[1+nSerial], argv[0], iSerial); sqlite3_result_blob(context, aRet, nRet, SQLITE_TRANSIENT); sqlite3DbFree(db, aRet); @@ -1152,6 +1333,68 @@ void sqlite3AnalyzeFunctions(void){ } } +/* +** Attempt to extract a value from pExpr and use it to construct *ppVal. +** +** If pAlloc is not NULL, then an UnpackedRecord object is created for +** pAlloc if one does not exist and the new value is added to the +** UnpackedRecord object. +** +** A value is extracted in the following cases: +** +** * (pExpr==0). In this case the value is assumed to be an SQL NULL, +** +** * The expression is a bound variable, and this is a reprepare, or +** +** * The expression is a literal value. +** +** On success, *ppVal is made to point to the extracted value. The caller +** is responsible for ensuring that the value is eventually freed. +*/ +static int stat4ValueFromExpr( + Parse *pParse, /* Parse context */ + Expr *pExpr, /* The expression to extract a value from */ + u8 affinity, /* Affinity to use */ + struct ValueNewStat4Ctx *pAlloc,/* How to allocate space. Or NULL */ + sqlite3_value **ppVal /* OUT: New value object (or NULL) */ +){ + int rc = SQLITE_OK; + sqlite3_value *pVal = 0; + sqlite3 *db = pParse->db; + + /* Skip over any TK_COLLATE nodes */ + pExpr = sqlite3ExprSkipCollate(pExpr); + + if( !pExpr ){ + pVal = valueNew(db, pAlloc); + if( pVal ){ + sqlite3VdbeMemSetNull((Mem*)pVal); + } + }else if( pExpr->op==TK_VARIABLE + || NEVER(pExpr->op==TK_REGISTER && pExpr->op2==TK_VARIABLE) + ){ + Vdbe *v; + int iBindVar = pExpr->iColumn; + sqlite3VdbeSetVarmask(pParse->pVdbe, iBindVar); + if( (v = pParse->pReprepare)!=0 ){ + pVal = valueNew(db, pAlloc); + if( pVal ){ + rc = sqlite3VdbeMemCopy((Mem*)pVal, &v->aVar[iBindVar-1]); + if( rc==SQLITE_OK ){ + sqlite3ValueApplyAffinity(pVal, affinity, ENC(db)); + } + pVal->db = pParse->db; + } + } + }else{ + rc = valueFromExpr(db, pExpr, ENC(db), affinity, &pVal, pAlloc); + } + + assert( pVal==0 || pVal->db==db ); + *ppVal = pVal; + return rc; +} + /* ** This function is used to allocate and populate UnpackedRecord ** structures intended to be compared against sample index keys stored @@ -1191,50 +1434,88 @@ int sqlite3Stat4ProbeSetValue( int iVal, /* Array element to populate */ int *pbOk /* OUT: True if value was extracted */ ){ - int rc = SQLITE_OK; + int rc; sqlite3_value *pVal = 0; - sqlite3 *db = pParse->db; - - struct ValueNewStat4Ctx alloc; + alloc.pParse = pParse; alloc.pIdx = pIdx; alloc.ppRec = ppRec; alloc.iVal = iVal; - /* Skip over any TK_COLLATE nodes */ - pExpr = sqlite3ExprSkipCollate(pExpr); - - if( !pExpr ){ - pVal = valueNew(db, &alloc); - if( pVal ){ - sqlite3VdbeMemSetNull((Mem*)pVal); - } - }else if( pExpr->op==TK_VARIABLE - || NEVER(pExpr->op==TK_REGISTER && pExpr->op2==TK_VARIABLE) - ){ - Vdbe *v; - int iBindVar = pExpr->iColumn; - sqlite3VdbeSetVarmask(pParse->pVdbe, iBindVar); - if( (v = pParse->pReprepare)!=0 ){ - pVal = valueNew(db, &alloc); - if( pVal ){ - rc = sqlite3VdbeMemCopy((Mem*)pVal, &v->aVar[iBindVar-1]); - if( rc==SQLITE_OK ){ - sqlite3ValueApplyAffinity(pVal, affinity, ENC(db)); - } - pVal->db = pParse->db; - } - } - }else{ - rc = valueFromExpr(db, pExpr, ENC(db), affinity, &pVal, &alloc); - } + rc = stat4ValueFromExpr(pParse, pExpr, affinity, &alloc, &pVal); + assert( pVal==0 || pVal->db==pParse->db ); *pbOk = (pVal!=0); - - assert( pVal==0 || pVal->db==db ); return rc; } +/* +** Attempt to extract a value from expression pExpr using the methods +** as described for sqlite3Stat4ProbeSetValue() above. +** +** If successful, set *ppVal to point to a new value object and return +** SQLITE_OK. If no value can be extracted, but no other error occurs +** (e.g. OOM), return SQLITE_OK and set *ppVal to NULL. Or, if an error +** does occur, return an SQLite error code. The final value of *ppVal +** is undefined in this case. +*/ +int sqlite3Stat4ValueFromExpr( + Parse *pParse, /* Parse context */ + Expr *pExpr, /* The expression to extract a value from */ + u8 affinity, /* Affinity to use */ + sqlite3_value **ppVal /* OUT: New value object (or NULL) */ +){ + return stat4ValueFromExpr(pParse, pExpr, affinity, 0, ppVal); +} + +/* +** Extract the iCol-th column from the nRec-byte record in pRec. Write +** the column value into *ppVal. If *ppVal is initially NULL then a new +** sqlite3_value object is allocated. +** +** If *ppVal is initially NULL then the caller is responsible for +** ensuring that the value written into *ppVal is eventually freed. +*/ +int sqlite3Stat4Column( + sqlite3 *db, /* Database handle */ + const void *pRec, /* Pointer to buffer containing record */ + int nRec, /* Size of buffer pRec in bytes */ + int iCol, /* Column to extract */ + sqlite3_value **ppVal /* OUT: Extracted value */ +){ + u32 t; /* a column type code */ + int nHdr; /* Size of the header in the record */ + int iHdr; /* Next unread header byte */ + int iField; /* Next unread data byte */ + int szField; /* Size of the current data field */ + int i; /* Column index */ + u8 *a = (u8*)pRec; /* Typecast byte array */ + Mem *pMem = *ppVal; /* Write result into this Mem object */ + + assert( iCol>0 ); + iHdr = getVarint32(a, nHdr); + if( nHdr>nRec || iHdr>=nHdr ) return SQLITE_CORRUPT_BKPT; + iField = nHdr; + for(i=0; i<=iCol; i++){ + iHdr += getVarint32(&a[iHdr], t); + testcase( iHdr==nHdr ); + testcase( iHdr==nHdr+1 ); + if( iHdr>nHdr ) return SQLITE_CORRUPT_BKPT; + szField = sqlite3VdbeSerialTypeLen(t); + iField += szField; + } + testcase( iField==nRec ); + testcase( iField==nRec+1 ); + if( iField>nRec ) return SQLITE_CORRUPT_BKPT; + if( pMem==0 ){ + pMem = *ppVal = sqlite3ValueNew(db); + if( pMem==0 ) return SQLITE_NOMEM; + } + sqlite3VdbeSerialGet(&a[iField-szField], t, pMem); + pMem->enc = ENC(db); + return SQLITE_OK; +} + /* ** Unless it is NULL, the argument must be an UnpackedRecord object returned ** by an earlier call to sqlite3Stat4ProbeSetValue(). This call deletes @@ -1247,7 +1528,7 @@ void sqlite3Stat4ProbeFree(UnpackedRecord *pRec){ Mem *aMem = pRec->aMem; sqlite3 *db = aMem[0].db; for(i=0; ipKeyInfo); sqlite3DbFree(db, pRec); diff --git a/src/vdbesort.c b/src/vdbesort.c index d3690244c4..d022af9af6 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1,5 +1,5 @@ /* -** 2011 July 9 +** 2011-07-09 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -10,44 +10,205 @@ ** ************************************************************************* ** This file contains code for the VdbeSorter object, used in concert with -** a VdbeCursor to sort large numbers of keys (as may be required, for -** example, by CREATE INDEX statements on tables too large to fit in main -** memory). +** a VdbeCursor to sort large numbers of keys for CREATE INDEX statements +** or by SELECT statements with ORDER BY clauses that cannot be satisfied +** using indexes and without LIMIT clauses. +** +** The VdbeSorter object implements a multi-threaded external merge sort +** algorithm that is efficient even if the number of elements being sorted +** exceeds the available memory. +** +** Here is the (internal, non-API) interface between this module and the +** rest of the SQLite system: +** +** sqlite3VdbeSorterInit() Create a new VdbeSorter object. +** +** sqlite3VdbeSorterWrite() Add a single new row to the VdbeSorter +** object. The row is a binary blob in the +** OP_MakeRecord format that contains both +** the ORDER BY key columns and result columns +** in the case of a SELECT w/ ORDER BY, or +** the complete record for an index entry +** in the case of a CREATE INDEX. +** +** sqlite3VdbeSorterRewind() Sort all content previously added. +** Position the read cursor on the +** first sorted element. +** +** sqlite3VdbeSorterNext() Advance the read cursor to the next sorted +** element. +** +** sqlite3VdbeSorterRowkey() Return the complete binary blob for the +** row currently under the read cursor. +** +** sqlite3VdbeSorterCompare() Compare the binary blob for the row +** currently under the read cursor against +** another binary blob X and report if +** X is strictly less than the read cursor. +** Used to enforce uniqueness in a +** CREATE UNIQUE INDEX statement. +** +** sqlite3VdbeSorterClose() Close the VdbeSorter object and reclaim +** all resources. +** +** sqlite3VdbeSorterReset() Refurbish the VdbeSorter for reuse. This +** is like Close() followed by Init() only +** much faster. +** +** The interfaces above must be called in a particular order. Write() can +** only occur in between Init()/Reset() and Rewind(). Next(), Rowkey(), and +** Compare() can only occur in between Rewind() and Close()/Reset(). i.e. +** +** Init() +** for each record: Write() +** Rewind() +** Rowkey()/Compare() +** Next() +** Close() +** +** Algorithm: +** +** Records passed to the sorter via calls to Write() are initially held +** unsorted in main memory. Assuming the amount of memory used never exceeds +** a threshold, when Rewind() is called the set of records is sorted using +** an in-memory merge sort. In this case, no temporary files are required +** and subsequent calls to Rowkey(), Next() and Compare() read records +** directly from main memory. +** +** If the amount of space used to store records in main memory exceeds the +** threshold, then the set of records currently in memory are sorted and +** written to a temporary file in "Packed Memory Array" (PMA) format. +** A PMA created at this point is known as a "level-0 PMA". Higher levels +** of PMAs may be created by merging existing PMAs together - for example +** merging two or more level-0 PMAs together creates a level-1 PMA. +** +** The threshold for the amount of main memory to use before flushing +** records to a PMA is roughly the same as the limit configured for the +** page-cache of the main database. Specifically, the threshold is set to +** the value returned by "PRAGMA main.page_size" multipled by +** that returned by "PRAGMA main.cache_size", in bytes. +** +** If the sorter is running in single-threaded mode, then all PMAs generated +** are appended to a single temporary file. Or, if the sorter is running in +** multi-threaded mode then up to (N+1) temporary files may be opened, where +** N is the configured number of worker threads. In this case, instead of +** sorting the records and writing the PMA to a temporary file itself, the +** calling thread usually launches a worker thread to do so. Except, if +** there are already N worker threads running, the main thread does the work +** itself. +** +** The sorter is running in multi-threaded mode if (a) the library was built +** with pre-processor symbol SQLITE_MAX_WORKER_THREADS set to a value greater +** than zero, and (b) worker threads have been enabled at runtime by calling +** "PRAGMA threads=N" with some value of N greater than 0. +** +** When Rewind() is called, any data remaining in memory is flushed to a +** final PMA. So at this point the data is stored in some number of sorted +** PMAs within temporary files on disk. +** +** If there are fewer than SORTER_MAX_MERGE_COUNT PMAs in total and the +** sorter is running in single-threaded mode, then these PMAs are merged +** incrementally as keys are retreived from the sorter by the VDBE. The +** MergeEngine object, described in further detail below, performs this +** merge. +** +** Or, if running in multi-threaded mode, then a background thread is +** launched to merge the existing PMAs. Once the background thread has +** merged T bytes of data into a single sorted PMA, the main thread +** begins reading keys from that PMA while the background thread proceeds +** with merging the next T bytes of data. And so on. +** +** Parameter T is set to half the value of the memory threshold used +** by Write() above to determine when to create a new PMA. +** +** If there are more than SORTER_MAX_MERGE_COUNT PMAs in total when +** Rewind() is called, then a hierarchy of incremental-merges is used. +** First, T bytes of data from the first SORTER_MAX_MERGE_COUNT PMAs on +** disk are merged together. Then T bytes of data from the second set, and +** so on, such that no operation ever merges more than SORTER_MAX_MERGE_COUNT +** PMAs at a time. This done is to improve locality. +** +** If running in multi-threaded mode and there are more than +** SORTER_MAX_MERGE_COUNT PMAs on disk when Rewind() is called, then more +** than one background thread may be created. Specifically, there may be +** one background thread for each temporary file on disk, and one background +** thread to merge the output of each of the others to a single PMA for +** the main thread to read from. */ - #include "sqliteInt.h" #include "vdbeInt.h" - -typedef struct VdbeSorterIter VdbeSorterIter; -typedef struct SorterRecord SorterRecord; -typedef struct FileWriter FileWriter; +/* +** If SQLITE_DEBUG_SORTER_THREADS is defined, this module outputs various +** messages to stderr that may be helpful in understanding the performance +** characteristics of the sorter in multi-threaded mode. +*/ +#if 0 +# define SQLITE_DEBUG_SORTER_THREADS 1 +#endif /* -** NOTES ON DATA STRUCTURE USED FOR N-WAY MERGES: +** Hard-coded maximum amount of data to accumulate in memory before flushing +** to a level 0 PMA. The purpose of this limit is to prevent various integer +** overflows. 512MiB. +*/ +#define SQLITE_MAX_MXPMASIZE (1<<29) + +/* +** Private objects used by the sorter +*/ +typedef struct MergeEngine MergeEngine; /* Merge PMAs together */ +typedef struct PmaReader PmaReader; /* Incrementally read one PMA */ +typedef struct PmaWriter PmaWriter; /* Incrementally write one PMA */ +typedef struct SorterRecord SorterRecord; /* A record being sorted */ +typedef struct SortSubtask SortSubtask; /* A sub-task in the sort process */ +typedef struct SorterFile SorterFile; /* Temporary file object wrapper */ +typedef struct SorterList SorterList; /* In-memory list of records */ +typedef struct IncrMerger IncrMerger; /* Read & merge multiple PMAs */ + +/* +** A container for a temp file handle and the current amount of data +** stored in the file. +*/ +struct SorterFile { + sqlite3_file *pFd; /* File handle */ + i64 iEof; /* Bytes of data stored in pFd */ +}; + +/* +** An in-memory list of objects to be sorted. ** -** As keys are added to the sorter, they are written to disk in a series -** of sorted packed-memory-arrays (PMAs). The size of each PMA is roughly -** the same as the cache-size allowed for temporary databases. In order -** to allow the caller to extract keys from the sorter in sorted order, -** all PMAs currently stored on disk must be merged together. This comment -** describes the data structure used to do so. The structure supports -** merging any number of arrays in a single pass with no redundant comparison -** operations. +** If aMemory==0 then each object is allocated separately and the objects +** are connected using SorterRecord.u.pNext. If aMemory!=0 then all objects +** are stored in the aMemory[] bulk memory, one right after the other, and +** are connected using SorterRecord.u.iNext. +*/ +struct SorterList { + SorterRecord *pList; /* Linked list of records */ + u8 *aMemory; /* If non-NULL, bulk memory to hold pList */ + int szPMA; /* Size of pList as PMA in bytes */ +}; + +/* +** The MergeEngine object is used to combine two or more smaller PMAs into +** one big PMA using a merge operation. Separate PMAs all need to be +** combined into one big PMA in order to be able to step through the sorted +** records in order. ** -** The aIter[] array contains an iterator for each of the PMAs being merged. -** An aIter[] iterator either points to a valid key or else is at EOF. For -** the purposes of the paragraphs below, we assume that the array is actually -** N elements in size, where N is the smallest power of 2 greater to or equal -** to the number of iterators being merged. The extra aIter[] elements are -** treated as if they are empty (always at EOF). +** The aReadr[] array contains a PmaReader object for each of the PMAs being +** merged. An aReadr[] object either points to a valid key or else is at EOF. +** ("EOF" means "End Of File". When aReadr[] is at EOF there is no more data.) +** For the purposes of the paragraphs below, we assume that the array is +** actually N elements in size, where N is the smallest power of 2 greater +** to or equal to the number of PMAs being merged. The extra aReadr[] elements +** are treated as if they are empty (always at EOF). ** ** The aTree[] array is also N elements in size. The value of N is stored in -** the VdbeSorter.nTree variable. +** the MergeEngine.nTree variable. ** ** The final (N/2) elements of aTree[] contain the results of comparing -** pairs of iterator keys together. Element i contains the result of -** comparing aIter[2*i-N] and aIter[2*i-N+1]. Whichever key is smaller, the +** pairs of PMA keys together. Element i contains the result of +** comparing aReadr[2*i-N] and aReadr[2*i-N+1]. Whichever key is smaller, the ** aTree element is set to the index of it. ** ** For the purposes of this comparison, EOF is considered greater than any @@ -55,34 +216,34 @@ typedef struct FileWriter FileWriter; ** values), it doesn't matter which index is stored. ** ** The (N/4) elements of aTree[] that precede the final (N/2) described -** above contains the index of the smallest of each block of 4 iterators. -** And so on. So that aTree[1] contains the index of the iterator that +** above contains the index of the smallest of each block of 4 PmaReaders +** And so on. So that aTree[1] contains the index of the PmaReader that ** currently points to the smallest key value. aTree[0] is unused. ** ** Example: ** -** aIter[0] -> Banana -** aIter[1] -> Feijoa -** aIter[2] -> Elderberry -** aIter[3] -> Currant -** aIter[4] -> Grapefruit -** aIter[5] -> Apple -** aIter[6] -> Durian -** aIter[7] -> EOF +** aReadr[0] -> Banana +** aReadr[1] -> Feijoa +** aReadr[2] -> Elderberry +** aReadr[3] -> Currant +** aReadr[4] -> Grapefruit +** aReadr[5] -> Apple +** aReadr[6] -> Durian +** aReadr[7] -> EOF ** ** aTree[] = { X, 5 0, 5 0, 3, 5, 6 } ** ** The current element is "Apple" (the value of the key indicated by -** iterator 5). When the Next() operation is invoked, iterator 5 will +** PmaReader 5). When the Next() operation is invoked, PmaReader 5 will ** be advanced to the next key in its segment. Say the next key is ** "Eggplant": ** -** aIter[5] -> Eggplant +** aReadr[5] -> Eggplant ** -** The contents of aTree[] are updated first by comparing the new iterator -** 5 key to the current key of iterator 4 (still "Grapefruit"). The iterator +** The contents of aTree[] are updated first by comparing the new PmaReader +** 5 key to the current key of PmaReader 4 (still "Grapefruit"). The PmaReader ** 5 value is still smaller, so aTree[6] is set to 5. And so on up the tree. -** The value of iterator 6 - "Durian" - is now smaller than that of iterator +** The value of PmaReader 6 - "Durian" - is now smaller than that of PmaReader ** 5, so aTree[3] is set to 6. Key 0 is smaller than key 6 (Bananafile2. And instead of using a +** background thread to prepare data for the PmaReader, with a single +** threaded IncrMerger the allocate part of pTask->file2 is "refilled" with +** keys from pMerger by the calling thread whenever the PmaReader runs out +** of data. */ -struct FileWriter { +struct IncrMerger { + SortSubtask *pTask; /* Task that owns this merger */ + MergeEngine *pMerger; /* Merge engine thread reads data from */ + i64 iStartOff; /* Offset to start writing file at */ + int mxSz; /* Maximum bytes of data to store */ + int bEof; /* Set to true when merge is finished */ + int bUseThread; /* True to use a bg thread for this object */ + SorterFile aFile[2]; /* aFile[0] for reading, [1] for writing */ +}; + +/* +** An instance of this object is used for writing a PMA. +** +** The PMA is written one record at a time. Each record is of an arbitrary +** size. But I/O is more efficient if it occurs in page-sized blocks where +** each block is aligned on a page boundary. This object caches writes to +** the PMA so that aligned, page-size blocks are written. +*/ +struct PmaWriter { int eFWErr; /* Non-zero if in an error state */ u8 *aBuffer; /* Pointer to write buffer */ int nBuffer; /* Size of write buffer in bytes */ int iBufStart; /* First byte of buffer to write */ int iBufEnd; /* Last byte of buffer to write */ i64 iWriteOff; /* Offset of start of buffer in file */ - sqlite3_file *pFile; /* File to write to */ + sqlite3_file *pFd; /* File handle to write to */ }; /* -** A structure to store a single record. All in-memory records are connected -** together into a linked list headed at VdbeSorter.pRecord using the -** SorterRecord.pNext pointer. +** This object is the header on a single record while that record is being +** held in memory and prior to being written out as part of a PMA. +** +** How the linked list is connected depends on how memory is being managed +** by this module. If using a separate allocation for each in-memory record +** (VdbeSorter.list.aMemory==0), then the list is always connected using the +** SorterRecord.u.pNext pointers. +** +** Or, if using the single large allocation method (VdbeSorter.list.aMemory!=0), +** then while records are being accumulated the list is linked using the +** SorterRecord.u.iNext offset. This is because the aMemory[] array may +** be sqlite3Realloc()ed while records are being accumulated. Once the VM +** has finished passing records to the sorter, or when the in-memory buffer +** is full, the list is sorted. As part of the sorting process, it is +** converted to use the SorterRecord.u.pNext pointers. See function +** vdbeSorterSort() for details. */ struct SorterRecord { - void *pVal; - int nVal; - SorterRecord *pNext; + int nVal; /* Size of the record in bytes */ + union { + SorterRecord *pNext; /* Pointer to next record in list */ + int iNext; /* Offset within aMemory of next record */ + } u; + /* The data for the record immediately follows this header */ }; -/* Minimum allowable value for the VdbeSorter.nWorking variable */ -#define SORTER_MIN_WORKING 10 +/* Return a pointer to the buffer containing the record data for SorterRecord +** object p. Should be used as if: +** +** void *SRVAL(SorterRecord *p) { return (void*)&p[1]; } +*/ +#define SRVAL(p) ((void*)((SorterRecord*)(p) + 1)) -/* Maximum number of segments to merge in a single pass. */ +/* The minimum PMA size is set to this value multiplied by the database +** page size in bytes. */ +#ifndef SQLITE_SORTER_PMASZ +# define SQLITE_SORTER_PMASZ 10 +#endif + +/* Maximum number of PMAs that a single MergeEngine can merge */ #define SORTER_MAX_MERGE_COUNT 16 +static int vdbeIncrSwap(IncrMerger*); +static void vdbeIncrFree(IncrMerger *); + /* -** Free all memory belonging to the VdbeSorterIter object passed as the second +** Free all memory belonging to the PmaReader object passed as the ** argument. All structure fields are set to zero before returning. */ -static void vdbeSorterIterZero(sqlite3 *db, VdbeSorterIter *pIter){ - sqlite3DbFree(db, pIter->aAlloc); - sqlite3DbFree(db, pIter->aBuffer); - memset(pIter, 0, sizeof(VdbeSorterIter)); +static void vdbePmaReaderClear(PmaReader *pReadr){ + sqlite3_free(pReadr->aAlloc); + sqlite3_free(pReadr->aBuffer); + if( pReadr->aMap ) sqlite3OsUnfetch(pReadr->pFd, 0, pReadr->aMap); + vdbeIncrFree(pReadr->pIncr); + memset(pReadr, 0, sizeof(PmaReader)); } /* -** Read nByte bytes of data from the stream of data iterated by object p. +** Read the next nByte bytes of data from the PMA p. ** If successful, set *ppOut to point to a buffer containing the data ** and return SQLITE_OK. Otherwise, if an error occurs, return an SQLite ** error code. ** -** The buffer indicated by *ppOut may only be considered valid until the +** The buffer returned in *ppOut is only valid until the ** next call to this function. */ -static int vdbeSorterIterRead( - sqlite3 *db, /* Database handle (for malloc) */ - VdbeSorterIter *p, /* Iterator */ +static int vdbePmaReadBlob( + PmaReader *p, /* PmaReader from which to take the blob */ int nByte, /* Bytes of data to read */ u8 **ppOut /* OUT: Pointer to buffer containing data */ ){ int iBuf; /* Offset within buffer to read from */ int nAvail; /* Bytes of data available in buffer */ + + if( p->aMap ){ + *ppOut = &p->aMap[p->iReadOff]; + p->iReadOff += nByte; + return SQLITE_OK; + } + assert( p->aBuffer ); /* If there is no more data to be read from the buffer, read the next @@ -201,8 +513,8 @@ static int vdbeSorterIterRead( } assert( nRead>0 ); - /* Read data from the file. Return early if an error occurs. */ - rc = sqlite3OsRead(p->pFile, p->aBuffer, nRead, p->iReadOff); + /* Readr data from the file. Return early if an error occurs. */ + rc = sqlite3OsRead(p->pFd, p->aBuffer, nRead, p->iReadOff); assert( rc!=SQLITE_IOERR_SHORT_READ ); if( rc!=SQLITE_OK ) return rc; } @@ -222,11 +534,13 @@ static int vdbeSorterIterRead( /* Extend the p->aAlloc[] allocation if required. */ if( p->nAllocnAlloc*2; + u8 *aNew; + int nNew = MAX(128, p->nAlloc*2); while( nByte>nNew ) nNew = nNew*2; - p->aAlloc = sqlite3DbReallocOrFree(db, p->aAlloc, nNew); - if( !p->aAlloc ) return SQLITE_NOMEM; + aNew = sqlite3Realloc(p->aAlloc, nNew); + if( !aNew ) return SQLITE_NOMEM; p->nAlloc = nNew; + p->aAlloc = aNew; } /* Copy as much data as is available in the buffer into the start of @@ -238,13 +552,13 @@ static int vdbeSorterIterRead( /* The following loop copies up to p->nBuffer bytes per iteration into ** the p->aAlloc[] buffer. */ while( nRem>0 ){ - int rc; /* vdbeSorterIterRead() return code */ + int rc; /* vdbePmaReadBlob() return code */ int nCopy; /* Number of bytes to copy */ u8 *aNext; /* Pointer to buffer to copy data from */ nCopy = nRem; if( nRem>p->nBuffer ) nCopy = p->nBuffer; - rc = vdbeSorterIterRead(db, p, nCopy, &aNext); + rc = vdbePmaReadBlob(p, nCopy, &aNext); if( rc!=SQLITE_OK ) return rc; assert( aNext!=p->aAlloc ); memcpy(&p->aAlloc[nByte - nRem], aNext, nCopy); @@ -261,108 +575,174 @@ static int vdbeSorterIterRead( ** Read a varint from the stream of data accessed by p. Set *pnOut to ** the value read. */ -static int vdbeSorterIterVarint(sqlite3 *db, VdbeSorterIter *p, u64 *pnOut){ +static int vdbePmaReadVarint(PmaReader *p, u64 *pnOut){ int iBuf; - iBuf = p->iReadOff % p->nBuffer; - if( iBuf && (p->nBuffer-iBuf)>=9 ){ - p->iReadOff += sqlite3GetVarint(&p->aBuffer[iBuf], pnOut); + if( p->aMap ){ + p->iReadOff += sqlite3GetVarint(&p->aMap[p->iReadOff], pnOut); }else{ - u8 aVarint[16], *a; - int i = 0, rc; - do{ - rc = vdbeSorterIterRead(db, p, 1, &a); - if( rc ) return rc; - aVarint[(i++)&0xf] = a[0]; - }while( (a[0]&0x80)!=0 ); - sqlite3GetVarint(aVarint, pnOut); + iBuf = p->iReadOff % p->nBuffer; + if( iBuf && (p->nBuffer-iBuf)>=9 ){ + p->iReadOff += sqlite3GetVarint(&p->aBuffer[iBuf], pnOut); + }else{ + u8 aVarint[16], *a; + int i = 0, rc; + do{ + rc = vdbePmaReadBlob(p, 1, &a); + if( rc ) return rc; + aVarint[(i++)&0xf] = a[0]; + }while( (a[0]&0x80)!=0 ); + sqlite3GetVarint(aVarint, pnOut); + } } return SQLITE_OK; } +/* +** Attempt to memory map file pFile. If successful, set *pp to point to the +** new mapping and return SQLITE_OK. If the mapping is not attempted +** (because the file is too large or the VFS layer is configured not to use +** mmap), return SQLITE_OK and set *pp to NULL. +** +** Or, if an error occurs, return an SQLite error code. The final value of +** *pp is undefined in this case. +*/ +static int vdbeSorterMapFile(SortSubtask *pTask, SorterFile *pFile, u8 **pp){ + int rc = SQLITE_OK; + if( pFile->iEof<=(i64)(pTask->pSorter->db->nMaxSorterMmap) ){ + sqlite3_file *pFd = pFile->pFd; + if( pFd->pMethods->iVersion>=3 ){ + rc = sqlite3OsFetch(pFd, 0, (int)pFile->iEof, (void**)pp); + testcase( rc!=SQLITE_OK ); + } + } + return rc; +} /* -** Advance iterator pIter to the next key in its PMA. Return SQLITE_OK if -** no error occurs, or an SQLite error code if one does. +** Attach PmaReader pReadr to file pFile (if it is not already attached to +** that file) and seek it to offset iOff within the file. Return SQLITE_OK +** if successful, or an SQLite error code if an error occurs. */ -static int vdbeSorterIterNext( - sqlite3 *db, /* Database handle (for sqlite3DbMalloc() ) */ - VdbeSorterIter *pIter /* Iterator to advance */ +static int vdbePmaReaderSeek( + SortSubtask *pTask, /* Task context */ + PmaReader *pReadr, /* Reader whose cursor is to be moved */ + SorterFile *pFile, /* Sorter file to read from */ + i64 iOff /* Offset in pFile */ ){ - int rc; /* Return Code */ - u64 nRec = 0; /* Size of record in bytes */ + int rc = SQLITE_OK; - if( pIter->iReadOff>=pIter->iEof ){ - /* This is an EOF condition */ - vdbeSorterIterZero(db, pIter); - return SQLITE_OK; + assert( pReadr->pIncr==0 || pReadr->pIncr->bEof==0 ); + + if( sqlite3FaultSim(201) ) return SQLITE_IOERR_READ; + if( pReadr->aMap ){ + sqlite3OsUnfetch(pReadr->pFd, 0, pReadr->aMap); + pReadr->aMap = 0; } + pReadr->iReadOff = iOff; + pReadr->iEof = pFile->iEof; + pReadr->pFd = pFile->pFd; - rc = vdbeSorterIterVarint(db, pIter, &nRec); - if( rc==SQLITE_OK ){ - pIter->nKey = (int)nRec; - rc = vdbeSorterIterRead(db, pIter, (int)nRec, &pIter->aKey); + rc = vdbeSorterMapFile(pTask, pFile, &pReadr->aMap); + if( rc==SQLITE_OK && pReadr->aMap==0 ){ + int pgsz = pTask->pSorter->pgsz; + int iBuf = pReadr->iReadOff % pgsz; + if( pReadr->aBuffer==0 ){ + pReadr->aBuffer = (u8*)sqlite3Malloc(pgsz); + if( pReadr->aBuffer==0 ) rc = SQLITE_NOMEM; + pReadr->nBuffer = pgsz; + } + if( rc==SQLITE_OK && iBuf ){ + int nRead = pgsz - iBuf; + if( (pReadr->iReadOff + nRead) > pReadr->iEof ){ + nRead = (int)(pReadr->iEof - pReadr->iReadOff); + } + rc = sqlite3OsRead( + pReadr->pFd, &pReadr->aBuffer[iBuf], nRead, pReadr->iReadOff + ); + testcase( rc!=SQLITE_OK ); + } } return rc; } /* -** Initialize iterator pIter to scan through the PMA stored in file pFile -** starting at offset iStart and ending at offset iEof-1. This function -** leaves the iterator pointing to the first key in the PMA (or EOF if the -** PMA is empty). +** Advance PmaReader pReadr to the next key in its PMA. Return SQLITE_OK if +** no error occurs, or an SQLite error code if one does. */ -static int vdbeSorterIterInit( - sqlite3 *db, /* Database handle */ - const VdbeSorter *pSorter, /* Sorter object */ - i64 iStart, /* Start offset in pFile */ - VdbeSorterIter *pIter, /* Iterator to populate */ - i64 *pnByte /* IN/OUT: Increment this value by PMA size */ -){ - int rc = SQLITE_OK; - int nBuf; +static int vdbePmaReaderNext(PmaReader *pReadr){ + int rc = SQLITE_OK; /* Return Code */ + u64 nRec = 0; /* Size of record in bytes */ - nBuf = sqlite3BtreeGetPageSize(db->aDb[0].pBt); - assert( pSorter->iWriteOff>iStart ); - assert( pIter->aAlloc==0 ); - assert( pIter->aBuffer==0 ); - pIter->pFile = pSorter->pTemp1; - pIter->iReadOff = iStart; - pIter->nAlloc = 128; - pIter->aAlloc = (u8 *)sqlite3DbMallocRaw(db, pIter->nAlloc); - pIter->nBuffer = nBuf; - pIter->aBuffer = (u8 *)sqlite3DbMallocRaw(db, nBuf); - - if( !pIter->aBuffer ){ - rc = SQLITE_NOMEM; - }else{ - int iBuf; - - iBuf = iStart % nBuf; - if( iBuf ){ - int nRead = nBuf - iBuf; - if( (iStart + nRead) > pSorter->iWriteOff ){ - nRead = (int)(pSorter->iWriteOff - iStart); + if( pReadr->iReadOff>=pReadr->iEof ){ + IncrMerger *pIncr = pReadr->pIncr; + int bEof = 1; + if( pIncr ){ + rc = vdbeIncrSwap(pIncr); + if( rc==SQLITE_OK && pIncr->bEof==0 ){ + rc = vdbePmaReaderSeek( + pIncr->pTask, pReadr, &pIncr->aFile[0], pIncr->iStartOff + ); + bEof = 0; } - rc = sqlite3OsRead( - pSorter->pTemp1, &pIter->aBuffer[iBuf], nRead, iStart - ); } - if( rc==SQLITE_OK ){ - u64 nByte; /* Size of PMA in bytes */ - pIter->iEof = pSorter->iWriteOff; - rc = vdbeSorterIterVarint(db, pIter, &nByte); - pIter->iEof = pIter->iReadOff + nByte; - *pnByte += nByte; + if( bEof ){ + /* This is an EOF condition */ + vdbePmaReaderClear(pReadr); + testcase( rc!=SQLITE_OK ); + return rc; } } if( rc==SQLITE_OK ){ - rc = vdbeSorterIterNext(db, pIter); + rc = vdbePmaReadVarint(pReadr, &nRec); + } + if( rc==SQLITE_OK ){ + pReadr->nKey = (int)nRec; + rc = vdbePmaReadBlob(pReadr, (int)nRec, &pReadr->aKey); + testcase( rc!=SQLITE_OK ); + } + + return rc; +} + +/* +** Initialize PmaReader pReadr to scan through the PMA stored in file pFile +** starting at offset iStart and ending at offset iEof-1. This function +** leaves the PmaReader pointing to the first key in the PMA (or EOF if the +** PMA is empty). +** +** If the pnByte parameter is NULL, then it is assumed that the file +** contains a single PMA, and that that PMA omits the initial length varint. +*/ +static int vdbePmaReaderInit( + SortSubtask *pTask, /* Task context */ + SorterFile *pFile, /* Sorter file to read from */ + i64 iStart, /* Start offset in pFile */ + PmaReader *pReadr, /* PmaReader to populate */ + i64 *pnByte /* IN/OUT: Increment this value by PMA size */ +){ + int rc; + + assert( pFile->iEof>iStart ); + assert( pReadr->aAlloc==0 && pReadr->nAlloc==0 ); + assert( pReadr->aBuffer==0 ); + assert( pReadr->aMap==0 ); + + rc = vdbePmaReaderSeek(pTask, pReadr, pFile, iStart); + if( rc==SQLITE_OK ){ + u64 nByte; /* Size of PMA in bytes */ + rc = vdbePmaReadVarint(pReadr, &nByte); + pReadr->iEof = pReadr->iReadOff + nByte; + *pnByte += nByte; + } + + if( rc==SQLITE_OK ){ + rc = vdbePmaReaderNext(pReadr); } return rc; } @@ -370,126 +750,128 @@ static int vdbeSorterIterInit( /* ** Compare key1 (buffer pKey1, size nKey1 bytes) with key2 (buffer pKey2, -** size nKey2 bytes). Argument pKeyInfo supplies the collation functions -** used by the comparison. If an error occurs, return an SQLite error code. -** Otherwise, return SQLITE_OK and set *pRes to a negative, zero or positive -** value, depending on whether key1 is smaller, equal to or larger than key2. +** size nKey2 bytes). Use (pTask->pKeyInfo) for the collation sequences +** used by the comparison. Return the result of the comparison. ** -** If the bOmitRowid argument is non-zero, assume both keys end in a rowid -** field. For the purposes of the comparison, ignore it. Also, if bOmitRowid -** is true and key1 contains even a single NULL value, it is considered to -** be less than key2. Even if key2 also contains NULL values. +** Before returning, object (pTask->pUnpacked) is populated with the +** unpacked version of key2. Or, if pKey2 is passed a NULL pointer, then it +** is assumed that the (pTask->pUnpacked) structure already contains the +** unpacked key to use as key2. ** -** If pKey2 is passed a NULL pointer, then it is assumed that the pCsr->aSpace -** has been allocated and contains an unpacked record that is used as key2. +** If an OOM error is encountered, (pTask->pUnpacked->error_rc) is set +** to SQLITE_NOMEM. */ -static void vdbeSorterCompare( - const VdbeCursor *pCsr, /* Cursor object (for pKeyInfo) */ - int nIgnore, /* Ignore the last nIgnore fields */ +static int vdbeSorterCompare( + SortSubtask *pTask, /* Subtask context (for pKeyInfo) */ const void *pKey1, int nKey1, /* Left side of comparison */ - const void *pKey2, int nKey2, /* Right side of comparison */ - int *pRes /* OUT: Result of comparison */ + const void *pKey2, int nKey2 /* Right side of comparison */ ){ - KeyInfo *pKeyInfo = pCsr->pKeyInfo; - VdbeSorter *pSorter = pCsr->pSorter; - UnpackedRecord *r2 = pSorter->pUnpacked; - int i; - + UnpackedRecord *r2 = pTask->pUnpacked; if( pKey2 ){ - sqlite3VdbeRecordUnpack(pKeyInfo, nKey2, pKey2, r2); + sqlite3VdbeRecordUnpack(pTask->pSorter->pKeyInfo, nKey2, pKey2, r2); } - - if( nIgnore ){ - r2->nField = pKeyInfo->nField - nIgnore; - assert( r2->nField>0 ); - for(i=0; inField; i++){ - if( r2->aMem[i].flags & MEM_Null ){ - *pRes = -1; - return; - } - } - assert( r2->default_rc==0 ); - } - - *pRes = sqlite3VdbeRecordCompare(nKey1, pKey1, r2, 0); -} - -/* -** This function is called to compare two iterator keys when merging -** multiple b-tree segments. Parameter iOut is the index of the aTree[] -** value to recalculate. -*/ -static int vdbeSorterDoCompare(const VdbeCursor *pCsr, int iOut){ - VdbeSorter *pSorter = pCsr->pSorter; - int i1; - int i2; - int iRes; - VdbeSorterIter *p1; - VdbeSorterIter *p2; - - assert( iOutnTree && iOut>0 ); - - if( iOut>=(pSorter->nTree/2) ){ - i1 = (iOut - pSorter->nTree/2) * 2; - i2 = i1 + 1; - }else{ - i1 = pSorter->aTree[iOut*2]; - i2 = pSorter->aTree[iOut*2+1]; - } - - p1 = &pSorter->aIter[i1]; - p2 = &pSorter->aIter[i2]; - - if( p1->pFile==0 ){ - iRes = i2; - }else if( p2->pFile==0 ){ - iRes = i1; - }else{ - int res; - assert( pCsr->pSorter->pUnpacked!=0 ); /* allocated in vdbeSorterMerge() */ - vdbeSorterCompare( - pCsr, 0, p1->aKey, p1->nKey, p2->aKey, p2->nKey, &res - ); - if( res<=0 ){ - iRes = i1; - }else{ - iRes = i2; - } - } - - pSorter->aTree[iOut] = iRes; - return SQLITE_OK; + return sqlite3VdbeRecordCompare(nKey1, pKey1, r2); } /* ** Initialize the temporary index cursor just opened as a sorter cursor. +** +** Usually, the sorter module uses the value of (pCsr->pKeyInfo->nField) +** to determine the number of fields that should be compared from the +** records being sorted. However, if the value passed as argument nField +** is non-zero and the sorter is able to guarantee a stable sort, nField +** is used instead. This is used when sorting records for a CREATE INDEX +** statement. In this case, keys are always delivered to the sorter in +** order of the primary key, which happens to be make up the final part +** of the records being sorted. So if the sort is stable, there is never +** any reason to compare PK fields and they can be ignored for a small +** performance boost. +** +** The sorter can guarantee a stable sort when running in single-threaded +** mode, but not in multi-threaded mode. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. */ -int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){ +int sqlite3VdbeSorterInit( + sqlite3 *db, /* Database connection (for malloc()) */ + int nField, /* Number of key fields in each record */ + VdbeCursor *pCsr /* Cursor that holds the new sorter */ +){ int pgsz; /* Page size of main database */ + int i; /* Used to iterate through aTask[] */ int mxCache; /* Cache size */ VdbeSorter *pSorter; /* The new sorter */ - char *d; /* Dummy */ + KeyInfo *pKeyInfo; /* Copy of pCsr->pKeyInfo with db==0 */ + int szKeyInfo; /* Size of pCsr->pKeyInfo in bytes */ + int sz; /* Size of pSorter in bytes */ + int rc = SQLITE_OK; +#if SQLITE_MAX_WORKER_THREADS==0 +# define nWorker 0 +#else + int nWorker; +#endif + + /* Initialize the upper limit on the number of worker threads */ +#if SQLITE_MAX_WORKER_THREADS>0 + if( sqlite3TempInMemory(db) || sqlite3GlobalConfig.bCoreMutex==0 ){ + nWorker = 0; + }else{ + nWorker = db->aLimit[SQLITE_LIMIT_WORKER_THREADS]; + } +#endif + + /* Do not allow the total number of threads (main thread + all workers) + ** to exceed the maximum merge count */ +#if SQLITE_MAX_WORKER_THREADS>=SORTER_MAX_MERGE_COUNT + if( nWorker>=SORTER_MAX_MERGE_COUNT ){ + nWorker = SORTER_MAX_MERGE_COUNT-1; + } +#endif assert( pCsr->pKeyInfo && pCsr->pBt==0 ); - pCsr->pSorter = pSorter = sqlite3DbMallocZero(db, sizeof(VdbeSorter)); + szKeyInfo = sizeof(KeyInfo) + (pCsr->pKeyInfo->nField-1)*sizeof(CollSeq*); + sz = sizeof(VdbeSorter) + nWorker * sizeof(SortSubtask); + + pSorter = (VdbeSorter*)sqlite3DbMallocZero(db, sz + szKeyInfo); + pCsr->pSorter = pSorter; if( pSorter==0 ){ - return SQLITE_NOMEM; - } - - pSorter->pUnpacked = sqlite3VdbeAllocUnpackedRecord(pCsr->pKeyInfo, 0, 0, &d); - if( pSorter->pUnpacked==0 ) return SQLITE_NOMEM; - assert( pSorter->pUnpacked==(UnpackedRecord *)d ); + rc = SQLITE_NOMEM; + }else{ + pSorter->pKeyInfo = pKeyInfo = (KeyInfo*)((u8*)pSorter + sz); + memcpy(pKeyInfo, pCsr->pKeyInfo, szKeyInfo); + pKeyInfo->db = 0; + if( nField && nWorker==0 ) pKeyInfo->nField = nField; + pSorter->pgsz = pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); + pSorter->nTask = nWorker + 1; + pSorter->bUseThreads = (pSorter->nTask>1); + pSorter->db = db; + for(i=0; inTask; i++){ + SortSubtask *pTask = &pSorter->aTask[i]; + pTask->pSorter = pSorter; + } - if( !sqlite3TempInMemory(db) ){ - pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); - pSorter->mnPmaSize = SORTER_MIN_WORKING * pgsz; - mxCache = db->aDb[0].pSchema->cache_size; - if( mxCachemxPmaSize = mxCache * pgsz; + if( !sqlite3TempInMemory(db) ){ + pSorter->mnPmaSize = SQLITE_SORTER_PMASZ * pgsz; + mxCache = db->aDb[0].pSchema->cache_size; + if( mxCachemxPmaSize = MIN((i64)mxCache*pgsz, SQLITE_MAX_MXPMASIZE); + + /* EVIDENCE-OF: R-26747-61719 When the application provides any amount of + ** scratch memory using SQLITE_CONFIG_SCRATCH, SQLite avoids unnecessary + ** large heap allocations. + */ + if( sqlite3GlobalConfig.pScratch==0 ){ + assert( pSorter->iMemory==0 ); + pSorter->nMemory = pgsz; + pSorter->list.aMemory = (u8*)sqlite3Malloc(pgsz); + if( !pSorter->list.aMemory ) rc = SQLITE_NOMEM; + } + } } - return SQLITE_OK; + return rc; } +#undef nWorker /* Defined at the top of this function */ /* ** Free the list of sorted records starting at pRecord. @@ -498,38 +880,233 @@ static void vdbeSorterRecordFree(sqlite3 *db, SorterRecord *pRecord){ SorterRecord *p; SorterRecord *pNext; for(p=pRecord; p; p=pNext){ - pNext = p->pNext; + pNext = p->u.pNext; sqlite3DbFree(db, p); } } +/* +** Free all resources owned by the object indicated by argument pTask. All +** fields of *pTask are zeroed before returning. +*/ +static void vdbeSortSubtaskCleanup(sqlite3 *db, SortSubtask *pTask){ + sqlite3DbFree(db, pTask->pUnpacked); + pTask->pUnpacked = 0; +#if SQLITE_MAX_WORKER_THREADS>0 + /* pTask->list.aMemory can only be non-zero if it was handed memory + ** from the main thread. That only occurs SQLITE_MAX_WORKER_THREADS>0 */ + if( pTask->list.aMemory ){ + sqlite3_free(pTask->list.aMemory); + pTask->list.aMemory = 0; + }else +#endif + { + assert( pTask->list.aMemory==0 ); + vdbeSorterRecordFree(0, pTask->list.pList); + } + pTask->list.pList = 0; + if( pTask->file.pFd ){ + sqlite3OsCloseFree(pTask->file.pFd); + pTask->file.pFd = 0; + pTask->file.iEof = 0; + } + if( pTask->file2.pFd ){ + sqlite3OsCloseFree(pTask->file2.pFd); + pTask->file2.pFd = 0; + pTask->file2.iEof = 0; + } +} + +#ifdef SQLITE_DEBUG_SORTER_THREADS +static void vdbeSorterWorkDebug(SortSubtask *pTask, const char *zEvent){ + i64 t; + int iTask = (pTask - pTask->pSorter->aTask); + sqlite3OsCurrentTimeInt64(pTask->pSorter->db->pVfs, &t); + fprintf(stderr, "%lld:%d %s\n", t, iTask, zEvent); +} +static void vdbeSorterRewindDebug(const char *zEvent){ + i64 t; + sqlite3OsCurrentTimeInt64(sqlite3_vfs_find(0), &t); + fprintf(stderr, "%lld:X %s\n", t, zEvent); +} +static void vdbeSorterPopulateDebug( + SortSubtask *pTask, + const char *zEvent +){ + i64 t; + int iTask = (pTask - pTask->pSorter->aTask); + sqlite3OsCurrentTimeInt64(pTask->pSorter->db->pVfs, &t); + fprintf(stderr, "%lld:bg%d %s\n", t, iTask, zEvent); +} +static void vdbeSorterBlockDebug( + SortSubtask *pTask, + int bBlocked, + const char *zEvent +){ + if( bBlocked ){ + i64 t; + sqlite3OsCurrentTimeInt64(pTask->pSorter->db->pVfs, &t); + fprintf(stderr, "%lld:main %s\n", t, zEvent); + } +} +#else +# define vdbeSorterWorkDebug(x,y) +# define vdbeSorterRewindDebug(y) +# define vdbeSorterPopulateDebug(x,y) +# define vdbeSorterBlockDebug(x,y,z) +#endif + +#if SQLITE_MAX_WORKER_THREADS>0 +/* +** Join thread pTask->thread. +*/ +static int vdbeSorterJoinThread(SortSubtask *pTask){ + int rc = SQLITE_OK; + if( pTask->pThread ){ +#ifdef SQLITE_DEBUG_SORTER_THREADS + int bDone = pTask->bDone; +#endif + void *pRet = SQLITE_INT_TO_PTR(SQLITE_ERROR); + vdbeSorterBlockDebug(pTask, !bDone, "enter"); + (void)sqlite3ThreadJoin(pTask->pThread, &pRet); + vdbeSorterBlockDebug(pTask, !bDone, "exit"); + rc = SQLITE_PTR_TO_INT(pRet); + assert( pTask->bDone==1 ); + pTask->bDone = 0; + pTask->pThread = 0; + } + return rc; +} + +/* +** Launch a background thread to run xTask(pIn). +*/ +static int vdbeSorterCreateThread( + SortSubtask *pTask, /* Thread will use this task object */ + void *(*xTask)(void*), /* Routine to run in a separate thread */ + void *pIn /* Argument passed into xTask() */ +){ + assert( pTask->pThread==0 && pTask->bDone==0 ); + return sqlite3ThreadCreate(&pTask->pThread, xTask, pIn); +} + +/* +** Join all outstanding threads launched by SorterWrite() to create +** level-0 PMAs. +*/ +static int vdbeSorterJoinAll(VdbeSorter *pSorter, int rcin){ + int rc = rcin; + int i; + + /* This function is always called by the main user thread. + ** + ** If this function is being called after SorterRewind() has been called, + ** it is possible that thread pSorter->aTask[pSorter->nTask-1].pThread + ** is currently attempt to join one of the other threads. To avoid a race + ** condition where this thread also attempts to join the same object, join + ** thread pSorter->aTask[pSorter->nTask-1].pThread first. */ + for(i=pSorter->nTask-1; i>=0; i--){ + SortSubtask *pTask = &pSorter->aTask[i]; + int rc2 = vdbeSorterJoinThread(pTask); + if( rc==SQLITE_OK ) rc = rc2; + } + return rc; +} +#else +# define vdbeSorterJoinAll(x,rcin) (rcin) +# define vdbeSorterJoinThread(pTask) SQLITE_OK +#endif + +/* +** Allocate a new MergeEngine object capable of handling up to +** nReader PmaReader inputs. +** +** nReader is automatically rounded up to the next power of two. +** nReader may not exceed SORTER_MAX_MERGE_COUNT even after rounding up. +*/ +static MergeEngine *vdbeMergeEngineNew(int nReader){ + int N = 2; /* Smallest power of two >= nReader */ + int nByte; /* Total bytes of space to allocate */ + MergeEngine *pNew; /* Pointer to allocated object to return */ + + assert( nReader<=SORTER_MAX_MERGE_COUNT ); + + while( NnTree = N; + pNew->pTask = 0; + pNew->aReadr = (PmaReader*)&pNew[1]; + pNew->aTree = (int*)&pNew->aReadr[N]; + } + return pNew; +} + +/* +** Free the MergeEngine object passed as the only argument. +*/ +static void vdbeMergeEngineFree(MergeEngine *pMerger){ + int i; + if( pMerger ){ + for(i=0; inTree; i++){ + vdbePmaReaderClear(&pMerger->aReadr[i]); + } + } + sqlite3_free(pMerger); +} + +/* +** Free all resources associated with the IncrMerger object indicated by +** the first argument. +*/ +static void vdbeIncrFree(IncrMerger *pIncr){ + if( pIncr ){ +#if SQLITE_MAX_WORKER_THREADS>0 + if( pIncr->bUseThread ){ + vdbeSorterJoinThread(pIncr->pTask); + if( pIncr->aFile[0].pFd ) sqlite3OsCloseFree(pIncr->aFile[0].pFd); + if( pIncr->aFile[1].pFd ) sqlite3OsCloseFree(pIncr->aFile[1].pFd); + } +#endif + vdbeMergeEngineFree(pIncr->pMerger); + sqlite3_free(pIncr); + } +} + /* ** Reset a sorting cursor back to its original empty state. */ void sqlite3VdbeSorterReset(sqlite3 *db, VdbeSorter *pSorter){ - if( pSorter->aIter ){ - int i; - for(i=0; inTree; i++){ - vdbeSorterIterZero(db, &pSorter->aIter[i]); - } - sqlite3DbFree(db, pSorter->aIter); - pSorter->aIter = 0; + int i; + (void)vdbeSorterJoinAll(pSorter, SQLITE_OK); + assert( pSorter->bUseThreads || pSorter->pReader==0 ); +#if SQLITE_MAX_WORKER_THREADS>0 + if( pSorter->pReader ){ + vdbePmaReaderClear(pSorter->pReader); + sqlite3DbFree(db, pSorter->pReader); + pSorter->pReader = 0; } - if( pSorter->pTemp1 ){ - sqlite3OsCloseFree(pSorter->pTemp1); - pSorter->pTemp1 = 0; +#endif + vdbeMergeEngineFree(pSorter->pMerger); + pSorter->pMerger = 0; + for(i=0; inTask; i++){ + SortSubtask *pTask = &pSorter->aTask[i]; + vdbeSortSubtaskCleanup(db, pTask); } - vdbeSorterRecordFree(db, pSorter->pRecord); - pSorter->pRecord = 0; - pSorter->iWriteOff = 0; - pSorter->iReadOff = 0; - pSorter->nInMemory = 0; - pSorter->nTree = 0; - pSorter->nPMA = 0; - pSorter->aTree = 0; + if( pSorter->list.aMemory==0 ){ + vdbeSorterRecordFree(0, pSorter->list.pList); + } + pSorter->list.pList = 0; + pSorter->list.szPMA = 0; + pSorter->bUsePMA = 0; + pSorter->iMemory = 0; + pSorter->mxKeysize = 0; + sqlite3DbFree(db, pSorter->pUnpacked); + pSorter->pUnpacked = 0; } - /* ** Free any cursor components allocated by sqlite3VdbeSorterXXX routines. */ @@ -537,54 +1114,110 @@ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ VdbeSorter *pSorter = pCsr->pSorter; if( pSorter ){ sqlite3VdbeSorterReset(db, pSorter); - sqlite3DbFree(db, pSorter->pUnpacked); + sqlite3_free(pSorter->list.aMemory); sqlite3DbFree(db, pSorter); pCsr->pSorter = 0; } } +#if SQLITE_MAX_MMAP_SIZE>0 +/* +** The first argument is a file-handle open on a temporary file. The file +** is guaranteed to be nByte bytes or smaller in size. This function +** attempts to extend the file to nByte bytes in size and to ensure that +** the VFS has memory mapped it. +** +** Whether or not the file does end up memory mapped of course depends on +** the specific VFS implementation. +*/ +static void vdbeSorterExtendFile(sqlite3 *db, sqlite3_file *pFd, i64 nByte){ + if( nByte<=(i64)(db->nMaxSorterMmap) && pFd->pMethods->iVersion>=3 ){ + void *p = 0; + int chunksize = 4*1024; + sqlite3OsFileControlHint(pFd, SQLITE_FCNTL_CHUNK_SIZE, &chunksize); + sqlite3OsFileControlHint(pFd, SQLITE_FCNTL_SIZE_HINT, &nByte); + sqlite3OsFetch(pFd, 0, (int)nByte, &p); + sqlite3OsUnfetch(pFd, 0, p); + } +} +#else +# define vdbeSorterExtendFile(x,y,z) +#endif + /* ** Allocate space for a file-handle and open a temporary file. If successful, -** set *ppFile to point to the malloc'd file-handle and return SQLITE_OK. -** Otherwise, set *ppFile to 0 and return an SQLite error code. +** set *ppFd to point to the malloc'd file-handle and return SQLITE_OK. +** Otherwise, set *ppFd to 0 and return an SQLite error code. */ -static int vdbeSorterOpenTempFile(sqlite3 *db, sqlite3_file **ppFile){ - int dummy; - return sqlite3OsOpenMalloc(db->pVfs, 0, ppFile, +static int vdbeSorterOpenTempFile( + sqlite3 *db, /* Database handle doing sort */ + i64 nExtend, /* Attempt to extend file to this size */ + sqlite3_file **ppFd +){ + int rc; + rc = sqlite3OsOpenMalloc(db->pVfs, 0, ppFd, SQLITE_OPEN_TEMP_JOURNAL | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | - SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE, &dummy + SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE, &rc ); + if( rc==SQLITE_OK ){ + i64 max = SQLITE_MAX_MMAP_SIZE; + sqlite3OsFileControlHint(*ppFd, SQLITE_FCNTL_MMAP_SIZE, (void*)&max); + if( nExtend>0 ){ + vdbeSorterExtendFile(db, *ppFd, nExtend); + } + } + return rc; } +/* +** If it has not already been allocated, allocate the UnpackedRecord +** structure at pTask->pUnpacked. Return SQLITE_OK if successful (or +** if no allocation was required), or SQLITE_NOMEM otherwise. +*/ +static int vdbeSortAllocUnpacked(SortSubtask *pTask){ + if( pTask->pUnpacked==0 ){ + char *pFree; + pTask->pUnpacked = sqlite3VdbeAllocUnpackedRecord( + pTask->pSorter->pKeyInfo, 0, 0, &pFree + ); + assert( pTask->pUnpacked==(UnpackedRecord*)pFree ); + if( pFree==0 ) return SQLITE_NOMEM; + pTask->pUnpacked->nField = pTask->pSorter->pKeyInfo->nField; + pTask->pUnpacked->errCode = 0; + } + return SQLITE_OK; +} + + /* ** Merge the two sorted lists p1 and p2 into a single list. ** Set *ppOut to the head of the new list. */ static void vdbeSorterMerge( - const VdbeCursor *pCsr, /* For pKeyInfo */ + SortSubtask *pTask, /* Calling thread context */ SorterRecord *p1, /* First list to merge */ SorterRecord *p2, /* Second list to merge */ SorterRecord **ppOut /* OUT: Head of merged list */ ){ SorterRecord *pFinal = 0; SorterRecord **pp = &pFinal; - void *pVal2 = p2 ? p2->pVal : 0; + void *pVal2 = p2 ? SRVAL(p2) : 0; while( p1 && p2 ){ int res; - vdbeSorterCompare(pCsr, 0, p1->pVal, p1->nVal, pVal2, p2->nVal, &res); + res = vdbeSorterCompare(pTask, SRVAL(p1), p1->nVal, pVal2, p2->nVal); if( res<=0 ){ *pp = p1; - pp = &p1->pNext; - p1 = p1->pNext; + pp = &p1->u.pNext; + p1 = p1->u.pNext; pVal2 = 0; }else{ *pp = p2; - pp = &p2->pNext; - p2 = p2->pNext; + pp = &p2->u.pNext; + p2 = p2->u.pNext; if( p2==0 ) break; - pVal2 = p2->pVal; + pVal2 = SRVAL(p2); } } *pp = p1 ? p1 : p2; @@ -592,27 +1225,41 @@ static void vdbeSorterMerge( } /* -** Sort the linked list of records headed at pCsr->pRecord. Return SQLITE_OK -** if successful, or an SQLite error code (i.e. SQLITE_NOMEM) if an error -** occurs. +** Sort the linked list of records headed at pTask->pList. Return +** SQLITE_OK if successful, or an SQLite error code (i.e. SQLITE_NOMEM) if +** an error occurs. */ -static int vdbeSorterSort(const VdbeCursor *pCsr){ +static int vdbeSorterSort(SortSubtask *pTask, SorterList *pList){ int i; SorterRecord **aSlot; SorterRecord *p; - VdbeSorter *pSorter = pCsr->pSorter; + int rc; + + rc = vdbeSortAllocUnpacked(pTask); + if( rc!=SQLITE_OK ) return rc; aSlot = (SorterRecord **)sqlite3MallocZero(64 * sizeof(SorterRecord *)); if( !aSlot ){ return SQLITE_NOMEM; } - p = pSorter->pRecord; + p = pList->pList; while( p ){ - SorterRecord *pNext = p->pNext; - p->pNext = 0; + SorterRecord *pNext; + if( pList->aMemory ){ + if( (u8*)p==pList->aMemory ){ + pNext = 0; + }else{ + assert( p->u.iNextaMemory) ); + pNext = (SorterRecord*)&pList->aMemory[p->u.iNext]; + } + }else{ + pNext = p->u.pNext; + } + + p->u.pNext = 0; for(i=0; aSlot[i]; i++){ - vdbeSorterMerge(pCsr, p, aSlot[i], &p); + vdbeSorterMerge(pTask, p, aSlot[i], &p); aSlot[i] = 0; } aSlot[i] = p; @@ -621,42 +1268,43 @@ static int vdbeSorterSort(const VdbeCursor *pCsr){ p = 0; for(i=0; i<64; i++){ - vdbeSorterMerge(pCsr, p, aSlot[i], &p); + vdbeSorterMerge(pTask, p, aSlot[i], &p); } - pSorter->pRecord = p; + pList->pList = p; sqlite3_free(aSlot); - return SQLITE_OK; + assert( pTask->pUnpacked->errCode==SQLITE_OK + || pTask->pUnpacked->errCode==SQLITE_NOMEM + ); + return pTask->pUnpacked->errCode; } /* -** Initialize a file-writer object. +** Initialize a PMA-writer object. */ -static void fileWriterInit( - sqlite3 *db, /* Database (for malloc) */ - sqlite3_file *pFile, /* File to write to */ - FileWriter *p, /* Object to populate */ - i64 iStart /* Offset of pFile to begin writing at */ +static void vdbePmaWriterInit( + sqlite3_file *pFd, /* File handle to write to */ + PmaWriter *p, /* Object to populate */ + int nBuf, /* Buffer size */ + i64 iStart /* Offset of pFd to begin writing at */ ){ - int nBuf = sqlite3BtreeGetPageSize(db->aDb[0].pBt); - - memset(p, 0, sizeof(FileWriter)); - p->aBuffer = (u8 *)sqlite3DbMallocRaw(db, nBuf); + memset(p, 0, sizeof(PmaWriter)); + p->aBuffer = (u8*)sqlite3Malloc(nBuf); if( !p->aBuffer ){ p->eFWErr = SQLITE_NOMEM; }else{ p->iBufEnd = p->iBufStart = (iStart % nBuf); p->iWriteOff = iStart - p->iBufStart; p->nBuffer = nBuf; - p->pFile = pFile; + p->pFd = pFd; } } /* -** Write nData bytes of data to the file-write object. Return SQLITE_OK +** Write nData bytes of data to the PMA. Return SQLITE_OK ** if successful, or an SQLite error code if an error occurs. */ -static void fileWriterWrite(FileWriter *p, u8 *pData, int nData){ +static void vdbePmaWriteBlob(PmaWriter *p, u8 *pData, int nData){ int nRem = nData; while( nRem>0 && p->eFWErr==0 ){ int nCopy = nRem; @@ -667,7 +1315,7 @@ static void fileWriterWrite(FileWriter *p, u8 *pData, int nData){ memcpy(&p->aBuffer[p->iBufEnd], &pData[nData-nRem], nCopy); p->iBufEnd += nCopy; if( p->iBufEnd==p->nBuffer ){ - p->eFWErr = sqlite3OsWrite(p->pFile, + p->eFWErr = sqlite3OsWrite(p->pFd, &p->aBuffer[p->iBufStart], p->iBufEnd - p->iBufStart, p->iWriteOff + p->iBufStart ); @@ -681,43 +1329,44 @@ static void fileWriterWrite(FileWriter *p, u8 *pData, int nData){ } /* -** Flush any buffered data to disk and clean up the file-writer object. -** The results of using the file-writer after this call are undefined. +** Flush any buffered data to disk and clean up the PMA-writer object. +** The results of using the PMA-writer after this call are undefined. ** Return SQLITE_OK if flushing the buffered data succeeds or is not ** required. Otherwise, return an SQLite error code. ** ** Before returning, set *piEof to the offset immediately following the ** last byte written to the file. */ -static int fileWriterFinish(sqlite3 *db, FileWriter *p, i64 *piEof){ +static int vdbePmaWriterFinish(PmaWriter *p, i64 *piEof){ int rc; if( p->eFWErr==0 && ALWAYS(p->aBuffer) && p->iBufEnd>p->iBufStart ){ - p->eFWErr = sqlite3OsWrite(p->pFile, + p->eFWErr = sqlite3OsWrite(p->pFd, &p->aBuffer[p->iBufStart], p->iBufEnd - p->iBufStart, p->iWriteOff + p->iBufStart ); } *piEof = (p->iWriteOff + p->iBufEnd); - sqlite3DbFree(db, p->aBuffer); + sqlite3_free(p->aBuffer); rc = p->eFWErr; - memset(p, 0, sizeof(FileWriter)); + memset(p, 0, sizeof(PmaWriter)); return rc; } /* -** Write value iVal encoded as a varint to the file-write object. Return +** Write value iVal encoded as a varint to the PMA. Return ** SQLITE_OK if successful, or an SQLite error code if an error occurs. */ -static void fileWriterWriteVarint(FileWriter *p, u64 iVal){ +static void vdbePmaWriteVarint(PmaWriter *p, u64 iVal){ int nByte; u8 aByte[10]; nByte = sqlite3PutVarint(aByte, iVal); - fileWriterWrite(p, aByte, nByte); + vdbePmaWriteBlob(p, aByte, nByte); } /* -** Write the current contents of the in-memory linked-list to a PMA. Return -** SQLITE_OK if successful, or an SQLite error code otherwise. +** Write the current contents of in-memory linked-list pList to a level-0 +** PMA in the temp file belonging to sub-task pTask. Return SQLITE_OK if +** successful, or an SQLite error code otherwise. ** ** The format of a PMA is: ** @@ -728,76 +1377,246 @@ static void fileWriterWriteVarint(FileWriter *p, u64 iVal){ ** Each record consists of a varint followed by a blob of data (the ** key). The varint is the number of bytes in the blob of data. */ -static int vdbeSorterListToPMA(sqlite3 *db, const VdbeCursor *pCsr){ +static int vdbeSorterListToPMA(SortSubtask *pTask, SorterList *pList){ + sqlite3 *db = pTask->pSorter->db; int rc = SQLITE_OK; /* Return code */ - VdbeSorter *pSorter = pCsr->pSorter; - FileWriter writer; + PmaWriter writer; /* Object used to write to the file */ - memset(&writer, 0, sizeof(FileWriter)); +#ifdef SQLITE_DEBUG + /* Set iSz to the expected size of file pTask->file after writing the PMA. + ** This is used by an assert() statement at the end of this function. */ + i64 iSz = pList->szPMA + sqlite3VarintLen(pList->szPMA) + pTask->file.iEof; +#endif - if( pSorter->nInMemory==0 ){ - assert( pSorter->pRecord==0 ); - return rc; - } - - rc = vdbeSorterSort(pCsr); + vdbeSorterWorkDebug(pTask, "enter"); + memset(&writer, 0, sizeof(PmaWriter)); + assert( pList->szPMA>0 ); /* If the first temporary PMA file has not been opened, open it now. */ - if( rc==SQLITE_OK && pSorter->pTemp1==0 ){ - rc = vdbeSorterOpenTempFile(db, &pSorter->pTemp1); - assert( rc!=SQLITE_OK || pSorter->pTemp1 ); - assert( pSorter->iWriteOff==0 ); - assert( pSorter->nPMA==0 ); + if( pTask->file.pFd==0 ){ + rc = vdbeSorterOpenTempFile(db, 0, &pTask->file.pFd); + assert( rc!=SQLITE_OK || pTask->file.pFd ); + assert( pTask->file.iEof==0 ); + assert( pTask->nPMA==0 ); + } + + /* Try to get the file to memory map */ + if( rc==SQLITE_OK ){ + vdbeSorterExtendFile(db, pTask->file.pFd, pTask->file.iEof+pList->szPMA+9); + } + + /* Sort the list */ + if( rc==SQLITE_OK ){ + rc = vdbeSorterSort(pTask, pList); } if( rc==SQLITE_OK ){ SorterRecord *p; SorterRecord *pNext = 0; - fileWriterInit(db, pSorter->pTemp1, &writer, pSorter->iWriteOff); - pSorter->nPMA++; - fileWriterWriteVarint(&writer, pSorter->nInMemory); - for(p=pSorter->pRecord; p; p=pNext){ - pNext = p->pNext; - fileWriterWriteVarint(&writer, p->nVal); - fileWriterWrite(&writer, p->pVal, p->nVal); - sqlite3DbFree(db, p); + vdbePmaWriterInit(pTask->file.pFd, &writer, pTask->pSorter->pgsz, + pTask->file.iEof); + pTask->nPMA++; + vdbePmaWriteVarint(&writer, pList->szPMA); + for(p=pList->pList; p; p=pNext){ + pNext = p->u.pNext; + vdbePmaWriteVarint(&writer, p->nVal); + vdbePmaWriteBlob(&writer, SRVAL(p), p->nVal); + if( pList->aMemory==0 ) sqlite3_free(p); + } + pList->pList = p; + rc = vdbePmaWriterFinish(&writer, &pTask->file.iEof); + } + + vdbeSorterWorkDebug(pTask, "exit"); + assert( rc!=SQLITE_OK || pList->pList==0 ); + assert( rc!=SQLITE_OK || pTask->file.iEof==iSz ); + return rc; +} + +/* +** Advance the MergeEngine to its next entry. +** Set *pbEof to true there is no next entry because +** the MergeEngine has reached the end of all its inputs. +** +** Return SQLITE_OK if successful or an error code if an error occurs. +*/ +static int vdbeMergeEngineStep( + MergeEngine *pMerger, /* The merge engine to advance to the next row */ + int *pbEof /* Set TRUE at EOF. Set false for more content */ +){ + int rc; + int iPrev = pMerger->aTree[1];/* Index of PmaReader to advance */ + SortSubtask *pTask = pMerger->pTask; + + /* Advance the current PmaReader */ + rc = vdbePmaReaderNext(&pMerger->aReadr[iPrev]); + + /* Update contents of aTree[] */ + if( rc==SQLITE_OK ){ + int i; /* Index of aTree[] to recalculate */ + PmaReader *pReadr1; /* First PmaReader to compare */ + PmaReader *pReadr2; /* Second PmaReader to compare */ + u8 *pKey2; /* To pReadr2->aKey, or 0 if record cached */ + + /* Find the first two PmaReaders to compare. The one that was just + ** advanced (iPrev) and the one next to it in the array. */ + pReadr1 = &pMerger->aReadr[(iPrev & 0xFFFE)]; + pReadr2 = &pMerger->aReadr[(iPrev | 0x0001)]; + pKey2 = pReadr2->aKey; + + for(i=(pMerger->nTree+iPrev)/2; i>0; i=i/2){ + /* Compare pReadr1 and pReadr2. Store the result in variable iRes. */ + int iRes; + if( pReadr1->pFd==0 ){ + iRes = +1; + }else if( pReadr2->pFd==0 ){ + iRes = -1; + }else{ + iRes = vdbeSorterCompare(pTask, + pReadr1->aKey, pReadr1->nKey, pKey2, pReadr2->nKey + ); + } + + /* If pReadr1 contained the smaller value, set aTree[i] to its index. + ** Then set pReadr2 to the next PmaReader to compare to pReadr1. In this + ** case there is no cache of pReadr2 in pTask->pUnpacked, so set + ** pKey2 to point to the record belonging to pReadr2. + ** + ** Alternatively, if pReadr2 contains the smaller of the two values, + ** set aTree[i] to its index and update pReadr1. If vdbeSorterCompare() + ** was actually called above, then pTask->pUnpacked now contains + ** a value equivalent to pReadr2. So set pKey2 to NULL to prevent + ** vdbeSorterCompare() from decoding pReadr2 again. + ** + ** If the two values were equal, then the value from the oldest + ** PMA should be considered smaller. The VdbeSorter.aReadr[] array + ** is sorted from oldest to newest, so pReadr1 contains older values + ** than pReadr2 iff (pReadr1aTree[i] = (int)(pReadr1 - pMerger->aReadr); + pReadr2 = &pMerger->aReadr[ pMerger->aTree[i ^ 0x0001] ]; + pKey2 = pReadr2->aKey; + }else{ + if( pReadr1->pFd ) pKey2 = 0; + pMerger->aTree[i] = (int)(pReadr2 - pMerger->aReadr); + pReadr1 = &pMerger->aReadr[ pMerger->aTree[i ^ 0x0001] ]; + } + } + *pbEof = (pMerger->aReadr[pMerger->aTree[1]].pFd==0); + } + + return (rc==SQLITE_OK ? pTask->pUnpacked->errCode : rc); +} + +#if SQLITE_MAX_WORKER_THREADS>0 +/* +** The main routine for background threads that write level-0 PMAs. +*/ +static void *vdbeSorterFlushThread(void *pCtx){ + SortSubtask *pTask = (SortSubtask*)pCtx; + int rc; /* Return code */ + assert( pTask->bDone==0 ); + rc = vdbeSorterListToPMA(pTask, &pTask->list); + pTask->bDone = 1; + return SQLITE_INT_TO_PTR(rc); +} +#endif /* SQLITE_MAX_WORKER_THREADS>0 */ + +/* +** Flush the current contents of VdbeSorter.list to a new PMA, possibly +** using a background thread. +*/ +static int vdbeSorterFlushPMA(VdbeSorter *pSorter){ +#if SQLITE_MAX_WORKER_THREADS==0 + pSorter->bUsePMA = 1; + return vdbeSorterListToPMA(&pSorter->aTask[0], &pSorter->list); +#else + int rc = SQLITE_OK; + int i; + SortSubtask *pTask = 0; /* Thread context used to create new PMA */ + int nWorker = (pSorter->nTask-1); + + /* Set the flag to indicate that at least one PMA has been written. + ** Or will be, anyhow. */ + pSorter->bUsePMA = 1; + + /* Select a sub-task to sort and flush the current list of in-memory + ** records to disk. If the sorter is running in multi-threaded mode, + ** round-robin between the first (pSorter->nTask-1) tasks. Except, if + ** the background thread from a sub-tasks previous turn is still running, + ** skip it. If the first (pSorter->nTask-1) sub-tasks are all still busy, + ** fall back to using the final sub-task. The first (pSorter->nTask-1) + ** sub-tasks are prefered as they use background threads - the final + ** sub-task uses the main thread. */ + for(i=0; iiPrev + i + 1) % nWorker; + pTask = &pSorter->aTask[iTest]; + if( pTask->bDone ){ + rc = vdbeSorterJoinThread(pTask); + } + if( rc!=SQLITE_OK || pTask->pThread==0 ) break; + } + + if( rc==SQLITE_OK ){ + if( i==nWorker ){ + /* Use the foreground thread for this operation */ + rc = vdbeSorterListToPMA(&pSorter->aTask[nWorker], &pSorter->list); + }else{ + /* Launch a background thread for this operation */ + u8 *aMem = pTask->list.aMemory; + void *pCtx = (void*)pTask; + + assert( pTask->pThread==0 && pTask->bDone==0 ); + assert( pTask->list.pList==0 ); + assert( pTask->list.aMemory==0 || pSorter->list.aMemory!=0 ); + + pSorter->iPrev = (u8)(pTask - pSorter->aTask); + pTask->list = pSorter->list; + pSorter->list.pList = 0; + pSorter->list.szPMA = 0; + if( aMem ){ + pSorter->list.aMemory = aMem; + pSorter->nMemory = sqlite3MallocSize(aMem); + }else if( pSorter->list.aMemory ){ + pSorter->list.aMemory = sqlite3Malloc(pSorter->nMemory); + if( !pSorter->list.aMemory ) return SQLITE_NOMEM; + } + + rc = vdbeSorterCreateThread(pTask, vdbeSorterFlushThread, pCtx); } - pSorter->pRecord = p; - rc = fileWriterFinish(db, &writer, &pSorter->iWriteOff); } return rc; +#endif /* SQLITE_MAX_WORKER_THREADS!=0 */ } /* ** Add a record to the sorter. */ int sqlite3VdbeSorterWrite( - sqlite3 *db, /* Database handle */ - const VdbeCursor *pCsr, /* Sorter cursor */ + const VdbeCursor *pCsr, /* Sorter cursor */ Mem *pVal /* Memory cell containing record */ ){ VdbeSorter *pSorter = pCsr->pSorter; int rc = SQLITE_OK; /* Return Code */ SorterRecord *pNew; /* New list element */ + int bFlush; /* True to flush contents of memory to PMA */ + int nReq; /* Bytes of memory required */ + int nPMA; /* Bytes of PMA space required */ + assert( pSorter ); - pSorter->nInMemory += sqlite3VarintLen(pVal->n) + pVal->n; - pNew = (SorterRecord *)sqlite3DbMallocRaw(db, pVal->n + sizeof(SorterRecord)); - if( pNew==0 ){ - rc = SQLITE_NOMEM; - }else{ - pNew->pVal = (void *)&pNew[1]; - memcpy(pNew->pVal, pVal->z, pVal->n); - pNew->nVal = pVal->n; - pNew->pNext = pSorter->pRecord; - pSorter->pRecord = pNew; - } - - /* See if the contents of the sorter should now be written out. They - ** are written out when either of the following are true: + /* Figure out whether or not the current contents of memory should be + ** flushed to a PMA before continuing. If so, do so. + ** + ** If using the single large allocation mode (pSorter->aMemory!=0), then + ** flush the contents of memory to a new PMA if (a) at least one value is + ** already in memory and (b) the new value will not fit in memory. + ** + ** Or, if using separate allocations for each record, flush the contents + ** of memory to a PMA if either of the following are true: ** ** * The total memory allocated for the in-memory list is greater ** than (page-size * cache-size), or @@ -805,161 +1624,778 @@ int sqlite3VdbeSorterWrite( ** * The total memory allocated for the in-memory list is greater ** than (page-size * 10) and sqlite3HeapNearlyFull() returns true. */ - if( rc==SQLITE_OK && pSorter->mxPmaSize>0 && ( - (pSorter->nInMemory>pSorter->mxPmaSize) - || (pSorter->nInMemory>pSorter->mnPmaSize && sqlite3HeapNearlyFull()) - )){ -#ifdef SQLITE_DEBUG - i64 nExpect = pSorter->iWriteOff - + sqlite3VarintLen(pSorter->nInMemory) - + pSorter->nInMemory; + nReq = pVal->n + sizeof(SorterRecord); + nPMA = pVal->n + sqlite3VarintLen(pVal->n); + if( pSorter->mxPmaSize ){ + if( pSorter->list.aMemory ){ + bFlush = pSorter->iMemory && (pSorter->iMemory+nReq) > pSorter->mxPmaSize; + }else{ + bFlush = ( + (pSorter->list.szPMA > pSorter->mxPmaSize) + || (pSorter->list.szPMA > pSorter->mnPmaSize && sqlite3HeapNearlyFull()) + ); + } + if( bFlush ){ + rc = vdbeSorterFlushPMA(pSorter); + pSorter->list.szPMA = 0; + pSorter->iMemory = 0; + assert( rc!=SQLITE_OK || pSorter->list.pList==0 ); + } + } + + pSorter->list.szPMA += nPMA; + if( nPMA>pSorter->mxKeysize ){ + pSorter->mxKeysize = nPMA; + } + + if( pSorter->list.aMemory ){ + int nMin = pSorter->iMemory + nReq; + + if( nMin>pSorter->nMemory ){ + u8 *aNew; + int nNew = pSorter->nMemory * 2; + while( nNew < nMin ) nNew = nNew*2; + if( nNew > pSorter->mxPmaSize ) nNew = pSorter->mxPmaSize; + if( nNew < nMin ) nNew = nMin; + + aNew = sqlite3Realloc(pSorter->list.aMemory, nNew); + if( !aNew ) return SQLITE_NOMEM; + pSorter->list.pList = (SorterRecord*)( + aNew + ((u8*)pSorter->list.pList - pSorter->list.aMemory) + ); + pSorter->list.aMemory = aNew; + pSorter->nMemory = nNew; + } + + pNew = (SorterRecord*)&pSorter->list.aMemory[pSorter->iMemory]; + pSorter->iMemory += ROUND8(nReq); + pNew->u.iNext = (int)((u8*)(pSorter->list.pList) - pSorter->list.aMemory); + }else{ + pNew = (SorterRecord *)sqlite3Malloc(nReq); + if( pNew==0 ){ + return SQLITE_NOMEM; + } + pNew->u.pNext = pSorter->list.pList; + } + + memcpy(SRVAL(pNew), pVal->z, pVal->n); + pNew->nVal = pVal->n; + pSorter->list.pList = pNew; + + return rc; +} + +/* +** Read keys from pIncr->pMerger and populate pIncr->aFile[1]. The format +** of the data stored in aFile[1] is the same as that used by regular PMAs, +** except that the number-of-bytes varint is omitted from the start. +*/ +static int vdbeIncrPopulate(IncrMerger *pIncr){ + int rc = SQLITE_OK; + int rc2; + i64 iStart = pIncr->iStartOff; + SorterFile *pOut = &pIncr->aFile[1]; + SortSubtask *pTask = pIncr->pTask; + MergeEngine *pMerger = pIncr->pMerger; + PmaWriter writer; + assert( pIncr->bEof==0 ); + + vdbeSorterPopulateDebug(pTask, "enter"); + + vdbePmaWriterInit(pOut->pFd, &writer, pTask->pSorter->pgsz, iStart); + while( rc==SQLITE_OK ){ + int dummy; + PmaReader *pReader = &pMerger->aReadr[ pMerger->aTree[1] ]; + int nKey = pReader->nKey; + i64 iEof = writer.iWriteOff + writer.iBufEnd; + + /* Check if the output file is full or if the input has been exhausted. + ** In either case exit the loop. */ + if( pReader->pFd==0 ) break; + if( (iEof + nKey + sqlite3VarintLen(nKey))>(iStart + pIncr->mxSz) ) break; + + /* Write the next key to the output. */ + vdbePmaWriteVarint(&writer, nKey); + vdbePmaWriteBlob(&writer, pReader->aKey, nKey); + assert( pIncr->pMerger->pTask==pTask ); + rc = vdbeMergeEngineStep(pIncr->pMerger, &dummy); + } + + rc2 = vdbePmaWriterFinish(&writer, &pOut->iEof); + if( rc==SQLITE_OK ) rc = rc2; + vdbeSorterPopulateDebug(pTask, "exit"); + return rc; +} + +#if SQLITE_MAX_WORKER_THREADS>0 +/* +** The main routine for background threads that populate aFile[1] of +** multi-threaded IncrMerger objects. +*/ +static void *vdbeIncrPopulateThread(void *pCtx){ + IncrMerger *pIncr = (IncrMerger*)pCtx; + void *pRet = SQLITE_INT_TO_PTR( vdbeIncrPopulate(pIncr) ); + pIncr->pTask->bDone = 1; + return pRet; +} + +/* +** Launch a background thread to populate aFile[1] of pIncr. +*/ +static int vdbeIncrBgPopulate(IncrMerger *pIncr){ + void *p = (void*)pIncr; + assert( pIncr->bUseThread ); + return vdbeSorterCreateThread(pIncr->pTask, vdbeIncrPopulateThread, p); +} #endif - rc = vdbeSorterListToPMA(db, pCsr); - pSorter->nInMemory = 0; - assert( rc!=SQLITE_OK || (nExpect==pSorter->iWriteOff) ); + +/* +** This function is called when the PmaReader corresponding to pIncr has +** finished reading the contents of aFile[0]. Its purpose is to "refill" +** aFile[0] such that the PmaReader should start rereading it from the +** beginning. +** +** For single-threaded objects, this is accomplished by literally reading +** keys from pIncr->pMerger and repopulating aFile[0]. +** +** For multi-threaded objects, all that is required is to wait until the +** background thread is finished (if it is not already) and then swap +** aFile[0] and aFile[1] in place. If the contents of pMerger have not +** been exhausted, this function also launches a new background thread +** to populate the new aFile[1]. +** +** SQLITE_OK is returned on success, or an SQLite error code otherwise. +*/ +static int vdbeIncrSwap(IncrMerger *pIncr){ + int rc = SQLITE_OK; + +#if SQLITE_MAX_WORKER_THREADS>0 + if( pIncr->bUseThread ){ + rc = vdbeSorterJoinThread(pIncr->pTask); + + if( rc==SQLITE_OK ){ + SorterFile f0 = pIncr->aFile[0]; + pIncr->aFile[0] = pIncr->aFile[1]; + pIncr->aFile[1] = f0; + } + + if( rc==SQLITE_OK ){ + if( pIncr->aFile[0].iEof==pIncr->iStartOff ){ + pIncr->bEof = 1; + }else{ + rc = vdbeIncrBgPopulate(pIncr); + } + } + }else +#endif + { + rc = vdbeIncrPopulate(pIncr); + pIncr->aFile[0] = pIncr->aFile[1]; + if( pIncr->aFile[0].iEof==pIncr->iStartOff ){ + pIncr->bEof = 1; + } } return rc; } /* -** Helper function for sqlite3VdbeSorterRewind(). +** Allocate and return a new IncrMerger object to read data from pMerger. +** +** If an OOM condition is encountered, return NULL. In this case free the +** pMerger argument before returning. */ -static int vdbeSorterInitMerge( - sqlite3 *db, /* Database handle */ - const VdbeCursor *pCsr, /* Cursor handle for this sorter */ - i64 *pnByte /* Sum of bytes in all opened PMAs */ +static int vdbeIncrMergerNew( + SortSubtask *pTask, /* The thread that will be using the new IncrMerger */ + MergeEngine *pMerger, /* The MergeEngine that the IncrMerger will control */ + IncrMerger **ppOut /* Write the new IncrMerger here */ ){ + int rc = SQLITE_OK; + IncrMerger *pIncr = *ppOut = (IncrMerger*) + (sqlite3FaultSim(100) ? 0 : sqlite3MallocZero(sizeof(*pIncr))); + if( pIncr ){ + pIncr->pMerger = pMerger; + pIncr->pTask = pTask; + pIncr->mxSz = MAX(pTask->pSorter->mxKeysize+9,pTask->pSorter->mxPmaSize/2); + pTask->file2.iEof += pIncr->mxSz; + }else{ + vdbeMergeEngineFree(pMerger); + rc = SQLITE_NOMEM; + } + return rc; +} + +#if SQLITE_MAX_WORKER_THREADS>0 +/* +** Set the "use-threads" flag on object pIncr. +*/ +static void vdbeIncrMergerSetThreads(IncrMerger *pIncr){ + pIncr->bUseThread = 1; + pIncr->pTask->file2.iEof -= pIncr->mxSz; +} +#endif /* SQLITE_MAX_WORKER_THREADS>0 */ + + + +/* +** Recompute pMerger->aTree[iOut] by comparing the next keys on the +** two PmaReaders that feed that entry. Neither of the PmaReaders +** are advanced. This routine merely does the comparison. +*/ +static void vdbeMergeEngineCompare( + MergeEngine *pMerger, /* Merge engine containing PmaReaders to compare */ + int iOut /* Store the result in pMerger->aTree[iOut] */ +){ + int i1; + int i2; + int iRes; + PmaReader *p1; + PmaReader *p2; + + assert( iOutnTree && iOut>0 ); + + if( iOut>=(pMerger->nTree/2) ){ + i1 = (iOut - pMerger->nTree/2) * 2; + i2 = i1 + 1; + }else{ + i1 = pMerger->aTree[iOut*2]; + i2 = pMerger->aTree[iOut*2+1]; + } + + p1 = &pMerger->aReadr[i1]; + p2 = &pMerger->aReadr[i2]; + + if( p1->pFd==0 ){ + iRes = i2; + }else if( p2->pFd==0 ){ + iRes = i1; + }else{ + int res; + assert( pMerger->pTask->pUnpacked!=0 ); /* from vdbeSortSubtaskMain() */ + res = vdbeSorterCompare( + pMerger->pTask, p1->aKey, p1->nKey, p2->aKey, p2->nKey + ); + if( res<=0 ){ + iRes = i1; + }else{ + iRes = i2; + } + } + + pMerger->aTree[iOut] = iRes; +} + +/* +** Allowed values for the eMode parameter to vdbeMergeEngineInit() +** and vdbePmaReaderIncrMergeInit(). +** +** Only INCRINIT_NORMAL is valid in single-threaded builds (when +** SQLITE_MAX_WORKER_THREADS==0). The other values are only used +** when there exists one or more separate worker threads. +*/ +#define INCRINIT_NORMAL 0 +#define INCRINIT_TASK 1 +#define INCRINIT_ROOT 2 + +/* Forward reference. +** The vdbeIncrMergeInit() and vdbePmaReaderIncrMergeInit() routines call each +** other (when building a merge tree). +*/ +static int vdbePmaReaderIncrMergeInit(PmaReader *pReadr, int eMode); + +/* +** Initialize the MergeEngine object passed as the second argument. Once this +** function returns, the first key of merged data may be read from the +** MergeEngine object in the usual fashion. +** +** If argument eMode is INCRINIT_ROOT, then it is assumed that any IncrMerge +** objects attached to the PmaReader objects that the merger reads from have +** already been populated, but that they have not yet populated aFile[0] and +** set the PmaReader objects up to read from it. In this case all that is +** required is to call vdbePmaReaderNext() on each PmaReader to point it at +** its first key. +** +** Otherwise, if eMode is any value other than INCRINIT_ROOT, then use +** vdbePmaReaderIncrMergeInit() to initialize each PmaReader that feeds data +** to pMerger. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +*/ +static int vdbeMergeEngineInit( + SortSubtask *pTask, /* Thread that will run pMerger */ + MergeEngine *pMerger, /* MergeEngine to initialize */ + int eMode /* One of the INCRINIT_XXX constants */ +){ + int rc = SQLITE_OK; /* Return code */ + int i; /* For looping over PmaReader objects */ + int nTree = pMerger->nTree; + + /* eMode is always INCRINIT_NORMAL in single-threaded mode */ + assert( SQLITE_MAX_WORKER_THREADS>0 || eMode==INCRINIT_NORMAL ); + + /* Verify that the MergeEngine is assigned to a single thread */ + assert( pMerger->pTask==0 ); + pMerger->pTask = pTask; + + for(i=0; i0 && eMode==INCRINIT_ROOT ){ + /* PmaReaders should be normally initialized in order, as if they are + ** reading from the same temp file this makes for more linear file IO. + ** However, in the INCRINIT_ROOT case, if PmaReader aReadr[nTask-1] is + ** in use it will block the vdbePmaReaderNext() call while it uses + ** the main thread to fill its buffer. So calling PmaReaderNext() + ** on this PmaReader before any of the multi-threaded PmaReaders takes + ** better advantage of multi-processor hardware. */ + rc = vdbePmaReaderNext(&pMerger->aReadr[nTree-i-1]); + }else{ + rc = vdbePmaReaderIncrMergeInit(&pMerger->aReadr[i], INCRINIT_NORMAL); + } + if( rc!=SQLITE_OK ) return rc; + } + + for(i=pMerger->nTree-1; i>0; i--){ + vdbeMergeEngineCompare(pMerger, i); + } + return pTask->pUnpacked->errCode; +} + +/* +** Initialize the IncrMerge field of a PmaReader. +** +** If the PmaReader passed as the first argument is not an incremental-reader +** (if pReadr->pIncr==0), then this function is a no-op. Otherwise, it serves +** to open and/or initialize the temp file related fields of the IncrMerge +** object at (pReadr->pIncr). +** +** If argument eMode is set to INCRINIT_NORMAL, then all PmaReaders +** in the sub-tree headed by pReadr are also initialized. Data is then loaded +** into the buffers belonging to pReadr and it is set to +** point to the first key in its range. +** +** If argument eMode is set to INCRINIT_TASK, then pReadr is guaranteed +** to be a multi-threaded PmaReader and this function is being called in a +** background thread. In this case all PmaReaders in the sub-tree are +** initialized as for INCRINIT_NORMAL and the aFile[1] buffer belonging to +** pReadr is populated. However, pReadr itself is not set up to point +** to its first key. A call to vdbePmaReaderNext() is still required to do +** that. +** +** The reason this function does not call vdbePmaReaderNext() immediately +** in the INCRINIT_TASK case is that vdbePmaReaderNext() assumes that it has +** to block on thread (pTask->thread) before accessing aFile[1]. But, since +** this entire function is being run by thread (pTask->thread), that will +** lead to the current background thread attempting to join itself. +** +** Finally, if argument eMode is set to INCRINIT_ROOT, it may be assumed +** that pReadr->pIncr is a multi-threaded IncrMerge objects, and that all +** child-trees have already been initialized using IncrInit(INCRINIT_TASK). +** In this case vdbePmaReaderNext() is called on all child PmaReaders and +** the current PmaReader set to point to the first key in its range. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +*/ +static int vdbePmaReaderIncrMergeInit(PmaReader *pReadr, int eMode){ + int rc = SQLITE_OK; + IncrMerger *pIncr = pReadr->pIncr; + + /* eMode is always INCRINIT_NORMAL in single-threaded mode */ + assert( SQLITE_MAX_WORKER_THREADS>0 || eMode==INCRINIT_NORMAL ); + + if( pIncr ){ + SortSubtask *pTask = pIncr->pTask; + sqlite3 *db = pTask->pSorter->db; + + rc = vdbeMergeEngineInit(pTask, pIncr->pMerger, eMode); + + /* Set up the required files for pIncr. A multi-theaded IncrMerge object + ** requires two temp files to itself, whereas a single-threaded object + ** only requires a region of pTask->file2. */ + if( rc==SQLITE_OK ){ + int mxSz = pIncr->mxSz; +#if SQLITE_MAX_WORKER_THREADS>0 + if( pIncr->bUseThread ){ + rc = vdbeSorterOpenTempFile(db, mxSz, &pIncr->aFile[0].pFd); + if( rc==SQLITE_OK ){ + rc = vdbeSorterOpenTempFile(db, mxSz, &pIncr->aFile[1].pFd); + } + }else +#endif + /*if( !pIncr->bUseThread )*/{ + if( pTask->file2.pFd==0 ){ + assert( pTask->file2.iEof>0 ); + rc = vdbeSorterOpenTempFile(db, pTask->file2.iEof, &pTask->file2.pFd); + pTask->file2.iEof = 0; + } + if( rc==SQLITE_OK ){ + pIncr->aFile[1].pFd = pTask->file2.pFd; + pIncr->iStartOff = pTask->file2.iEof; + pTask->file2.iEof += mxSz; + } + } + } + +#if SQLITE_MAX_WORKER_THREADS>0 + if( rc==SQLITE_OK && pIncr->bUseThread ){ + /* Use the current thread to populate aFile[1], even though this + ** PmaReader is multi-threaded. The reason being that this function + ** is already running in background thread pIncr->pTask->thread. */ + assert( eMode==INCRINIT_ROOT || eMode==INCRINIT_TASK ); + rc = vdbeIncrPopulate(pIncr); + } +#endif + + if( rc==SQLITE_OK + && (SQLITE_MAX_WORKER_THREADS==0 || eMode!=INCRINIT_TASK) + ){ + rc = vdbePmaReaderNext(pReadr); + } + } + return rc; +} + +#if SQLITE_MAX_WORKER_THREADS>0 +/* +** The main routine for vdbePmaReaderIncrMergeInit() operations run in +** background threads. +*/ +static void *vdbePmaReaderBgInit(void *pCtx){ + PmaReader *pReader = (PmaReader*)pCtx; + void *pRet = SQLITE_INT_TO_PTR( + vdbePmaReaderIncrMergeInit(pReader,INCRINIT_TASK) + ); + pReader->pIncr->pTask->bDone = 1; + return pRet; +} + +/* +** Use a background thread to invoke vdbePmaReaderIncrMergeInit(INCRINIT_TASK) +** on the PmaReader object passed as the first argument. +** +** This call will initialize the various fields of the pReadr->pIncr +** structure and, if it is a multi-threaded IncrMerger, launch a +** background thread to populate aFile[1]. +*/ +static int vdbePmaReaderBgIncrInit(PmaReader *pReadr){ + void *pCtx = (void*)pReadr; + return vdbeSorterCreateThread(pReadr->pIncr->pTask, vdbePmaReaderBgInit, pCtx); +} +#endif + +/* +** Allocate a new MergeEngine object to merge the contents of nPMA level-0 +** PMAs from pTask->file. If no error occurs, set *ppOut to point to +** the new object and return SQLITE_OK. Or, if an error does occur, set *ppOut +** to NULL and return an SQLite error code. +** +** When this function is called, *piOffset is set to the offset of the +** first PMA to read from pTask->file. Assuming no error occurs, it is +** set to the offset immediately following the last byte of the last +** PMA before returning. If an error does occur, then the final value of +** *piOffset is undefined. +*/ +static int vdbeMergeEngineLevel0( + SortSubtask *pTask, /* Sorter task to read from */ + int nPMA, /* Number of PMAs to read */ + i64 *piOffset, /* IN/OUT: Readr offset in pTask->file */ + MergeEngine **ppOut /* OUT: New merge-engine */ +){ + MergeEngine *pNew; /* Merge engine to return */ + i64 iOff = *piOffset; + int i; + int rc = SQLITE_OK; + + *ppOut = pNew = vdbeMergeEngineNew(nPMA); + if( pNew==0 ) rc = SQLITE_NOMEM; + + for(i=0; iaReadr[i]; + rc = vdbePmaReaderInit(pTask, &pTask->file, iOff, pReadr, &nDummy); + iOff = pReadr->iEof; + } + + if( rc!=SQLITE_OK ){ + vdbeMergeEngineFree(pNew); + *ppOut = 0; + } + *piOffset = iOff; + return rc; +} + +/* +** Return the depth of a tree comprising nPMA PMAs, assuming a fanout of +** SORTER_MAX_MERGE_COUNT. The returned value does not include leaf nodes. +** +** i.e. +** +** nPMA<=16 -> TreeDepth() == 0 +** nPMA<=256 -> TreeDepth() == 1 +** nPMA<=65536 -> TreeDepth() == 2 +*/ +static int vdbeSorterTreeDepth(int nPMA){ + int nDepth = 0; + i64 nDiv = SORTER_MAX_MERGE_COUNT; + while( nDiv < (i64)nPMA ){ + nDiv = nDiv * SORTER_MAX_MERGE_COUNT; + nDepth++; + } + return nDepth; +} + +/* +** pRoot is the root of an incremental merge-tree with depth nDepth (according +** to vdbeSorterTreeDepth()). pLeaf is the iSeq'th leaf to be added to the +** tree, counting from zero. This function adds pLeaf to the tree. +** +** If successful, SQLITE_OK is returned. If an error occurs, an SQLite error +** code is returned and pLeaf is freed. +*/ +static int vdbeSorterAddToTree( + SortSubtask *pTask, /* Task context */ + int nDepth, /* Depth of tree according to TreeDepth() */ + int iSeq, /* Sequence number of leaf within tree */ + MergeEngine *pRoot, /* Root of tree */ + MergeEngine *pLeaf /* Leaf to add to tree */ +){ + int rc = SQLITE_OK; + int nDiv = 1; + int i; + MergeEngine *p = pRoot; + IncrMerger *pIncr; + + rc = vdbeIncrMergerNew(pTask, pLeaf, &pIncr); + + for(i=1; iaReadr[iIter]; + + if( pReadr->pIncr==0 ){ + MergeEngine *pNew = vdbeMergeEngineNew(SORTER_MAX_MERGE_COUNT); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = vdbeIncrMergerNew(pTask, pNew, &pReadr->pIncr); + } + } + if( rc==SQLITE_OK ){ + p = pReadr->pIncr->pMerger; + nDiv = nDiv / SORTER_MAX_MERGE_COUNT; + } + } + + if( rc==SQLITE_OK ){ + p->aReadr[iSeq % SORTER_MAX_MERGE_COUNT].pIncr = pIncr; + }else{ + vdbeIncrFree(pIncr); + } + return rc; +} + +/* +** This function is called as part of a SorterRewind() operation on a sorter +** that has already written two or more level-0 PMAs to one or more temp +** files. It builds a tree of MergeEngine/IncrMerger/PmaReader objects that +** can be used to incrementally merge all PMAs on disk. +** +** If successful, SQLITE_OK is returned and *ppOut set to point to the +** MergeEngine object at the root of the tree before returning. Or, if an +** error occurs, an SQLite error code is returned and the final value +** of *ppOut is undefined. +*/ +static int vdbeSorterMergeTreeBuild( + VdbeSorter *pSorter, /* The VDBE cursor that implements the sort */ + MergeEngine **ppOut /* Write the MergeEngine here */ +){ + MergeEngine *pMain = 0; + int rc = SQLITE_OK; + int iTask; + +#if SQLITE_MAX_WORKER_THREADS>0 + /* If the sorter uses more than one task, then create the top-level + ** MergeEngine here. This MergeEngine will read data from exactly + ** one PmaReader per sub-task. */ + assert( pSorter->bUseThreads || pSorter->nTask==1 ); + if( pSorter->nTask>1 ){ + pMain = vdbeMergeEngineNew(pSorter->nTask); + if( pMain==0 ) rc = SQLITE_NOMEM; + } +#endif + + for(iTask=0; rc==SQLITE_OK && iTasknTask; iTask++){ + SortSubtask *pTask = &pSorter->aTask[iTask]; + assert( pTask->nPMA>0 || SQLITE_MAX_WORKER_THREADS>0 ); + if( SQLITE_MAX_WORKER_THREADS==0 || pTask->nPMA ){ + MergeEngine *pRoot = 0; /* Root node of tree for this task */ + int nDepth = vdbeSorterTreeDepth(pTask->nPMA); + i64 iReadOff = 0; + + if( pTask->nPMA<=SORTER_MAX_MERGE_COUNT ){ + rc = vdbeMergeEngineLevel0(pTask, pTask->nPMA, &iReadOff, &pRoot); + }else{ + int i; + int iSeq = 0; + pRoot = vdbeMergeEngineNew(SORTER_MAX_MERGE_COUNT); + if( pRoot==0 ) rc = SQLITE_NOMEM; + for(i=0; inPMA && rc==SQLITE_OK; i += SORTER_MAX_MERGE_COUNT){ + MergeEngine *pMerger = 0; /* New level-0 PMA merger */ + int nReader; /* Number of level-0 PMAs to merge */ + + nReader = MIN(pTask->nPMA - i, SORTER_MAX_MERGE_COUNT); + rc = vdbeMergeEngineLevel0(pTask, nReader, &iReadOff, &pMerger); + if( rc==SQLITE_OK ){ + rc = vdbeSorterAddToTree(pTask, nDepth, iSeq++, pRoot, pMerger); + } + } + } + + if( rc==SQLITE_OK ){ +#if SQLITE_MAX_WORKER_THREADS>0 + if( pMain!=0 ){ + rc = vdbeIncrMergerNew(pTask, pRoot, &pMain->aReadr[iTask].pIncr); + }else +#endif + { + assert( pMain==0 ); + pMain = pRoot; + } + }else{ + vdbeMergeEngineFree(pRoot); + } + } + } + + if( rc!=SQLITE_OK ){ + vdbeMergeEngineFree(pMain); + pMain = 0; + } + *ppOut = pMain; + return rc; +} + +/* +** This function is called as part of an sqlite3VdbeSorterRewind() operation +** on a sorter that has written two or more PMAs to temporary files. It sets +** up either VdbeSorter.pMerger (for single threaded sorters) or pReader +** (for multi-threaded sorters) so that it can be used to iterate through +** all records stored in the sorter. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +*/ +static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ + int rc; /* Return code */ + SortSubtask *pTask0 = &pSorter->aTask[0]; + MergeEngine *pMain = 0; +#if SQLITE_MAX_WORKER_THREADS + sqlite3 *db = pTask0->pSorter->db; +#endif + + rc = vdbeSorterMergeTreeBuild(pSorter, &pMain); + if( rc==SQLITE_OK ){ +#if SQLITE_MAX_WORKER_THREADS + assert( pSorter->bUseThreads==0 || pSorter->nTask>1 ); + if( pSorter->bUseThreads ){ + int iTask; + PmaReader *pReadr = 0; + SortSubtask *pLast = &pSorter->aTask[pSorter->nTask-1]; + rc = vdbeSortAllocUnpacked(pLast); + if( rc==SQLITE_OK ){ + pReadr = (PmaReader*)sqlite3DbMallocZero(db, sizeof(PmaReader)); + pSorter->pReader = pReadr; + if( pReadr==0 ) rc = SQLITE_NOMEM; + } + if( rc==SQLITE_OK ){ + rc = vdbeIncrMergerNew(pLast, pMain, &pReadr->pIncr); + if( rc==SQLITE_OK ){ + vdbeIncrMergerSetThreads(pReadr->pIncr); + for(iTask=0; iTask<(pSorter->nTask-1); iTask++){ + IncrMerger *pIncr; + if( (pIncr = pMain->aReadr[iTask].pIncr) ){ + vdbeIncrMergerSetThreads(pIncr); + assert( pIncr->pTask!=pLast ); + } + } + for(iTask=0; rc==SQLITE_OK && iTasknTask; iTask++){ + PmaReader *p = &pMain->aReadr[iTask]; + assert( p->pIncr==0 || p->pIncr->pTask==&pSorter->aTask[iTask] ); + if( p->pIncr ){ + if( iTask==pSorter->nTask-1 ){ + rc = vdbePmaReaderIncrMergeInit(p, INCRINIT_TASK); + }else{ + rc = vdbePmaReaderBgIncrInit(p); + } + } + } + } + pMain = 0; + } + if( rc==SQLITE_OK ){ + rc = vdbePmaReaderIncrMergeInit(pReadr, INCRINIT_ROOT); + } + }else +#endif + { + rc = vdbeMergeEngineInit(pTask0, pMain, INCRINIT_NORMAL); + pSorter->pMerger = pMain; + pMain = 0; + } + } + + if( rc!=SQLITE_OK ){ + vdbeMergeEngineFree(pMain); + } + return rc; +} + + +/* +** Once the sorter has been populated by calls to sqlite3VdbeSorterWrite, +** this function is called to prepare for iterating through the records +** in sorted order. +*/ +int sqlite3VdbeSorterRewind(const VdbeCursor *pCsr, int *pbEof){ VdbeSorter *pSorter = pCsr->pSorter; int rc = SQLITE_OK; /* Return code */ - int i; /* Used to iterator through aIter[] */ - i64 nByte = 0; /* Total bytes in all opened PMAs */ - - /* Initialize the iterators. */ - for(i=0; iaIter[i]; - rc = vdbeSorterIterInit(db, pSorter, pSorter->iReadOff, pIter, &nByte); - pSorter->iReadOff = pIter->iEof; - assert( rc!=SQLITE_OK || pSorter->iReadOff<=pSorter->iWriteOff ); - if( rc!=SQLITE_OK || pSorter->iReadOff>=pSorter->iWriteOff ) break; - } - - /* Initialize the aTree[] array. */ - for(i=pSorter->nTree-1; rc==SQLITE_OK && i>0; i--){ - rc = vdbeSorterDoCompare(pCsr, i); - } - - *pnByte = nByte; - return rc; -} - -/* -** Once the sorter has been populated, this function is called to prepare -** for iterating through its contents in sorted order. -*/ -int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ - VdbeSorter *pSorter = pCsr->pSorter; - int rc; /* Return code */ - sqlite3_file *pTemp2 = 0; /* Second temp file to use */ - i64 iWrite2 = 0; /* Write offset for pTemp2 */ - int nIter; /* Number of iterators used */ - int nByte; /* Bytes of space required for aIter/aTree */ - int N = 2; /* Power of 2 >= nIter */ assert( pSorter ); /* If no data has been written to disk, then do not do so now. Instead, ** sort the VdbeSorter.pRecord list. The vdbe layer will read data directly ** from the in-memory list. */ - if( pSorter->nPMA==0 ){ - *pbEof = !pSorter->pRecord; - assert( pSorter->aTree==0 ); - return vdbeSorterSort(pCsr); - } - - /* Write the current in-memory list to a PMA. */ - rc = vdbeSorterListToPMA(db, pCsr); - if( rc!=SQLITE_OK ) return rc; - - /* Allocate space for aIter[] and aTree[]. */ - nIter = pSorter->nPMA; - if( nIter>SORTER_MAX_MERGE_COUNT ) nIter = SORTER_MAX_MERGE_COUNT; - assert( nIter>0 ); - while( NaIter = (VdbeSorterIter *)sqlite3DbMallocZero(db, nByte); - if( !pSorter->aIter ) return SQLITE_NOMEM; - pSorter->aTree = (int *)&pSorter->aIter[N]; - pSorter->nTree = N; - - do { - int iNew; /* Index of new, merged, PMA */ - - for(iNew=0; - rc==SQLITE_OK && iNew*SORTER_MAX_MERGE_COUNTnPMA; - iNew++ - ){ - int rc2; /* Return code from fileWriterFinish() */ - FileWriter writer; /* Object used to write to disk */ - i64 nWrite; /* Number of bytes in new PMA */ - - memset(&writer, 0, sizeof(FileWriter)); - - /* If there are SORTER_MAX_MERGE_COUNT or less PMAs in file pTemp1, - ** initialize an iterator for each of them and break out of the loop. - ** These iterators will be incrementally merged as the VDBE layer calls - ** sqlite3VdbeSorterNext(). - ** - ** Otherwise, if pTemp1 contains more than SORTER_MAX_MERGE_COUNT PMAs, - ** initialize interators for SORTER_MAX_MERGE_COUNT of them. These PMAs - ** are merged into a single PMA that is written to file pTemp2. - */ - rc = vdbeSorterInitMerge(db, pCsr, &nWrite); - assert( rc!=SQLITE_OK || pSorter->aIter[ pSorter->aTree[1] ].pFile ); - if( rc!=SQLITE_OK || pSorter->nPMA<=SORTER_MAX_MERGE_COUNT ){ - break; - } - - /* Open the second temp file, if it is not already open. */ - if( pTemp2==0 ){ - assert( iWrite2==0 ); - rc = vdbeSorterOpenTempFile(db, &pTemp2); - } - - if( rc==SQLITE_OK ){ - int bEof = 0; - fileWriterInit(db, pTemp2, &writer, iWrite2); - fileWriterWriteVarint(&writer, nWrite); - while( rc==SQLITE_OK && bEof==0 ){ - VdbeSorterIter *pIter = &pSorter->aIter[ pSorter->aTree[1] ]; - assert( pIter->pFile ); - - fileWriterWriteVarint(&writer, pIter->nKey); - fileWriterWrite(&writer, pIter->aKey, pIter->nKey); - rc = sqlite3VdbeSorterNext(db, pCsr, &bEof); - } - rc2 = fileWriterFinish(db, &writer, &iWrite2); - if( rc==SQLITE_OK ) rc = rc2; - } - } - - if( pSorter->nPMA<=SORTER_MAX_MERGE_COUNT ){ - break; + if( pSorter->bUsePMA==0 ){ + if( pSorter->list.pList ){ + *pbEof = 0; + rc = vdbeSorterSort(&pSorter->aTask[0], &pSorter->list); }else{ - sqlite3_file *pTmp = pSorter->pTemp1; - pSorter->nPMA = iNew; - pSorter->pTemp1 = pTemp2; - pTemp2 = pTmp; - pSorter->iWriteOff = iWrite2; - pSorter->iReadOff = 0; - iWrite2 = 0; + *pbEof = 1; } - }while( rc==SQLITE_OK ); - - if( pTemp2 ){ - sqlite3OsCloseFree(pTemp2); + return rc; } - *pbEof = (pSorter->aIter[pSorter->aTree[1]].pFile==0); + + /* Write the current in-memory list to a PMA. When the VdbeSorterWrite() + ** function flushes the contents of memory to disk, it immediately always + ** creates a new list consisting of a single key immediately afterwards. + ** So the list is never empty at this point. */ + assert( pSorter->list.pList ); + rc = vdbeSorterFlushPMA(pSorter); + + /* Join all threads */ + rc = vdbeSorterJoinAll(pSorter, rc); + + vdbeSorterRewindDebug("rewind"); + + /* Assuming no errors have occurred, set up a merger structure to + ** incrementally read and merge all remaining PMAs. */ + assert( pSorter->pReader==0 ); + if( rc==SQLITE_OK ){ + rc = vdbeSorterSetupMerge(pSorter); + *pbEof = 0; + } + + vdbeSorterRewindDebug("rewinddone"); return rc; } @@ -970,63 +2406,27 @@ int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ VdbeSorter *pSorter = pCsr->pSorter; int rc; /* Return code */ - if( pSorter->aTree ){ - int iPrev = pSorter->aTree[1];/* Index of iterator to advance */ - rc = vdbeSorterIterNext(db, &pSorter->aIter[iPrev]); - if( rc==SQLITE_OK ){ - int i; /* Index of aTree[] to recalculate */ - VdbeSorterIter *pIter1; /* First iterator to compare */ - VdbeSorterIter *pIter2; /* Second iterator to compare */ - u8 *pKey2; /* To pIter2->aKey, or 0 if record cached */ - - /* Find the first two iterators to compare. The one that was just - ** advanced (iPrev) and the one next to it in the array. */ - pIter1 = &pSorter->aIter[(iPrev & 0xFFFE)]; - pIter2 = &pSorter->aIter[(iPrev | 0x0001)]; - pKey2 = pIter2->aKey; - - for(i=(pSorter->nTree+iPrev)/2; i>0; i=i/2){ - /* Compare pIter1 and pIter2. Store the result in variable iRes. */ - int iRes; - if( pIter1->pFile==0 ){ - iRes = +1; - }else if( pIter2->pFile==0 ){ - iRes = -1; - }else{ - vdbeSorterCompare(pCsr, 0, - pIter1->aKey, pIter1->nKey, pKey2, pIter2->nKey, &iRes - ); - } - - /* If pIter1 contained the smaller value, set aTree[i] to its index. - ** Then set pIter2 to the next iterator to compare to pIter1. In this - ** case there is no cache of pIter2 in pSorter->pUnpacked, so set - ** pKey2 to point to the record belonging to pIter2. - ** - ** Alternatively, if pIter2 contains the smaller of the two values, - ** set aTree[i] to its index and update pIter1. If vdbeSorterCompare() - ** was actually called above, then pSorter->pUnpacked now contains - ** a value equivalent to pIter2. So set pKey2 to NULL to prevent - ** vdbeSorterCompare() from decoding pIter2 again. */ - if( iRes<=0 ){ - pSorter->aTree[i] = (int)(pIter1 - pSorter->aIter); - pIter2 = &pSorter->aIter[ pSorter->aTree[i ^ 0x0001] ]; - pKey2 = pIter2->aKey; - }else{ - if( pIter1->pFile ) pKey2 = 0; - pSorter->aTree[i] = (int)(pIter2 - pSorter->aIter); - pIter1 = &pSorter->aIter[ pSorter->aTree[i ^ 0x0001] ]; - } - - } - *pbEof = (pSorter->aIter[pSorter->aTree[1]].pFile==0); + assert( pSorter->bUsePMA || (pSorter->pReader==0 && pSorter->pMerger==0) ); + if( pSorter->bUsePMA ){ + assert( pSorter->pReader==0 || pSorter->pMerger==0 ); + assert( pSorter->bUseThreads==0 || pSorter->pReader ); + assert( pSorter->bUseThreads==1 || pSorter->pMerger ); +#if SQLITE_MAX_WORKER_THREADS>0 + if( pSorter->bUseThreads ){ + rc = vdbePmaReaderNext(pSorter->pReader); + *pbEof = (pSorter->pReader->pFd==0); + }else +#endif + /*if( !pSorter->bUseThreads )*/ { + assert( pSorter->pMerger->pTask==(&pSorter->aTask[0]) ); + rc = vdbeMergeEngineStep(pSorter->pMerger, pbEof); } }else{ - SorterRecord *pFree = pSorter->pRecord; - pSorter->pRecord = pFree->pNext; - pFree->pNext = 0; - vdbeSorterRecordFree(db, pFree); - *pbEof = !pSorter->pRecord; + SorterRecord *pFree = pSorter->list.pList; + pSorter->list.pList = pFree->u.pNext; + pFree->u.pNext = 0; + if( pSorter->list.aMemory==0 ) vdbeSorterRecordFree(db, pFree); + *pbEof = !pSorter->list.pList; rc = SQLITE_OK; } return rc; @@ -1041,14 +2441,21 @@ static void *vdbeSorterRowkey( int *pnKey /* OUT: Size of current key in bytes */ ){ void *pKey; - if( pSorter->aTree ){ - VdbeSorterIter *pIter; - pIter = &pSorter->aIter[ pSorter->aTree[1] ]; - *pnKey = pIter->nKey; - pKey = pIter->aKey; + if( pSorter->bUsePMA ){ + PmaReader *pReader; +#if SQLITE_MAX_WORKER_THREADS>0 + if( pSorter->bUseThreads ){ + pReader = pSorter->pReader; + }else +#endif + /*if( !pSorter->bUseThreads )*/{ + pReader = &pSorter->pMerger->aReadr[pSorter->pMerger->aTree[1]]; + } + *pnKey = pReader->nKey; + pKey = pReader->aKey; }else{ - *pnKey = pSorter->pRecord->nVal; - pKey = pSorter->pRecord->pVal; + *pnKey = pSorter->list.pList->nVal; + pKey = SRVAL(pSorter->list.pList); } return pKey; } @@ -1061,7 +2468,7 @@ int sqlite3VdbeSorterRowkey(const VdbeCursor *pCsr, Mem *pOut){ void *pKey; int nKey; /* Sorter key to copy into pOut */ pKey = vdbeSorterRowkey(pSorter, &nKey); - if( sqlite3VdbeMemGrow(pOut, nKey, 0) ){ + if( sqlite3VdbeMemClearAndResize(pOut, nKey) ){ return SQLITE_NOMEM; } pOut->n = nKey; @@ -1076,21 +2483,47 @@ int sqlite3VdbeSorterRowkey(const VdbeCursor *pCsr, Mem *pOut){ ** passed as the first argument currently points to. For the purposes of ** the comparison, ignore the rowid field at the end of each record. ** +** If the sorter cursor key contains any NULL values, consider it to be +** less than pVal. Even if pVal also contains NULL values. +** ** If an error occurs, return an SQLite error code (i.e. SQLITE_NOMEM). ** Otherwise, set *pRes to a negative, zero or positive value if the ** key in pVal is smaller than, equal to or larger than the current sorter ** key. +** +** This routine forms the core of the OP_SorterCompare opcode, which in +** turn is used to verify uniqueness when constructing a UNIQUE INDEX. */ int sqlite3VdbeSorterCompare( const VdbeCursor *pCsr, /* Sorter cursor */ Mem *pVal, /* Value to compare to current sorter key */ - int nIgnore, /* Ignore this many fields at the end */ + int nKeyCol, /* Compare this many columns */ int *pRes /* OUT: Result of comparison */ ){ VdbeSorter *pSorter = pCsr->pSorter; + UnpackedRecord *r2 = pSorter->pUnpacked; + KeyInfo *pKeyInfo = pCsr->pKeyInfo; + int i; void *pKey; int nKey; /* Sorter key to compare pVal with */ + if( r2==0 ){ + char *p; + r2 = pSorter->pUnpacked = sqlite3VdbeAllocUnpackedRecord(pKeyInfo,0,0,&p); + assert( pSorter->pUnpacked==(UnpackedRecord*)p ); + if( r2==0 ) return SQLITE_NOMEM; + r2->nField = nKeyCol; + } + assert( r2->nField==nKeyCol ); + pKey = vdbeSorterRowkey(pSorter, &nKey); - vdbeSorterCompare(pCsr, nIgnore, pVal->z, pVal->n, pKey, nKey, pRes); + sqlite3VdbeRecordUnpack(pKeyInfo, nKey, pKey, r2); + for(i=0; iaMem[i].flags & MEM_Null ){ + *pRes = -1; + return SQLITE_OK; + } + } + + *pRes = sqlite3VdbeRecordCompare(pVal->n, pVal->z, r2); return SQLITE_OK; } diff --git a/src/vdbetrace.c b/src/vdbetrace.c index 4a39e26521..507c2f12fc 100644 --- a/src/vdbetrace.c +++ b/src/vdbetrace.c @@ -64,7 +64,7 @@ static int findNextHostParameter(const char *zSql, int *pnToken){ ** ALGORITHM: Scan the input string looking for host parameters in any of ** these forms: ?, ?N, $A, @A, :A. Take care to avoid text within ** string literals, quoted identifier names, and comments. For text forms, -** the host parameter index is found by scanning the perpared +** the host parameter index is found by scanning the prepared ** statement for the corresponding OP_Variable opcode. Once the host ** parameter index is known, locate the value in p->aVar[]. Then render ** the value as a literal in place of the host parameter name. @@ -127,7 +127,7 @@ char *sqlite3VdbeExpandSql( }else if( pVar->flags & MEM_Int ){ sqlite3XPrintf(&out, 0, "%lld", pVar->u.i); }else if( pVar->flags & MEM_Real ){ - sqlite3XPrintf(&out, 0, "%!.15g", pVar->r); + sqlite3XPrintf(&out, 0, "%!.15g", pVar->u.r); }else if( pVar->flags & MEM_Str ){ int nOut; /* Number of bytes of the string text to include in output */ #ifndef SQLITE_OMIT_UTF16 @@ -183,118 +183,3 @@ char *sqlite3VdbeExpandSql( } #endif /* #ifndef SQLITE_OMIT_TRACE */ - -/***************************************************************************** -** The following code implements the data-structure explaining logic -** for the Vdbe. -*/ - -#if defined(SQLITE_ENABLE_TREE_EXPLAIN) - -/* -** Allocate a new Explain object -*/ -void sqlite3ExplainBegin(Vdbe *pVdbe){ - if( pVdbe ){ - Explain *p; - sqlite3BeginBenignMalloc(); - p = (Explain *)sqlite3MallocZero( sizeof(Explain) ); - if( p ){ - p->pVdbe = pVdbe; - sqlite3_free(pVdbe->pExplain); - pVdbe->pExplain = p; - sqlite3StrAccumInit(&p->str, p->zBase, sizeof(p->zBase), - SQLITE_MAX_LENGTH); - p->str.useMalloc = 2; - }else{ - sqlite3EndBenignMalloc(); - } - } -} - -/* -** Return true if the Explain ends with a new-line. -*/ -static int endsWithNL(Explain *p){ - return p && p->str.zText && p->str.nChar - && p->str.zText[p->str.nChar-1]=='\n'; -} - -/* -** Append text to the indentation -*/ -void sqlite3ExplainPrintf(Vdbe *pVdbe, const char *zFormat, ...){ - Explain *p; - if( pVdbe && (p = pVdbe->pExplain)!=0 ){ - va_list ap; - if( p->nIndent && endsWithNL(p) ){ - int n = p->nIndent; - if( n>ArraySize(p->aIndent) ) n = ArraySize(p->aIndent); - sqlite3AppendSpace(&p->str, p->aIndent[n-1]); - } - va_start(ap, zFormat); - sqlite3VXPrintf(&p->str, SQLITE_PRINTF_INTERNAL, zFormat, ap); - va_end(ap); - } -} - -/* -** Append a '\n' if there is not already one. -*/ -void sqlite3ExplainNL(Vdbe *pVdbe){ - Explain *p; - if( pVdbe && (p = pVdbe->pExplain)!=0 && !endsWithNL(p) ){ - sqlite3StrAccumAppend(&p->str, "\n", 1); - } -} - -/* -** Push a new indentation level. Subsequent lines will be indented -** so that they begin at the current cursor position. -*/ -void sqlite3ExplainPush(Vdbe *pVdbe){ - Explain *p; - if( pVdbe && (p = pVdbe->pExplain)!=0 ){ - if( p->str.zText && p->nIndentaIndent) ){ - const char *z = p->str.zText; - int i = p->str.nChar-1; - int x; - while( i>=0 && z[i]!='\n' ){ i--; } - x = (p->str.nChar - 1) - i; - if( p->nIndent && xaIndent[p->nIndent-1] ){ - x = p->aIndent[p->nIndent-1]; - } - p->aIndent[p->nIndent] = x; - } - p->nIndent++; - } -} - -/* -** Pop the indentation stack by one level. -*/ -void sqlite3ExplainPop(Vdbe *p){ - if( p && p->pExplain ) p->pExplain->nIndent--; -} - -/* -** Free the indentation structure -*/ -void sqlite3ExplainFinish(Vdbe *pVdbe){ - if( pVdbe && pVdbe->pExplain ){ - sqlite3_free(pVdbe->zExplain); - sqlite3ExplainNL(pVdbe); - pVdbe->zExplain = sqlite3StrAccumFinish(&pVdbe->pExplain->str); - sqlite3_free(pVdbe->pExplain); - pVdbe->pExplain = 0; - sqlite3EndBenignMalloc(); - } -} - -/* -** Return the explanation of a virtual machine. -*/ -const char *sqlite3VdbeExplanation(Vdbe *pVdbe){ - return (pVdbe && pVdbe->zExplain) ? pVdbe->zExplain : 0; -} -#endif /* defined(SQLITE_DEBUG) */ diff --git a/src/vtab.c b/src/vtab.c index 7b4867f9c0..6834b18826 100644 --- a/src/vtab.c +++ b/src/vtab.c @@ -43,7 +43,7 @@ static int createModule( sqlite3_mutex_enter(db->mutex); nName = sqlite3Strlen30(zName); - if( sqlite3HashFind(&db->aModule, zName, nName) ){ + if( sqlite3HashFind(&db->aModule, zName) ){ rc = SQLITE_MISUSE_BKPT; }else{ Module *pMod; @@ -56,7 +56,7 @@ static int createModule( pMod->pModule = pModule; pMod->pAux = pAux; pMod->xDestroy = xDestroy; - pDel = (Module *)sqlite3HashInsert(&db->aModule,zCopy,nName,(void*)pMod); + pDel = (Module *)sqlite3HashInsert(&db->aModule,zCopy,(void*)pMod); assert( pDel==0 || pDel==pMod ); if( pDel ){ db->mallocFailed = 1; @@ -81,6 +81,9 @@ int sqlite3_create_module( const sqlite3_module *pModule, /* The definition of the module */ void *pAux /* Context pointer for xCreate/xConnect */ ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT; +#endif return createModule(db, zName, pModule, pAux, 0); } @@ -94,6 +97,9 @@ int sqlite3_create_module_v2( void *pAux, /* Context pointer for xCreate/xConnect */ void (*xDestroy)(void *) /* Module destructor function */ ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT; +#endif return createModule(db, zName, pModule, pAux, xDestroy); } @@ -326,7 +332,12 @@ void sqlite3VtabBeginParse( addModuleArgument(db, pTable, sqlite3NameFromToken(db, pModuleName)); addModuleArgument(db, pTable, 0); addModuleArgument(db, pTable, sqlite3DbStrDup(db, pTable->zName)); - pParse->sNameToken.n = (int)(&pModuleName->z[pModuleName->n] - pName1->z); + assert( (pParse->sNameToken.z==pName2->z && pName2->z!=0) + || (pParse->sNameToken.z==pName1->z && pName2->z==0) + ); + pParse->sNameToken.n = (int)( + &pModuleName->z[pModuleName->n] - pParse->sNameToken.z + ); #ifndef SQLITE_OMIT_AUTHORIZATION /* Creating a virtual table invokes the authorization callback twice. @@ -425,9 +436,8 @@ void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){ Table *pOld; Schema *pSchema = pTab->pSchema; const char *zName = pTab->zName; - int nName = sqlite3Strlen30(zName); assert( sqlite3SchemaMutexHeld(db, 0, pSchema) ); - pOld = sqlite3HashInsert(&pSchema->tblHash, zName, nName, pTab); + pOld = sqlite3HashInsert(&pSchema->tblHash, zName, pTab); if( pOld ){ db->mallocFailed = 1; assert( pTab==pOld ); /* Malloc must have failed inside HashInsert() */ @@ -520,6 +530,7 @@ static int vtabCallConstructor( }else if( ALWAYS(pVTable->pVtab) ){ /* Justification of ALWAYS(): A correct vtab constructor must allocate ** the sqlite3_vtab object if successful. */ + memset(pVTable->pVtab, 0, sizeof(pVTable->pVtab[0])); pVTable->pVtab->pModule = pMod->pModule; pVTable->nRef = 1; if( sCtx.pTab ){ @@ -593,7 +604,7 @@ int sqlite3VtabCallConnect(Parse *pParse, Table *pTab){ /* Locate the required virtual table module */ zMod = pTab->azModuleArg[0]; - pMod = (Module*)sqlite3HashFind(&db->aModule, zMod, sqlite3Strlen30(zMod)); + pMod = (Module*)sqlite3HashFind(&db->aModule, zMod); if( !pMod ){ const char *zModule = pTab->azModuleArg[0]; @@ -661,7 +672,7 @@ int sqlite3VtabCallCreate(sqlite3 *db, int iDb, const char *zTab, char **pzErr){ /* Locate the required virtual table module */ zMod = pTab->azModuleArg[0]; - pMod = (Module*)sqlite3HashFind(&db->aModule, zMod, sqlite3Strlen30(zMod)); + pMod = (Module*)sqlite3HashFind(&db->aModule, zMod); /* If the module has been registered and includes a Create method, ** invoke it now. If the module has not been registered, return an @@ -698,9 +709,12 @@ int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ Table *pTab; char *zErr = 0; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); if( !db->pVtabCtx || !(pTab = db->pVtabCtx->pTab) ){ - sqlite3Error(db, SQLITE_MISUSE, 0); + sqlite3Error(db, SQLITE_MISUSE); sqlite3_mutex_leave(db->mutex); return SQLITE_MISUSE_BKPT; } @@ -728,7 +742,7 @@ int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ } db->pVtabCtx->pTab = 0; }else{ - sqlite3Error(db, SQLITE_ERROR, (zErr ? "%s" : 0), zErr); + sqlite3ErrorWithMsg(db, SQLITE_ERROR, (zErr ? "%s" : 0), zErr); sqlite3DbFree(db, zErr); rc = SQLITE_ERROR; } @@ -1055,6 +1069,9 @@ int sqlite3_vtab_on_conflict(sqlite3 *db){ static const unsigned char aMap[] = { SQLITE_ROLLBACK, SQLITE_ABORT, SQLITE_FAIL, SQLITE_IGNORE, SQLITE_REPLACE }; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif assert( OE_Rollback==1 && OE_Abort==2 && OE_Fail==3 ); assert( OE_Ignore==4 && OE_Replace==5 ); assert( db->vtabOnConflict>=1 && db->vtabOnConflict<=5 ); @@ -1070,8 +1087,10 @@ int sqlite3_vtab_config(sqlite3 *db, int op, ...){ va_list ap; int rc = SQLITE_OK; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); - va_start(ap, op); switch( op ){ case SQLITE_VTAB_CONSTRAINT_SUPPORT: { @@ -1090,7 +1109,7 @@ int sqlite3_vtab_config(sqlite3 *db, int op, ...){ } va_end(ap); - if( rc!=SQLITE_OK ) sqlite3Error(db, rc, 0); + if( rc!=SQLITE_OK ) sqlite3Error(db, rc); sqlite3_mutex_leave(db->mutex); return rc; } diff --git a/src/wal.c b/src/wal.c index 70395f8bee..f2738a6727 100644 --- a/src/wal.c +++ b/src/wal.c @@ -574,7 +574,7 @@ static volatile WalIndexHdr *walIndexHdr(Wal *pWal){ ** The argument to this macro must be of type u32. On a little-endian ** architecture, it returns the u32 value that results from interpreting ** the 4 bytes as a big-endian value. On a big-endian architecture, it -** returns the value that would be produced by intepreting the 4 bytes +** returns the value that would be produced by interpreting the 4 bytes ** of the input value as a little-endian integer. */ #define BYTESWAP32(x) ( \ @@ -988,7 +988,7 @@ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ assert( idx <= HASHTABLE_NSLOT/2 + 1 ); /* If this is the first entry to be added to this hash-table, zero the - ** entire hash table and aPgno[] array before proceding. + ** entire hash table and aPgno[] array before proceeding. */ if( idx==1 ){ int nByte = (int)((u8 *)&aHash[HASHTABLE_NSLOT] - (u8 *)&aPgno[1]); @@ -1504,7 +1504,7 @@ static void walMergesort( ** Free an iterator allocated by walIteratorInit(). */ static void walIteratorFree(WalIterator *p){ - sqlite3ScratchFree(p); + sqlite3_free(p); } /* @@ -1539,7 +1539,7 @@ static int walIteratorInit(Wal *pWal, WalIterator **pp){ nByte = sizeof(WalIterator) + (nSegment-1)*sizeof(struct WalSegment) + iLast*sizeof(ht_slot); - p = (WalIterator *)sqlite3ScratchMalloc(nByte); + p = (WalIterator *)sqlite3_malloc(nByte); if( !p ){ return SQLITE_NOMEM; } @@ -1549,7 +1549,7 @@ static int walIteratorInit(Wal *pWal, WalIterator **pp){ /* Allocate temporary space used by the merge-sort routine. This block ** of memory will be freed before this function returns. */ - aTmp = (ht_slot *)sqlite3ScratchMalloc( + aTmp = (ht_slot *)sqlite3_malloc( sizeof(ht_slot) * (iLast>HASHTABLE_NPAGE?HASHTABLE_NPAGE:iLast) ); if( !aTmp ){ @@ -1586,7 +1586,7 @@ static int walIteratorInit(Wal *pWal, WalIterator **pp){ p->aSegment[i].aPgno = (u32 *)aPgno; } } - sqlite3ScratchFree(aTmp); + sqlite3_free(aTmp); if( rc!=SQLITE_OK ){ walIteratorFree(p); @@ -1623,6 +1623,38 @@ static int walPagesize(Wal *pWal){ return (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16); } +/* +** The following is guaranteed when this function is called: +** +** a) the WRITER lock is held, +** b) the entire log file has been checkpointed, and +** c) any existing readers are reading exclusively from the database +** file - there are no readers that may attempt to read a frame from +** the log file. +** +** This function updates the shared-memory structures so that the next +** client to write to the database (which may be this one) does so by +** writing frames into the start of the log file. +** +** The value of parameter salt1 is used as the aSalt[1] value in the +** new wal-index header. It should be passed a pseudo-random value (i.e. +** one obtained from sqlite3_randomness()). +*/ +static void walRestartHdr(Wal *pWal, u32 salt1){ + volatile WalCkptInfo *pInfo = walCkptInfo(pWal); + int i; /* Loop counter */ + u32 *aSalt = pWal->hdr.aSalt; /* Big-endian salt values */ + pWal->nCkpt++; + pWal->hdr.mxFrame = 0; + sqlite3Put4byte((u8*)&aSalt[0], 1 + sqlite3Get4byte((u8*)&aSalt[0])); + memcpy(&pWal->hdr.aSalt[1], &salt1, 4); + walIndexWriteHdr(pWal); + pInfo->nBackfill = 0; + pInfo->aReadMark[1] = 0; + for(i=2; iaReadMark[i] = READMARK_NOT_USED; + assert( pInfo->aReadMark[0]==0 ); +} + /* ** Copy as much content as we can from the WAL back into the database file ** in response to an sqlite3_wal_checkpoint() request or the equivalent. @@ -1646,7 +1678,7 @@ static int walPagesize(Wal *pWal){ ** database file. ** ** This routine uses and updates the nBackfill field of the wal-index header. -** This is the only routine tha will increase the value of nBackfill. +** This is the only routine that will increase the value of nBackfill. ** (A WAL reset or recovery will revert nBackfill to zero, but not increase ** its value.) ** @@ -1657,7 +1689,7 @@ static int walPagesize(Wal *pWal){ static int walCheckpoint( Wal *pWal, /* Wal connection */ int eMode, /* One of PASSIVE, FULL or RESTART */ - int (*xBusyCall)(void*), /* Function to call when busy */ + int (*xBusy)(void*), /* Function to call when busy */ void *pBusyArg, /* Context argument for xBusyHandler */ int sync_flags, /* Flags for OsSync() (or 0) */ u8 *zBuf /* Temporary buffer to use */ @@ -1671,7 +1703,6 @@ static int walCheckpoint( u32 mxPage; /* Max database page to write */ int i; /* Loop counter */ volatile WalCkptInfo *pInfo; /* The checkpoint status information */ - int (*xBusy)(void*) = 0; /* Function to call when waiting for locks */ szPage = walPagesize(pWal); testcase( szPage<=32768 ); @@ -1686,7 +1717,9 @@ static int walCheckpoint( } assert( pIter ); - if( eMode!=SQLITE_CHECKPOINT_PASSIVE ) xBusy = xBusyCall; + /* EVIDENCE-OF: R-62920-47450 The busy-handler callback is never invoked + ** in the SQLITE_CHECKPOINT_PASSIVE mode. */ + assert( eMode!=SQLITE_CHECKPOINT_PASSIVE || xBusy==0 ); /* Compute in mxSafeFrame the index of the last frame of the WAL that is ** safe to write into the database. Frames beyond mxSafeFrame might @@ -1775,19 +1808,38 @@ static int walCheckpoint( rc = SQLITE_OK; } - /* If this is an SQLITE_CHECKPOINT_RESTART operation, and the entire wal - ** file has been copied into the database file, then block until all - ** readers have finished using the wal file. This ensures that the next - ** process to write to the database restarts the wal file. + /* If this is an SQLITE_CHECKPOINT_RESTART or TRUNCATE operation, and the + ** entire wal file has been copied into the database file, then block + ** until all readers have finished using the wal file. This ensures that + ** the next process to write to the database restarts the wal file. */ if( rc==SQLITE_OK && eMode!=SQLITE_CHECKPOINT_PASSIVE ){ assert( pWal->writeLock ); if( pInfo->nBackfillhdr.mxFrame ){ rc = SQLITE_BUSY; - }else if( eMode==SQLITE_CHECKPOINT_RESTART ){ + }else if( eMode>=SQLITE_CHECKPOINT_RESTART ){ + u32 salt1; + sqlite3_randomness(4, &salt1); assert( mxSafeFrame==pWal->hdr.mxFrame ); rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(1), WAL_NREADER-1); if( rc==SQLITE_OK ){ + if( eMode==SQLITE_CHECKPOINT_TRUNCATE ){ + /* IMPLEMENTATION-OF: R-44699-57140 This mode works the same way as + ** SQLITE_CHECKPOINT_RESTART with the addition that it also + ** truncates the log file to zero bytes just prior to a + ** successful return. + ** + ** In theory, it might be safe to do this without updating the + ** wal-index header in shared memory, as all subsequent reader or + ** writer clients should see that the entire log file has been + ** checkpointed and behave accordingly. This seems unsafe though, + ** as it would leave the system in a state where the contents of + ** the wal-index header do not match the contents of the + ** file-system. To avoid this, update the wal-index header to + ** indicate that the log file contains zero valid frames. */ + walRestartHdr(pWal, salt1); + rc = sqlite3OsTruncate(pWal->pWalFd, 0); + } walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); } } @@ -1950,7 +2002,7 @@ static int walIndexTryHdr(Wal *pWal, int *pChanged){ ** wal-index from the WAL before returning. ** ** Set *pChanged to 1 if the wal-index header value in pWal->hdr is -** changed by this opertion. If pWal->hdr is unchanged, set *pChanged +** changed by this operation. If pWal->hdr is unchanged, set *pChanged ** to 0. ** ** If the wal-index header is successfully read, return SQLITE_OK. @@ -2154,7 +2206,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ ** may have been appended to the log before READ_LOCK(0) was obtained. ** When holding READ_LOCK(0), the reader ignores the entire log file, ** which implies that the database file contains a trustworthy - ** snapshoT. Since holding READ_LOCK(0) prevents a checkpoint from + ** snapshot. Since holding READ_LOCK(0) prevents a checkpoint from ** happening, this is usually correct. ** ** However, if frames have been appended to the log (or if the log @@ -2360,7 +2412,7 @@ int sqlite3WalFindFrame( for(iKey=walHash(pgno); aHash[iKey]; iKey=walNextHash(iKey)){ u32 iFrame = aHash[iKey] + iZero; if( iFrame<=iLast && aPgno[aHash[iKey]]==pgno ){ - /* assert( iFrame>iRead ); -- not true if there is corruption */ + assert( iFrame>iRead || CORRUPT_DB ); iRead = iFrame; } if( (nCollide--)==0 ){ @@ -2525,7 +2577,6 @@ int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){ } if( iMax!=pWal->hdr.mxFrame ) walCleanupHash(pWal); } - assert( rc==SQLITE_OK ); return rc; } @@ -2574,7 +2625,6 @@ int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){ return rc; } - /* ** This function is called just before writing a set of frames to the log ** file (see sqlite3WalFrames()). It checks to see if, instead of appending @@ -2607,20 +2657,8 @@ static int walRestartLog(Wal *pWal){ ** In theory it would be Ok to update the cache of the header only ** at this point. But updating the actual wal-index header is also ** safe and means there is no special case for sqlite3WalUndo() - ** to handle if this transaction is rolled back. - */ - int i; /* Loop counter */ - u32 *aSalt = pWal->hdr.aSalt; /* Big-endian salt values */ - - pWal->nCkpt++; - pWal->hdr.mxFrame = 0; - sqlite3Put4byte((u8*)&aSalt[0], 1 + sqlite3Get4byte((u8*)&aSalt[0])); - aSalt[1] = salt1; - walIndexWriteHdr(pWal); - pInfo->nBackfill = 0; - pInfo->aReadMark[1] = 0; - for(i=2; iaReadMark[i] = READMARK_NOT_USED; - assert( pInfo->aReadMark[0]==0 ); + ** to handle if this transaction is rolled back. */ + walRestartHdr(pWal, salt1); walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); }else if( rc!=SQLITE_BUSY ){ return rc; @@ -2822,7 +2860,7 @@ int sqlite3WalFrames( ** ** Padding and syncing only occur if this set of frames complete a ** transaction and if PRAGMA synchronous=FULL. If synchronous==NORMAL - ** or synchonous==OFF, then no padding or syncing are needed. + ** or synchronous==OFF, then no padding or syncing are needed. ** ** If SQLITE_IOCAP_POWERSAFE_OVERWRITE is defined, then padding is not ** needed and only the sync is done. If padding is needed, then the @@ -2908,7 +2946,7 @@ int sqlite3WalFrames( */ int sqlite3WalCheckpoint( Wal *pWal, /* Wal connection */ - int eMode, /* PASSIVE, FULL or RESTART */ + int eMode, /* PASSIVE, FULL, RESTART, or TRUNCATE */ int (*xBusy)(void*), /* Function to call when busy */ void *pBusyArg, /* Context argument for xBusyHandler */ int sync_flags, /* Flags to sync db file with (or 0) */ @@ -2920,29 +2958,42 @@ int sqlite3WalCheckpoint( int rc; /* Return code */ int isChanged = 0; /* True if a new wal-index header is loaded */ int eMode2 = eMode; /* Mode to pass to walCheckpoint() */ + int (*xBusy2)(void*) = xBusy; /* Busy handler for eMode2 */ assert( pWal->ckptLock==0 ); assert( pWal->writeLock==0 ); + /* EVIDENCE-OF: R-62920-47450 The busy-handler callback is never invoked + ** in the SQLITE_CHECKPOINT_PASSIVE mode. */ + assert( eMode!=SQLITE_CHECKPOINT_PASSIVE || xBusy==0 ); + if( pWal->readOnly ) return SQLITE_READONLY; WALTRACE(("WAL%p: checkpoint begins\n", pWal)); + + /* IMPLEMENTATION-OF: R-62028-47212 All calls obtain an exclusive + ** "checkpoint" lock on the database file. */ rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); if( rc ){ - /* Usually this is SQLITE_BUSY meaning that another thread or process - ** is already running a checkpoint, or maybe a recovery. But it might - ** also be SQLITE_IOERR. */ + /* EVIDENCE-OF: R-10421-19736 If any other process is running a + ** checkpoint operation at the same time, the lock cannot be obtained and + ** SQLITE_BUSY is returned. + ** EVIDENCE-OF: R-53820-33897 Even if there is a busy-handler configured, + ** it will not be invoked in this case. + */ + testcase( rc==SQLITE_BUSY ); + testcase( xBusy!=0 ); return rc; } pWal->ckptLock = 1; - /* If this is a blocking-checkpoint, then obtain the write-lock as well - ** to prevent any writers from running while the checkpoint is underway. - ** This has to be done before the call to walIndexReadHdr() below. + /* IMPLEMENTATION-OF: R-59782-36818 The SQLITE_CHECKPOINT_FULL, RESTART and + ** TRUNCATE modes also obtain the exclusive "writer" lock on the database + ** file. ** - ** If the writer lock cannot be obtained, then a passive checkpoint is - ** run instead. Since the checkpointer is not holding the writer lock, - ** there is no point in blocking waiting for any readers. Assuming no - ** other error occurs, this function will return SQLITE_BUSY to the caller. + ** EVIDENCE-OF: R-60642-04082 If the writer lock cannot be obtained + ** immediately, and a busy-handler is configured, it is invoked and the + ** writer lock retried until either the busy-handler returns 0 or the + ** lock is successfully obtained. */ if( eMode!=SQLITE_CHECKPOINT_PASSIVE ){ rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_WRITE_LOCK, 1); @@ -2950,6 +3001,7 @@ int sqlite3WalCheckpoint( pWal->writeLock = 1; }else if( rc==SQLITE_BUSY ){ eMode2 = SQLITE_CHECKPOINT_PASSIVE; + xBusy2 = 0; rc = SQLITE_OK; } } @@ -2967,7 +3019,7 @@ int sqlite3WalCheckpoint( if( pWal->hdr.mxFrame && walPagesize(pWal)!=nBuf ){ rc = SQLITE_CORRUPT_BKPT; }else{ - rc = walCheckpoint(pWal, eMode2, xBusy, pBusyArg, sync_flags, zBuf); + rc = walCheckpoint(pWal, eMode2, xBusy2, pBusyArg, sync_flags, zBuf); } /* If no error occurred, set the output variables. */ diff --git a/src/walker.c b/src/walker.c index 016ae77a92..e30bb60b5a 100644 --- a/src/walker.c +++ b/src/walker.c @@ -19,7 +19,7 @@ /* ** Walk an expression tree. Invoke the callback once for each node -** of the expression, while decending. (In other words, the callback +** of the expression, while descending. (In other words, the callback ** is invoked before visiting children.) ** ** The return value from the callback should be one of the WRC_* diff --git a/src/where.c b/src/where.c index fd5831872b..183a8cb667 100644 --- a/src/where.c +++ b/src/where.c @@ -222,10 +222,11 @@ static int whereClauseInsert(WhereClause *pWC, Expr *p, u8 wtFlags){ sqlite3DbFree(db, pOld); } pWC->nSlot = sqlite3DbMallocSize(db, pWC->a)/sizeof(pWC->a[0]); + memset(&pWC->a[pWC->nTerm], 0, sizeof(pWC->a[0])*(pWC->nSlot-pWC->nTerm)); } pTerm = &pWC->a[idx = pWC->nTerm++]; if( p && ExprHasProperty(p, EP_Unlikely) ){ - pTerm->truthProb = sqlite3LogEst(p->iTable) - 99; + pTerm->truthProb = sqlite3LogEst(p->iTable) - 270; }else{ pTerm->truthProb = 1; } @@ -364,11 +365,6 @@ static int allowedOp(int op){ return op==TK_IN || (op>=TK_EQ && op<=TK_GE) || op==TK_ISNULL; } -/* -** Swap two objects of type TYPE. -*/ -#define SWAP(TYPE,A,B) {TYPE t=A; A=B; B=t;} - /* ** Commute a comparison operator. Expressions of the form "X op Y" ** are converted into "Y op X". @@ -544,7 +540,7 @@ static WhereTerm *whereScanInit( if( pIdx && iColumn>=0 ){ pScan->idxaff = pIdx->pTable->aCol[iColumn].affinity; for(j=0; pIdx->aiColumn[j]!=iColumn; j++){ - if( NEVER(j>=pIdx->nKeyCol) ) return 0; + if( NEVER(j>pIdx->nColumn) ) return 0; } pScan->zCollName = pIdx->azColl[j]; }else{ @@ -701,7 +697,7 @@ static int isLikeOrGlob( ** value of the variable means there is no need to invoke the LIKE ** function, then no OP_Variable will be added to the program. ** This causes problems for the sqlite3_bind_parameter_name() - ** API. To workaround them, add a dummy OP_Variable here. + ** API. To work around them, add a dummy OP_Variable here. */ int r1 = sqlite3GetTempReg(pParse); sqlite3ExprCodeTarget(pParse, pRight, r1); @@ -761,6 +757,15 @@ static void transferJoinMarkings(Expr *pDerived, Expr *pBase){ } } +/* +** Mark term iChild as being a child of term iParent +*/ +static void markTermAsChild(WhereClause *pWC, int iChild, int iParent){ + pWC->a[iChild].iParent = iParent; + pWC->a[iChild].truthProb = pWC->a[iParent].truthProb; + pWC->a[iParent].nChild++; +} + #if !defined(SQLITE_OMIT_OR_OPTIMIZATION) && !defined(SQLITE_OMIT_SUBQUERY) /* ** Analyze a term that consists of two or more OR-connected @@ -821,7 +826,7 @@ static void transferJoinMarkings(Expr *pDerived, Expr *pBase){ ** appropriate for indexing exist. ** ** All examples A through E above satisfy case 2. But if a term -** also statisfies case 1 (such as B) we know that the optimizer will +** also satisfies case 1 (such as B) we know that the optimizer will ** always prefer case 1, so in that case we pretend that case 2 is not ** satisfied. ** @@ -979,7 +984,7 @@ static void exprAnalyzeOrTerm( } if( (chngToIN & getMask(&pWInfo->sMaskSet, pOrTerm->leftCursor))==0 ){ /* This term must be of the form t1.a==t2.b where t2 is in the - ** chngToIN set but t1 is not. This term will be either preceeded + ** chngToIN set but t1 is not. This term will be either preceded ** or follwed by an inverted copy (t2.b==t1.a). Skip this term ** and use its inversion. */ testcase( pOrTerm->wtFlags & TERM_COPIED ); @@ -1058,8 +1063,7 @@ static void exprAnalyzeOrTerm( testcase( idxNew==0 ); exprAnalyze(pSrc, pWC, idxNew); pTerm = &pWC->a[idxTerm]; - pWC->a[idxNew].iParent = idxTerm; - pTerm->nChild = 1; + markTermAsChild(pWC, idxNew, idxTerm); }else{ sqlite3ExprListDelete(db, pList); } @@ -1161,9 +1165,8 @@ static void exprAnalyze( idxNew = whereClauseInsert(pWC, pDup, TERM_VIRTUAL|TERM_DYNAMIC); if( idxNew==0 ) return; pNew = &pWC->a[idxNew]; - pNew->iParent = idxTerm; + markTermAsChild(pWC, idxNew, idxTerm); pTerm = &pWC->a[idxTerm]; - pTerm->nChild = 1; pTerm->wtFlags |= TERM_COPIED; if( pExpr->op==TK_EQ && !ExprHasProperty(pExpr, EP_FromJoin) @@ -1220,9 +1223,8 @@ static void exprAnalyze( testcase( idxNew==0 ); exprAnalyze(pSrc, pWC, idxNew); pTerm = &pWC->a[idxTerm]; - pWC->a[idxNew].iParent = idxTerm; + markTermAsChild(pWC, idxNew, idxTerm); } - pTerm->nChild = 2; } #endif /* SQLITE_OMIT_BETWEEN_OPTIMIZATION */ @@ -1297,9 +1299,8 @@ static void exprAnalyze( exprAnalyze(pSrc, pWC, idxNew2); pTerm = &pWC->a[idxTerm]; if( isComplete ){ - pWC->a[idxNew1].iParent = idxTerm; - pWC->a[idxNew2].iParent = idxTerm; - pTerm->nChild = 2; + markTermAsChild(pWC, idxNew1, idxTerm); + markTermAsChild(pWC, idxNew2, idxTerm); } } #endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */ @@ -1332,9 +1333,8 @@ static void exprAnalyze( pNewTerm->leftCursor = pLeft->iTable; pNewTerm->u.leftColumn = pLeft->iColumn; pNewTerm->eOperator = WO_MATCH; - pNewTerm->iParent = idxTerm; + markTermAsChild(pWC, idxNew, idxTerm); pTerm = &pWC->a[idxTerm]; - pTerm->nChild = 1; pTerm->wtFlags |= TERM_COPIED; pNewTerm->prereqAll = pTerm->prereqAll; } @@ -1355,7 +1355,7 @@ static void exprAnalyze( if( pExpr->op==TK_NOTNULL && pExpr->pLeft->op==TK_COLUMN && pExpr->pLeft->iColumn>=0 - && OptimizationEnabled(db, SQLITE_Stat3) + && OptimizationEnabled(db, SQLITE_Stat34) ){ Expr *pNewExpr; Expr *pLeft = pExpr->pLeft; @@ -1374,9 +1374,8 @@ static void exprAnalyze( pNewTerm->leftCursor = pLeft->iTable; pNewTerm->u.leftColumn = pLeft->iColumn; pNewTerm->eOperator = WO_GT; - pNewTerm->iParent = idxTerm; + markTermAsChild(pWC, idxNew, idxTerm); pTerm = &pWC->a[idxTerm]; - pTerm->nChild = 1; pTerm->wtFlags |= TERM_COPIED; pNewTerm->prereqAll = pTerm->prereqAll; } @@ -1390,7 +1389,7 @@ static void exprAnalyze( } /* -** This function searches pList for a entry that matches the iCol-th column +** This function searches pList for an entry that matches the iCol-th column ** of index pIdx. ** ** If such an expression is found, its index in pList->a[] is returned. If @@ -1470,7 +1469,7 @@ static int isDistinctRedundant( ** contain a "col=X" term are subject to a NOT NULL constraint. */ for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - if( pIdx->onError==OE_None ) continue; + if( !IsUniqueIndex(pIdx) ) continue; for(i=0; inKeyCol; i++){ i16 iCol = pIdx->aiColumn[i]; if( 0==findTerm(pWC, iBase, iCol, ~(Bitmask)0, WO_EQ, pIdx) ){ @@ -1596,6 +1595,8 @@ static void constructAutomaticIndex( Bitmask idxCols; /* Bitmap of columns used for indexing */ Bitmask extraCols; /* Bitmap of additional columns */ u8 sentWarning = 0; /* True if a warnning has been issued */ + Expr *pPartial = 0; /* Partial Index Expression */ + int iContinue = 0; /* Jump here to skip excluded rows */ /* Generate code to skip over the creation and initialization of the ** transient index on 2nd and subsequent iterations of the loop. */ @@ -1611,6 +1612,12 @@ static void constructAutomaticIndex( pLoop = pLevel->pWLoop; idxCols = 0; for(pTerm=pWC->a; pTermprereq==0 + && (pTerm->wtFlags & TERM_VIRTUAL)==0 + && sqlite3ExprIsTableConstant(pTerm->pExpr, pSrc->iCursor) ){ + pPartial = sqlite3ExprAnd(pParse->db, pPartial, + sqlite3ExprDup(pParse->db, pTerm->pExpr, 0)); + } if( termCanDriveIndex(pTerm, pSrc, notReady) ){ int iCol = pTerm->u.leftColumn; Bitmask cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol); @@ -1623,7 +1630,9 @@ static void constructAutomaticIndex( sentWarning = 1; } if( (idxCols & cMask)==0 ){ - if( whereLoopResize(pParse->db, pLoop, nKeyCol+1) ) return; + if( whereLoopResize(pParse->db, pLoop, nKeyCol+1) ){ + goto end_auto_index_create; + } pLoop->aLTerm[nKeyCol++] = pTerm; idxCols |= cMask; } @@ -1643,7 +1652,7 @@ static void constructAutomaticIndex( ** if they go out of sync. */ extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1)); - mxBitCol = (pTable->nCol >= BMS-1) ? BMS-1 : pTable->nCol; + mxBitCol = MIN(BMS-1,pTable->nCol); testcase( pTable->nCol==BMS-1 ); testcase( pTable->nCol==BMS-2 ); for(i=0; icolUsed & MASKBIT(BMS-1) ){ nKeyCol += pTable->nCol - BMS + 1; } - pLoop->wsFlags |= WHERE_COLUMN_EQ | WHERE_IDX_ONLY; /* Construct the Index object to describe this index */ pIdx = sqlite3AllocateIndexObject(pParse->db, nKeyCol+1, 0, &zNotUsed); - if( pIdx==0 ) return; + if( pIdx==0 ) goto end_auto_index_create; pLoop->u.btree.pIndex = pIdx; pIdx->zName = "auto-index"; pIdx->pTable = pTable; @@ -1708,18 +1716,29 @@ static void constructAutomaticIndex( VdbeComment((v, "for %s", pTable->zName)); /* Fill the automatic index with content */ + sqlite3ExprCachePush(pParse); addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v); + if( pPartial ){ + iContinue = sqlite3VdbeMakeLabel(v); + sqlite3ExprIfFalse(pParse, pPartial, iContinue, SQLITE_JUMPIFNULL); + pLoop->wsFlags |= WHERE_PARTIALIDX; + } regRecord = sqlite3GetTempReg(pParse); sqlite3GenerateIndexKey(pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0); sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); + if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue); sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v); sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX); sqlite3VdbeJumpHere(v, addrTop); sqlite3ReleaseTempReg(pParse, regRecord); + sqlite3ExprCachePop(pParse); /* Jump here when skipping the initialization */ sqlite3VdbeJumpHere(v, addrInit); + +end_auto_index_create: + sqlite3ExprDelete(pParse->db, pPartial); } #endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ @@ -1879,7 +1898,6 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ } #endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */ - #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 /* ** Estimate the location of a particular key among all keys in an @@ -1888,9 +1906,10 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ ** aStat[0] Est. number of rows less than pVal ** aStat[1] Est. number of rows equal to pVal ** -** Return SQLITE_OK on success. +** Return the index of the sample that is the smallest sample that +** is greater than or equal to pRec. */ -static void whereKeyStats( +static int whereKeyStats( Parse *pParse, /* Database connection */ Index *pIdx, /* Index to consider domain of */ UnpackedRecord *pRec, /* Vector of values to consider */ @@ -1913,7 +1932,7 @@ static void whereKeyStats( assert( pRec->nField>0 && iColnSampleCol ); do{ iTest = (iMin+i)/2; - res = sqlite3VdbeRecordCompare(aSample[iTest].n, aSample[iTest].p, pRec, 0); + res = sqlite3VdbeRecordCompare(aSample[iTest].n, aSample[iTest].p, pRec); if( res<0 ){ iMin = iTest+1; }else{ @@ -1928,16 +1947,16 @@ static void whereKeyStats( if( res==0 ){ /* If (res==0) is true, then sample $i must be equal to pRec */ assert( inSample ); - assert( 0==sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec, 0) + assert( 0==sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec) || pParse->db->mallocFailed ); }else{ /* Otherwise, pRec must be smaller than sample $i and larger than ** sample ($i-1). */ assert( i==pIdx->nSample - || sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec, 0)>0 + || sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)>0 || pParse->db->mallocFailed ); assert( i==0 - || sqlite3VdbeRecordCompare(aSample[i-1].n, aSample[i-1].p, pRec, 0)<0 + || sqlite3VdbeRecordCompare(aSample[i-1].n, aSample[i-1].p, pRec)<0 || pParse->db->mallocFailed ); } #endif /* ifdef SQLITE_DEBUG */ @@ -1959,7 +1978,7 @@ static void whereKeyStats( iUpper = i>=pIdx->nSample ? nRow0 : aSample[i].anLt[iCol]; iLower = aSample[i-1].anEq[iCol] + aSample[i-1].anLt[iCol]; } - aStat[1] = (pIdx->nKeyCol>iCol ? pIdx->aAvgEq[iCol] : 1); + aStat[1] = pIdx->aAvgEq[iCol]; if( iLower>=iUpper ){ iGap = 0; }else{ @@ -1972,6 +1991,7 @@ static void whereKeyStats( } aStat[0] = iLower + iGap; } + return i; } #endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ @@ -1998,6 +2018,115 @@ static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){ return nRet; } +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +/* +** This function is called to estimate the number of rows visited by a +** range-scan on a skip-scan index. For example: +** +** CREATE INDEX i1 ON t1(a, b, c); +** SELECT * FROM t1 WHERE a=? AND c BETWEEN ? AND ?; +** +** Value pLoop->nOut is currently set to the estimated number of rows +** visited for scanning (a=? AND b=?). This function reduces that estimate +** by some factor to account for the (c BETWEEN ? AND ?) expression based +** on the stat4 data for the index. this scan will be peformed multiple +** times (once for each (a,b) combination that matches a=?) is dealt with +** by the caller. +** +** It does this by scanning through all stat4 samples, comparing values +** extracted from pLower and pUpper with the corresponding column in each +** sample. If L and U are the number of samples found to be less than or +** equal to the values extracted from pLower and pUpper respectively, and +** N is the total number of samples, the pLoop->nOut value is adjusted +** as follows: +** +** nOut = nOut * ( min(U - L, 1) / N ) +** +** If pLower is NULL, or a value cannot be extracted from the term, L is +** set to zero. If pUpper is NULL, or a value cannot be extracted from it, +** U is set to N. +** +** Normally, this function sets *pbDone to 1 before returning. However, +** if no value can be extracted from either pLower or pUpper (and so the +** estimate of the number of rows delivered remains unchanged), *pbDone +** is left as is. +** +** If an error occurs, an SQLite error code is returned. Otherwise, +** SQLITE_OK. +*/ +static int whereRangeSkipScanEst( + Parse *pParse, /* Parsing & code generating context */ + WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */ + WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */ + WhereLoop *pLoop, /* Update the .nOut value of this loop */ + int *pbDone /* Set to true if at least one expr. value extracted */ +){ + Index *p = pLoop->u.btree.pIndex; + int nEq = pLoop->u.btree.nEq; + sqlite3 *db = pParse->db; + int nLower = -1; + int nUpper = p->nSample+1; + int rc = SQLITE_OK; + int iCol = p->aiColumn[nEq]; + u8 aff = iCol>=0 ? p->pTable->aCol[iCol].affinity : SQLITE_AFF_INTEGER; + CollSeq *pColl; + + sqlite3_value *p1 = 0; /* Value extracted from pLower */ + sqlite3_value *p2 = 0; /* Value extracted from pUpper */ + sqlite3_value *pVal = 0; /* Value extracted from record */ + + pColl = sqlite3LocateCollSeq(pParse, p->azColl[nEq]); + if( pLower ){ + rc = sqlite3Stat4ValueFromExpr(pParse, pLower->pExpr->pRight, aff, &p1); + nLower = 0; + } + if( pUpper && rc==SQLITE_OK ){ + rc = sqlite3Stat4ValueFromExpr(pParse, pUpper->pExpr->pRight, aff, &p2); + nUpper = p2 ? 0 : p->nSample; + } + + if( p1 || p2 ){ + int i; + int nDiff; + for(i=0; rc==SQLITE_OK && inSample; i++){ + rc = sqlite3Stat4Column(db, p->aSample[i].p, p->aSample[i].n, nEq, &pVal); + if( rc==SQLITE_OK && p1 ){ + int res = sqlite3MemCompare(p1, pVal, pColl); + if( res>=0 ) nLower++; + } + if( rc==SQLITE_OK && p2 ){ + int res = sqlite3MemCompare(p2, pVal, pColl); + if( res>=0 ) nUpper++; + } + } + nDiff = (nUpper - nLower); + if( nDiff<=0 ) nDiff = 1; + + /* If there is both an upper and lower bound specified, and the + ** comparisons indicate that they are close together, use the fallback + ** method (assume that the scan visits 1/64 of the rows) for estimating + ** the number of rows visited. Otherwise, estimate the number of rows + ** using the method described in the header comment for this function. */ + if( nDiff!=1 || pUpper==0 || pLower==0 ){ + int nAdjust = (sqlite3LogEst(p->nSample) - sqlite3LogEst(nDiff)); + pLoop->nOut -= nAdjust; + *pbDone = 1; + WHERETRACE(0x10, ("range skip-scan regions: %u..%u adjust=%d est=%d\n", + nLower, nUpper, nAdjust*-1, pLoop->nOut)); + } + + }else{ + assert( *pbDone==0 ); + } + + sqlite3ValueFree(p1); + sqlite3ValueFree(p2); + sqlite3ValueFree(pVal); + + return rc; +} +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ + /* ** This function is used to estimate the number of rows that will be visited ** by scanning an index for a range of values. The range may have an upper @@ -2013,7 +2142,7 @@ static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){ ** If either of the upper or lower bound is not present, then NULL is passed in ** place of the corresponding WhereTerm. ** -** The value in (pBuilder->pNew->u.btree.nEq) is the index of the index +** The value in (pBuilder->pNew->u.btree.nEq) is the number of the index ** column subject to the range constraint. Or, equivalently, the number of ** equality constraints optimized by the proposed index scan. For example, ** assuming index p is on t1(a, b), and the SQL query is: @@ -2029,9 +2158,9 @@ static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){ ** ** When this function is called, *pnOut is set to the sqlite3LogEst() of the ** number of rows that the index scan is expected to visit without -** considering the range constraints. If nEq is 0, this is the number of +** considering the range constraints. If nEq is 0, then *pnOut is the number of ** rows in the index. Assuming no error occurs, *pnOut is adjusted (reduced) -** to account for the range contraints pLower and pUpper. +** to account for the range constraints pLower and pUpper. ** ** In the absence of sqlite_stat4 ANALYZE data, or if such data cannot be ** used, a single range inequality reduces the search space by a factor of 4. @@ -2053,117 +2182,147 @@ static int whereRangeScanEst( Index *p = pLoop->u.btree.pIndex; int nEq = pLoop->u.btree.nEq; - if( p->nSample>0 - && nEq==pBuilder->nRecValid - && nEqnSampleCol - && OptimizationEnabled(pParse->db, SQLITE_Stat3) - ){ - UnpackedRecord *pRec = pBuilder->pRec; - tRowcnt a[2]; - u8 aff; + if( p->nSample>0 && nEqnSampleCol ){ + if( nEq==pBuilder->nRecValid ){ + UnpackedRecord *pRec = pBuilder->pRec; + tRowcnt a[2]; + u8 aff; - /* Variable iLower will be set to the estimate of the number of rows in - ** the index that are less than the lower bound of the range query. The - ** lower bound being the concatenation of $P and $L, where $P is the - ** key-prefix formed by the nEq values matched against the nEq left-most - ** columns of the index, and $L is the value in pLower. - ** - ** Or, if pLower is NULL or $L cannot be extracted from it (because it - ** is not a simple variable or literal value), the lower bound of the - ** range is $P. Due to a quirk in the way whereKeyStats() works, even - ** if $L is available, whereKeyStats() is called for both ($P) and - ** ($P:$L) and the larger of the two returned values used. - ** - ** Similarly, iUpper is to be set to the estimate of the number of rows - ** less than the upper bound of the range query. Where the upper bound - ** is either ($P) or ($P:$U). Again, even if $U is available, both values - ** of iUpper are requested of whereKeyStats() and the smaller used. - */ - tRowcnt iLower; - tRowcnt iUpper; + /* Variable iLower will be set to the estimate of the number of rows in + ** the index that are less than the lower bound of the range query. The + ** lower bound being the concatenation of $P and $L, where $P is the + ** key-prefix formed by the nEq values matched against the nEq left-most + ** columns of the index, and $L is the value in pLower. + ** + ** Or, if pLower is NULL or $L cannot be extracted from it (because it + ** is not a simple variable or literal value), the lower bound of the + ** range is $P. Due to a quirk in the way whereKeyStats() works, even + ** if $L is available, whereKeyStats() is called for both ($P) and + ** ($P:$L) and the larger of the two returned values is used. + ** + ** Similarly, iUpper is to be set to the estimate of the number of rows + ** less than the upper bound of the range query. Where the upper bound + ** is either ($P) or ($P:$U). Again, even if $U is available, both values + ** of iUpper are requested of whereKeyStats() and the smaller used. + ** + ** The number of rows between the two bounds is then just iUpper-iLower. + */ + tRowcnt iLower; /* Rows less than the lower bound */ + tRowcnt iUpper; /* Rows less than the upper bound */ + int iLwrIdx = -2; /* aSample[] for the lower bound */ + int iUprIdx = -1; /* aSample[] for the upper bound */ - if( nEq==p->nKeyCol ){ - aff = SQLITE_AFF_INTEGER; - }else{ - aff = p->pTable->aCol[p->aiColumn[nEq]].affinity; - } - /* Determine iLower and iUpper using ($P) only. */ - if( nEq==0 ){ - iLower = 0; - iUpper = sqlite3LogEstToInt(p->aiRowLogEst[0]); - }else{ - /* Note: this call could be optimized away - since the same values must - ** have been requested when testing key $P in whereEqualScanEst(). */ - whereKeyStats(pParse, p, pRec, 0, a); - iLower = a[0]; - iUpper = a[0] + a[1]; - } - - /* If possible, improve on the iLower estimate using ($P:$L). */ - if( pLower ){ - int bOk; /* True if value is extracted from pExpr */ - Expr *pExpr = pLower->pExpr->pRight; - assert( (pLower->eOperator & (WO_GT|WO_GE))!=0 ); - rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); - if( rc==SQLITE_OK && bOk ){ - tRowcnt iNew; - whereKeyStats(pParse, p, pRec, 0, a); - iNew = a[0] + ((pLower->eOperator & WO_GT) ? a[1] : 0); - if( iNew>iLower ) iLower = iNew; - nOut--; + if( pRec ){ + testcase( pRec->nField!=pBuilder->nRecValid ); + pRec->nField = pBuilder->nRecValid; } - } - - /* If possible, improve on the iUpper estimate using ($P:$U). */ - if( pUpper ){ - int bOk; /* True if value is extracted from pExpr */ - Expr *pExpr = pUpper->pExpr->pRight; - assert( (pUpper->eOperator & (WO_LT|WO_LE))!=0 ); - rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); - if( rc==SQLITE_OK && bOk ){ - tRowcnt iNew; - whereKeyStats(pParse, p, pRec, 1, a); - iNew = a[0] + ((pUpper->eOperator & WO_LE) ? a[1] : 0); - if( iNewpRec = pRec; - if( rc==SQLITE_OK ){ - if( iUpper>iLower ){ - nNew = sqlite3LogEst(iUpper - iLower); + if( nEq==p->nKeyCol ){ + aff = SQLITE_AFF_INTEGER; }else{ - nNew = 10; assert( 10==sqlite3LogEst(2) ); + aff = p->pTable->aCol[p->aiColumn[nEq]].affinity; } - if( nNewnRowEst0; + }else{ + /* Note: this call could be optimized away - since the same values must + ** have been requested when testing key $P in whereEqualScanEst(). */ + whereKeyStats(pParse, p, pRec, 0, a); + iLower = a[0]; + iUpper = a[0] + a[1]; } - pLoop->nOut = (LogEst)nOut; - WHERETRACE(0x10, ("range scan regions: %u..%u est=%d\n", - (u32)iLower, (u32)iUpper, nOut)); - return SQLITE_OK; + + assert( pLower==0 || (pLower->eOperator & (WO_GT|WO_GE))!=0 ); + assert( pUpper==0 || (pUpper->eOperator & (WO_LT|WO_LE))!=0 ); + assert( p->aSortOrder!=0 ); + if( p->aSortOrder[nEq] ){ + /* The roles of pLower and pUpper are swapped for a DESC index */ + SWAP(WhereTerm*, pLower, pUpper); + } + + /* If possible, improve on the iLower estimate using ($P:$L). */ + if( pLower ){ + int bOk; /* True if value is extracted from pExpr */ + Expr *pExpr = pLower->pExpr->pRight; + rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); + if( rc==SQLITE_OK && bOk ){ + tRowcnt iNew; + iLwrIdx = whereKeyStats(pParse, p, pRec, 0, a); + iNew = a[0] + ((pLower->eOperator & (WO_GT|WO_LE)) ? a[1] : 0); + if( iNew>iLower ) iLower = iNew; + nOut--; + pLower = 0; + } + } + + /* If possible, improve on the iUpper estimate using ($P:$U). */ + if( pUpper ){ + int bOk; /* True if value is extracted from pExpr */ + Expr *pExpr = pUpper->pExpr->pRight; + rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); + if( rc==SQLITE_OK && bOk ){ + tRowcnt iNew; + iUprIdx = whereKeyStats(pParse, p, pRec, 1, a); + iNew = a[0] + ((pUpper->eOperator & (WO_GT|WO_LE)) ? a[1] : 0); + if( iNewpRec = pRec; + if( rc==SQLITE_OK ){ + if( iUpper>iLower ){ + nNew = sqlite3LogEst(iUpper - iLower); + /* TUNING: If both iUpper and iLower are derived from the same + ** sample, then assume they are 4x more selective. This brings + ** the estimated selectivity more in line with what it would be + ** if estimated without the use of STAT3/4 tables. */ + if( iLwrIdx==iUprIdx ) nNew -= 20; assert( 20==sqlite3LogEst(4) ); + }else{ + nNew = 10; assert( 10==sqlite3LogEst(2) ); + } + if( nNewwtFlags & TERM_VNULL)==0 ); nNew = whereRangeAdjust(pLower, nOut); nNew = whereRangeAdjust(pUpper, nNew); - /* TUNING: If there is both an upper and lower limit, assume the range is + /* TUNING: If there is both an upper and lower limit and neither limit + ** has an application-defined likelihood(), assume the range is ** reduced by an additional 75%. This means that, by default, an open-ended ** range query (e.g. col > ?) is assumed to match 1/4 of the rows in the ** index. While a closed range (e.g. col BETWEEN ? AND ?) is estimated to ** match 1/64 of the index. */ - if( pLower && pUpper ) nNew -= 20; + if( pLower && pLower->truthProb>0 && pUpper && pUpper->truthProb>0 ){ + nNew -= 20; + } nOut -= (pLower!=0) + (pUpper!=0); if( nNew<10 ) nNew = 10; if( nNewnOut>nOut ){ + WHERETRACE(0x10,("Range scan lowers nOut from %d to %d\n", + pLoop->nOut, nOut)); + } +#endif pLoop->nOut = (LogEst)nOut; return rc; } @@ -2201,7 +2360,7 @@ static int whereEqualScanEst( int bOk; assert( nEq>=1 ); - assert( nEq<=(p->nKeyCol+1) ); + assert( nEq<=p->nColumn ); assert( p->aSample!=0 ); assert( p->nSample>0 ); assert( pBuilder->nRecValidp->nKeyCol ){ + if( nEq>=p->nColumn ){ *pnRow = 1; return SQLITE_OK; } @@ -2276,7 +2435,7 @@ static int whereInScanEst( if( rc==SQLITE_OK ){ if( nRowEst > nRow0 ) nRowEst = nRow0; *pnRow = nRowEst; - WHERETRACE(0x10,("IN row estimate: est=%g\n", nRowEst)); + WHERETRACE(0x10,("IN row estimate: est=%d\n", nRowEst)); } assert( pBuilder->nRecValid==nRecValid ); return rc; @@ -2408,7 +2567,7 @@ static int codeEqualityTerm( } assert( pX->op==TK_IN ); iReg = iTarget; - eType = sqlite3FindInIndex(pParse, pX, 0); + eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0); if( eType==IN_INDEX_INDEX_DESC ){ testcase( bRev ); bRev = !bRev; @@ -2513,7 +2672,7 @@ static int codeAllEqualityTerms( pLoop = pLevel->pWLoop; assert( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 ); nEq = pLoop->u.btree.nEq; - nSkip = pLoop->u.btree.nSkip; + nSkip = pLoop->nSkip; pIdx = pLoop->u.btree.pIndex; assert( pIdx!=0 ); @@ -2612,9 +2771,8 @@ static void explainAppendTerm( /* ** Argument pLevel describes a strategy for scanning table pTab. This -** function returns a pointer to a string buffer containing a description -** of the subset of table rows scanned by the strategy in the form of an -** SQL expression. Or, if all rows are scanned, NULL is returned. +** function appends text to pStr that describes the subset of table +** rows scanned by the strategy in the form of an SQL expression. ** ** For example, if the query: ** @@ -2624,58 +2782,49 @@ static void explainAppendTerm( ** string similar to: ** ** "a=? AND b>?" -** -** The returned pointer points to memory obtained from sqlite3DbMalloc(). -** It is the responsibility of the caller to free the buffer when it is -** no longer required. */ -static char *explainIndexRange(sqlite3 *db, WhereLoop *pLoop, Table *pTab){ +static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop, Table *pTab){ Index *pIndex = pLoop->u.btree.pIndex; u16 nEq = pLoop->u.btree.nEq; - u16 nSkip = pLoop->u.btree.nSkip; + u16 nSkip = pLoop->nSkip; int i, j; Column *aCol = pTab->aCol; i16 *aiColumn = pIndex->aiColumn; - StrAccum txt; - if( nEq==0 && (pLoop->wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))==0 ){ - return 0; - } - sqlite3StrAccumInit(&txt, 0, 0, SQLITE_MAX_LENGTH); - txt.db = db; - sqlite3StrAccumAppend(&txt, " (", 2); + if( nEq==0 && (pLoop->wsFlags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))==0 ) return; + sqlite3StrAccumAppend(pStr, " (", 2); for(i=0; inKeyCol ) ? "rowid" : aCol[aiColumn[i]].zName; + char *z = aiColumn[i] < 0 ? "rowid" : aCol[aiColumn[i]].zName; if( i>=nSkip ){ - explainAppendTerm(&txt, i, z, "="); + explainAppendTerm(pStr, i, z, "="); }else{ - if( i ) sqlite3StrAccumAppend(&txt, " AND ", 5); - sqlite3StrAccumAppend(&txt, "ANY(", 4); - sqlite3StrAccumAppendAll(&txt, z); - sqlite3StrAccumAppend(&txt, ")", 1); + if( i ) sqlite3StrAccumAppend(pStr, " AND ", 5); + sqlite3XPrintf(pStr, 0, "ANY(%s)", z); } } j = i; if( pLoop->wsFlags&WHERE_BTM_LIMIT ){ - char *z = (j==pIndex->nKeyCol ) ? "rowid" : aCol[aiColumn[j]].zName; - explainAppendTerm(&txt, i++, z, ">"); + char *z = aiColumn[j] < 0 ? "rowid" : aCol[aiColumn[j]].zName; + explainAppendTerm(pStr, i++, z, ">"); } if( pLoop->wsFlags&WHERE_TOP_LIMIT ){ - char *z = (j==pIndex->nKeyCol ) ? "rowid" : aCol[aiColumn[j]].zName; - explainAppendTerm(&txt, i, z, "<"); + char *z = aiColumn[j] < 0 ? "rowid" : aCol[aiColumn[j]].zName; + explainAppendTerm(pStr, i, z, "<"); } - sqlite3StrAccumAppend(&txt, ")", 1); - return sqlite3StrAccumFinish(&txt); + sqlite3StrAccumAppend(pStr, ")", 1); } /* ** This function is a no-op unless currently processing an EXPLAIN QUERY PLAN -** command. If the query being compiled is an EXPLAIN QUERY PLAN, a single -** record is added to the output to describe the table scan strategy in -** pLevel. +** command, or if either SQLITE_DEBUG or SQLITE_ENABLE_STMT_SCANSTATUS was +** defined at compile-time. If it is not a no-op, a single OP_Explain opcode +** is added to the output to describe the table scan strategy in pLevel. +** +** If an OP_Explain opcode is added to the VM, its address is returned. +** Otherwise, if no OP_Explain is coded, zero is returned. */ -static void explainOneScan( +static int explainOneScan( Parse *pParse, /* Parse context */ SrcList *pTabList, /* Table list this loop refers to */ WhereLevel *pLevel, /* Scan to write OP_Explain opcode for */ @@ -2683,82 +2832,136 @@ static void explainOneScan( int iFrom, /* Value for "from" column of output */ u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ ){ -#ifndef SQLITE_DEBUG + int ret = 0; +#if !defined(SQLITE_DEBUG) && !defined(SQLITE_ENABLE_STMT_SCANSTATUS) if( pParse->explain==2 ) #endif { struct SrcList_item *pItem = &pTabList->a[pLevel->iFrom]; Vdbe *v = pParse->pVdbe; /* VM being constructed */ sqlite3 *db = pParse->db; /* Database handle */ - char *zMsg; /* Text to add to EQP output */ int iId = pParse->iSelectId; /* Select id (left-most output column) */ int isSearch; /* True for a SEARCH. False for SCAN. */ WhereLoop *pLoop; /* The controlling WhereLoop object */ u32 flags; /* Flags that describe this loop */ + char *zMsg; /* Text to add to EQP output */ + StrAccum str; /* EQP output string */ + char zBuf[100]; /* Initial space for EQP output string */ pLoop = pLevel->pWLoop; flags = pLoop->wsFlags; - if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_ONETABLE_ONLY) ) return; + if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_ONETABLE_ONLY) ) return 0; isSearch = (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 || ((flags&WHERE_VIRTUALTABLE)==0 && (pLoop->u.btree.nEq>0)) || (wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX)); - zMsg = sqlite3MPrintf(db, "%s", isSearch?"SEARCH":"SCAN"); + sqlite3StrAccumInit(&str, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH); + str.db = db; + sqlite3StrAccumAppendAll(&str, isSearch ? "SEARCH" : "SCAN"); if( pItem->pSelect ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s SUBQUERY %d", zMsg,pItem->iSelectId); + sqlite3XPrintf(&str, 0, " SUBQUERY %d", pItem->iSelectId); }else{ - zMsg = sqlite3MAppendf(db, zMsg, "%s TABLE %s", zMsg, pItem->zName); + sqlite3XPrintf(&str, 0, " TABLE %s", pItem->zName); } if( pItem->zAlias ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s AS %s", zMsg, pItem->zAlias); + sqlite3XPrintf(&str, 0, " AS %s", pItem->zAlias); } - if( (flags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 - && ALWAYS(pLoop->u.btree.pIndex!=0) - ){ - const char *zFmt; - Index *pIdx = pLoop->u.btree.pIndex; - char *zWhere = explainIndexRange(db, pLoop, pItem->pTab); + if( (flags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 ){ + const char *zFmt = 0; + Index *pIdx; + + assert( pLoop->u.btree.pIndex!=0 ); + pIdx = pLoop->u.btree.pIndex; assert( !(flags&WHERE_AUTO_INDEX) || (flags&WHERE_IDX_ONLY) ); if( !HasRowid(pItem->pTab) && IsPrimaryKeyIndex(pIdx) ){ - zFmt = zWhere ? "%s USING PRIMARY KEY%.0s%s" : "%s%.0s%s"; + if( isSearch ){ + zFmt = "PRIMARY KEY"; + } + }else if( flags & WHERE_PARTIALIDX ){ + zFmt = "AUTOMATIC PARTIAL COVERING INDEX"; }else if( flags & WHERE_AUTO_INDEX ){ - zFmt = "%s USING AUTOMATIC COVERING INDEX%.0s%s"; + zFmt = "AUTOMATIC COVERING INDEX"; }else if( flags & WHERE_IDX_ONLY ){ - zFmt = "%s USING COVERING INDEX %s%s"; + zFmt = "COVERING INDEX %s"; }else{ - zFmt = "%s USING INDEX %s%s"; + zFmt = "INDEX %s"; + } + if( zFmt ){ + sqlite3StrAccumAppend(&str, " USING ", 7); + sqlite3XPrintf(&str, 0, zFmt, pIdx->zName); + explainIndexRange(&str, pLoop, pItem->pTab); } - zMsg = sqlite3MAppendf(db, zMsg, zFmt, zMsg, pIdx->zName, zWhere); - sqlite3DbFree(db, zWhere); }else if( (flags & WHERE_IPK)!=0 && (flags & WHERE_CONSTRAINT)!=0 ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s USING INTEGER PRIMARY KEY", zMsg); - + const char *zRange; if( flags&(WHERE_COLUMN_EQ|WHERE_COLUMN_IN) ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid=?)", zMsg); + zRange = "(rowid=?)"; }else if( (flags&WHERE_BOTH_LIMIT)==WHERE_BOTH_LIMIT ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid>? AND rowid? AND rowid?)", zMsg); - }else if( ALWAYS(flags&WHERE_TOP_LIMIT) ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid?)"; + }else{ + assert( flags&WHERE_TOP_LIMIT); + zRange = "(rowidu.vtab.idxNum, pLoop->u.vtab.idxStr); } #endif - zMsg = sqlite3MAppendf(db, zMsg, "%s", zMsg); - sqlite3VdbeAddOp4(v, OP_Explain, iId, iLevel, iFrom, zMsg, P4_DYNAMIC); +#ifdef SQLITE_EXPLAIN_ESTIMATED_ROWS + if( pLoop->nOut>=10 ){ + sqlite3XPrintf(&str, 0, " (~%llu rows)", sqlite3LogEstToInt(pLoop->nOut)); + }else{ + sqlite3StrAccumAppend(&str, " (~1 row)", 9); + } +#endif + zMsg = sqlite3StrAccumFinish(&str); + ret = sqlite3VdbeAddOp4(v, OP_Explain, iId, iLevel, iFrom, zMsg,P4_DYNAMIC); } + return ret; } #else -# define explainOneScan(u,v,w,x,y,z) +# define explainOneScan(u,v,w,x,y,z) 0 #endif /* SQLITE_OMIT_EXPLAIN */ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +/* +** Configure the VM passed as the first argument with an +** sqlite3_stmt_scanstatus() entry corresponding to the scan used to +** implement level pLvl. Argument pSrclist is a pointer to the FROM +** clause that the scan reads data from. +** +** If argument addrExplain is not 0, it must be the address of an +** OP_Explain instruction that describes the same loop. +*/ +static void addScanStatus( + Vdbe *v, /* Vdbe to add scanstatus entry to */ + SrcList *pSrclist, /* FROM clause pLvl reads data from */ + WhereLevel *pLvl, /* Level to add scanstatus() entry for */ + int addrExplain /* Address of OP_Explain (or 0) */ +){ + const char *zObj = 0; + WhereLoop *pLoop = pLvl->pWLoop; + if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 && pLoop->u.btree.pIndex!=0 ){ + zObj = pLoop->u.btree.pIndex->zName; + }else{ + zObj = pSrclist->a[pLvl->iFrom].zName; + } + sqlite3VdbeScanStatus( + v, addrExplain, pLvl->addrBody, pLvl->addrVisit, pLoop->nOut, zObj + ); +} +#else +# define addScanStatus(a, b, c, d) ((void)d) +#endif + + /* ** Generate code for the start of the iLevel-th loop in the WHERE clause @@ -3059,7 +3262,7 @@ static Bitmask codeOneLoopStart( pIdx = pLoop->u.btree.pIndex; iIdxCur = pLevel->iIdxCur; - assert( nEq>=pLoop->u.btree.nSkip ); + assert( nEq>=pLoop->nSkip ); /* If this loop satisfies a sort order (pOrderBy) request that ** was passed to this function to implement a "SELECT min(x) ..." @@ -3076,7 +3279,7 @@ static Bitmask codeOneLoopStart( && pWInfo->nOBSat>0 && (pIdx->nKeyCol>nEq) ){ - assert( pLoop->u.btree.nSkip==0 ); + assert( pLoop->nSkip==0 ); bSeekPastNull = 1; nExtraReg = 1; } @@ -3293,7 +3496,7 @@ static Bitmask codeOneLoopStart( ** B: ** ** Added 2014-05-26: If the table is a WITHOUT ROWID table, then - ** use an ephermeral index instead of a RowSet to record the primary + ** use an ephemeral index instead of a RowSet to record the primary ** keys of the rows we have already seen. ** */ @@ -3309,6 +3512,7 @@ static Bitmask codeOneLoopStart( int iRetInit; /* Address of regReturn init */ int untestedTerms = 0; /* Some terms not completely tested */ int ii; /* Loop counter */ + u16 wctrlFlags; /* Flags for sub-WHERE clause */ Expr *pAndExpr = 0; /* An ".. AND (...)" expression */ Table *pTab = pTabItem->pTab; @@ -3343,7 +3547,7 @@ static Bitmask codeOneLoopStart( } /* Initialize the rowset register to contain NULL. An SQL NULL is - ** equivalent to an empty rowset. Or, create an ephermeral index + ** equivalent to an empty rowset. Or, create an ephemeral index ** capable of holding primary keys in the case of a WITHOUT ROWID. ** ** Also initialize regReturn to contain the address of the instruction @@ -3388,10 +3592,9 @@ static Bitmask codeOneLoopStart( Expr *pExpr = pWC->a[iTerm].pExpr; if( &pWC->a[iTerm] == pTerm ) continue; if( ExprHasProperty(pExpr, EP_FromJoin) ) continue; - testcase( pWC->a[iTerm].wtFlags & TERM_ORINFO ); - testcase( pWC->a[iTerm].wtFlags & TERM_VIRTUAL ); - if( pWC->a[iTerm].wtFlags & (TERM_ORINFO|TERM_VIRTUAL) ) continue; + if( (pWC->a[iTerm].wtFlags & TERM_VIRTUAL)!=0 ) continue; if( (pWC->a[iTerm].eOperator & WO_ALL)==0 ) continue; + testcase( pWC->a[iTerm].wtFlags & TERM_ORINFO ); pExpr = sqlite3ExprDup(db, pExpr, 0); pAndExpr = sqlite3ExprAnd(db, pAndExpr, pExpr); } @@ -3404,6 +3607,9 @@ static Bitmask codeOneLoopStart( ** eliminating duplicates from other WHERE clauses, the action for each ** sub-WHERE clause is to to invoke the main loop body as a subroutine. */ + wctrlFlags = WHERE_OMIT_OPEN_CLOSE + | WHERE_FORCE_TABLE + | WHERE_ONETABLE_ONLY; for(ii=0; iinTerm; ii++){ WhereTerm *pOrTerm = &pOrWc->a[ii]; if( pOrTerm->leftCursor==iCur || (pOrTerm->eOperator & WO_AND)!=0 ){ @@ -3415,15 +3621,17 @@ static Bitmask codeOneLoopStart( pOrExpr = pAndExpr; } /* Loop through table entries that match term pOrTerm. */ + WHERETRACE(0xffff, ("Subplan for OR-clause:\n")); pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrExpr, 0, 0, - WHERE_OMIT_OPEN_CLOSE | WHERE_AND_ONLY | - WHERE_FORCE_TABLE | WHERE_ONETABLE_ONLY, iCovCur); + wctrlFlags, iCovCur); assert( pSubWInfo || pParse->nErr || db->mallocFailed ); if( pSubWInfo ){ WhereLoop *pSubLoop; - explainOneScan( + int addrExplain = explainOneScan( pParse, pOrTab, &pSubWInfo->a[0], iLevel, pLevel->iFrom, 0 ); + addScanStatus(v, pOrTab, &pSubWInfo->a[0], addrExplain); + /* This is the sub-WHERE clause body. First skip over ** duplicate rows from prior sub-WHERE clauses, and record the ** rowid (or PRIMARY KEY) for the current row so that the same @@ -3508,6 +3716,7 @@ static Bitmask codeOneLoopStart( ){ assert( pSubWInfo->a[0].iIdxCur==iCovCur ); pCov = pSubLoop->u.btree.pIndex; + wctrlFlags |= WHERE_REOPEN_IDX; }else{ pCov = 0; } @@ -3553,6 +3762,10 @@ static Bitmask codeOneLoopStart( } } +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + pLevel->addrVisit = sqlite3VdbeCurrentAddr(v); +#endif + /* Insert code to test every subexpression that can be completely ** computed using the current set of tables. */ @@ -3634,21 +3847,26 @@ static Bitmask codeOneLoopStart( return pLevel->notReady; } -#if defined(WHERETRACE_ENABLED) && defined(SQLITE_ENABLE_TREE_EXPLAIN) +#ifdef WHERETRACE_ENABLED /* -** Generate "Explanation" text for a WhereTerm. +** Print the content of a WhereTerm object */ -static void whereExplainTerm(Vdbe *v, WhereTerm *pTerm){ - char zType[4]; - memcpy(zType, "...", 4); - if( pTerm->wtFlags & TERM_VIRTUAL ) zType[0] = 'V'; - if( pTerm->eOperator & WO_EQUIV ) zType[1] = 'E'; - if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) zType[2] = 'L'; - sqlite3ExplainPrintf(v, "%s ", zType); - sqlite3ExplainExpr(v, pTerm->pExpr); +static void whereTermPrint(WhereTerm *pTerm, int iTerm){ + if( pTerm==0 ){ + sqlite3DebugPrintf("TERM-%-3d NULL\n", iTerm); + }else{ + char zType[4]; + memcpy(zType, "...", 4); + if( pTerm->wtFlags & TERM_VIRTUAL ) zType[0] = 'V'; + if( pTerm->eOperator & WO_EQUIV ) zType[1] = 'E'; + if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) zType[2] = 'L'; + sqlite3DebugPrintf("TERM-%-3d %p %s cursor=%-3d prob=%-3d op=0x%03x\n", + iTerm, pTerm, zType, pTerm->leftCursor, pTerm->truthProb, + pTerm->eOperator); + sqlite3TreeViewExpr(0, pTerm->pExpr, 0); + } } -#endif /* WHERETRACE_ENABLED && SQLITE_ENABLE_TREE_EXPLAIN */ - +#endif #ifdef WHERETRACE_ENABLED /* @@ -3664,8 +3882,8 @@ static void whereLoopPrint(WhereLoop *p, WhereClause *pWC){ sqlite3DebugPrintf(" %12s", pItem->zAlias ? pItem->zAlias : pTab->zName); if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ - const char *zName; - if( p->u.btree.pIndex && (zName = p->u.btree.pIndex->zName)!=0 ){ + const char *zName; + if( p->u.btree.pIndex && (zName = p->u.btree.pIndex->zName)!=0 ){ if( strncmp(zName, "sqlite_autoindex_", 17)==0 ){ int i = sqlite3Strlen30(zName) - 1; while( zName[i]!='_' ) i--; @@ -3686,29 +3904,18 @@ static void whereLoopPrint(WhereLoop *p, WhereClause *pWC){ sqlite3DebugPrintf(" %-19s", z); sqlite3_free(z); } - sqlite3DebugPrintf(" f %05x N %d", p->wsFlags, p->nLTerm); - sqlite3DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut); -#ifdef SQLITE_ENABLE_TREE_EXPLAIN - /* If the 0x100 bit of wheretracing is set, then show all of the constraint - ** expressions in the WhereLoop.aLTerm[] array. - */ - if( p->nLTerm && (sqlite3WhereTrace & 0x100)!=0 ){ /* WHERETRACE 0x100 */ - int i; - Vdbe *v = pWInfo->pParse->pVdbe; - sqlite3ExplainBegin(v); - for(i=0; inLTerm; i++){ - WhereTerm *pTerm = p->aLTerm[i]; - if( pTerm==0 ) continue; - sqlite3ExplainPrintf(v, " (%d) #%-2d ", i+1, (int)(pTerm-pWC->a)); - sqlite3ExplainPush(v); - whereExplainTerm(v, pTerm); - sqlite3ExplainPop(v); - sqlite3ExplainNL(v); - } - sqlite3ExplainFinish(v); - sqlite3DebugPrintf("%s", sqlite3VdbeExplanation(v)); + if( p->wsFlags & WHERE_SKIPSCAN ){ + sqlite3DebugPrintf(" f %05x %d-%d", p->wsFlags, p->nLTerm,p->nSkip); + }else{ + sqlite3DebugPrintf(" f %05x N %d", p->wsFlags, p->nLTerm); + } + sqlite3DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut); + if( p->nLTerm && (sqlite3WhereTrace & 0x100)!=0 ){ + int i; + for(i=0; inLTerm; i++){ + whereTermPrint(p->aLTerm[i], i); + } } -#endif } #endif @@ -3734,7 +3941,6 @@ static void whereLoopClearUnion(sqlite3 *db, WhereLoop *p){ p->u.vtab.idxStr = 0; }else if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 && p->u.btree.pIndex!=0 ){ sqlite3DbFree(db, p->u.btree.pIndex->zColAff); - sqlite3KeyInfoUnref(p->u.btree.pIndex->pKeyInfo); sqlite3DbFree(db, p->u.btree.pIndex); p->u.btree.pIndex = 0; } @@ -3809,10 +4015,11 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ } /* -** Return TRUE if both of the following are true: +** Return TRUE if all of the following are true: ** ** (1) X has the same or lower cost that Y ** (2) X is a proper subset of Y +** (3) X skips at least as many columns as Y ** ** By "proper subset" we mean that X uses fewer WHERE clause terms ** than Y and that every WHERE clause term used by X is also used @@ -3820,19 +4027,25 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ ** ** If X is a proper subset of Y then Y is a better choice and ought ** to have a lower cost. This routine returns TRUE when that cost -** relationship is inverted and needs to be adjusted. +** relationship is inverted and needs to be adjusted. The third rule +** was added because if X uses skip-scan less than Y it still might +** deserve a lower cost even if it is a proper subset of Y. */ static int whereLoopCheaperProperSubset( const WhereLoop *pX, /* First WhereLoop to compare */ const WhereLoop *pY /* Compare against this WhereLoop */ ){ int i, j; - if( pX->nLTerm >= pY->nLTerm ) return 0; /* X is not a subset of Y */ + if( pX->nLTerm-pX->nSkip >= pY->nLTerm-pY->nSkip ){ + return 0; /* X is not a subset of Y */ + } + if( pY->nSkip > pX->nSkip ) return 0; if( pX->rRun >= pY->rRun ){ if( pX->rRun > pY->rRun ) return 0; /* X costs more than Y */ if( pX->nOut > pY->nOut ) return 0; /* X costs more than Y */ } for(i=pX->nLTerm-1; i>=0; i--){ + if( pX->aLTerm[i]==0 ) continue; for(j=pY->nLTerm-1; j>=0; j--){ if( pY->aLTerm[j]==pX->aLTerm[i] ) break; } @@ -3854,33 +4067,24 @@ static int whereLoopCheaperProperSubset( ** To say "WhereLoop X is a proper subset of Y" means that X uses fewer ** WHERE clause terms than Y and that every WHERE clause term used by X is ** also used by Y. -** -** This adjustment is omitted for SKIPSCAN loops. In a SKIPSCAN loop, the -** WhereLoop.nLTerm field is not an accurate measure of the number of WHERE -** clause terms covered, since some of the first nLTerm entries in aLTerm[] -** will be NULL (because they are skipped). That makes it more difficult -** to compare the loops. We could add extra code to do the comparison, and -** perhaps we will someday. But SKIPSCAN is sufficiently uncommon, and this -** adjustment is sufficient minor, that it is very difficult to construct -** a test case where the extra code would improve the query plan. Better -** to avoid the added complexity and just omit cost adjustments to SKIPSCAN -** loops. */ static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ if( (pTemplate->wsFlags & WHERE_INDEXED)==0 ) return; - if( (pTemplate->wsFlags & WHERE_SKIPSCAN)!=0 ) return; for(; p; p=p->pNextLoop){ if( p->iTab!=pTemplate->iTab ) continue; if( (p->wsFlags & WHERE_INDEXED)==0 ) continue; - if( (p->wsFlags & WHERE_SKIPSCAN)!=0 ) continue; if( whereLoopCheaperProperSubset(p, pTemplate) ){ /* Adjust pTemplate cost downward so that it is cheaper than its - ** subset p */ + ** subset p. */ + WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n", + pTemplate->rRun, pTemplate->nOut, p->rRun, p->nOut-1)); pTemplate->rRun = p->rRun; pTemplate->nOut = p->nOut - 1; }else if( whereLoopCheaperProperSubset(pTemplate, p) ){ /* Adjust pTemplate cost upward so that it is costlier than p since ** pTemplate is a proper subset of p */ + WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n", + pTemplate->rRun, pTemplate->nOut, p->rRun, p->nOut+1)); pTemplate->rRun = p->rRun; pTemplate->nOut = p->nOut + 1; } @@ -3925,8 +4129,9 @@ static WhereLoop **whereLoopFindLesser( /* Any loop using an appliation-defined index (or PRIMARY KEY or ** UNIQUE constraint) with one or more == constraints is better - ** than an automatic index. */ + ** than an automatic index. Unless it is a skip-scan. */ if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 + && (pTemplate->nSkip)==0 && (pTemplate->wsFlags & WHERE_INDEXED)!=0 && (pTemplate->wsFlags & WHERE_COLUMN_EQ)!=0 && (p->prereq & pTemplate->prereq)==pTemplate->prereq @@ -4021,7 +4226,7 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ ** than pTemplate, so just ignore pTemplate */ #if WHERETRACE_ENABLED /* 0x8 */ if( sqlite3WhereTrace & 0x8 ){ - sqlite3DebugPrintf("ins-noop: "); + sqlite3DebugPrintf(" skip: "); whereLoopPrint(pTemplate, pBuilder->pWC); } #endif @@ -4037,10 +4242,10 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ #if WHERETRACE_ENABLED /* 0x8 */ if( sqlite3WhereTrace & 0x8 ){ if( p!=0 ){ - sqlite3DebugPrintf("ins-del: "); + sqlite3DebugPrintf("replace: "); whereLoopPrint(p, pBuilder->pWC); } - sqlite3DebugPrintf("ins-new: "); + sqlite3DebugPrintf(" add: "); whereLoopPrint(pTemplate, pBuilder->pWC); } #endif @@ -4064,7 +4269,7 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ *ppTail = pToDel->pNextLoop; #if WHERETRACE_ENABLED /* 0x8 */ if( sqlite3WhereTrace & 0x8 ){ - sqlite3DebugPrintf("ins-del: "); + sqlite3DebugPrintf(" delete: "); whereLoopPrint(pToDel, pBuilder->pWC); } #endif @@ -4085,19 +4290,42 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ ** Adjust the WhereLoop.nOut value downward to account for terms of the ** WHERE clause that reference the loop but which are not used by an ** index. +* +** For every WHERE clause term that is not used by the index +** and which has a truth probability assigned by one of the likelihood(), +** likely(), or unlikely() SQL functions, reduce the estimated number +** of output rows by the probability specified. ** -** In the current implementation, the first extra WHERE clause term reduces -** the number of output rows by a factor of 10 and each additional term -** reduces the number of output rows by sqrt(2). +** TUNING: For every WHERE clause term that is not used by the index +** and which does not have an assigned truth probability, heuristics +** described below are used to try to estimate the truth probability. +** TODO --> Perhaps this is something that could be improved by better +** table statistics. +** +** Heuristic 1: Estimate the truth probability as 93.75%. The 93.75% +** value corresponds to -1 in LogEst notation, so this means decrement +** the WhereLoop.nOut field for every such WHERE clause term. +** +** Heuristic 2: If there exists one or more WHERE clause terms of the +** form "x==EXPR" and EXPR is not a constant 0 or 1, then make sure the +** final output row estimate is no greater than 1/4 of the total number +** of rows in the table. In other words, assume that x==EXPR will filter +** out at least 3 out of 4 rows. If EXPR is -1 or 0 or 1, then maybe the +** "x" column is boolean or else -1 or 0 or 1 is a common default value +** on the "x" column and so in that case only cap the output row estimate +** at 1/2 instead of 1/4. */ -static void whereLoopOutputAdjust(WhereClause *pWC, WhereLoop *pLoop){ +static void whereLoopOutputAdjust( + WhereClause *pWC, /* The WHERE clause */ + WhereLoop *pLoop, /* The loop to adjust downward */ + LogEst nRow /* Number of rows in the entire table */ +){ WhereTerm *pTerm, *pX; Bitmask notAllowed = ~(pLoop->prereq|pLoop->maskSelf); - int i, j; + int i, j, k; + LogEst iReduce = 0; /* pLoop->nOut should not exceed nRow-iReduce */ - if( !OptimizationEnabled(pWC->pWInfo->pParse->db, SQLITE_AdjustOutEst) ){ - return; - } + assert( (pLoop->wsFlags & WHERE_AUTO_INDEX)==0 ); for(i=pWC->nTerm, pTerm=pWC->a; i>0; i--, pTerm++){ if( (pTerm->wtFlags & TERM_VIRTUAL)!=0 ) break; if( (pTerm->prereqAll & pLoop->maskSelf)==0 ) continue; @@ -4109,11 +4337,39 @@ static void whereLoopOutputAdjust(WhereClause *pWC, WhereLoop *pLoop){ if( pX->iParent>=0 && (&pWC->a[pX->iParent])==pTerm ) break; } if( j<0 ){ - pLoop->nOut += (pTerm->truthProb<=0 ? pTerm->truthProb : -1); + if( pTerm->truthProb<=0 ){ + /* If a truth probability is specified using the likelihood() hints, + ** then use the probability provided by the application. */ + pLoop->nOut += pTerm->truthProb; + }else{ + /* In the absence of explicit truth probabilities, use heuristics to + ** guess a reasonable truth probability. */ + pLoop->nOut--; + if( pTerm->eOperator&WO_EQ ){ + Expr *pRight = pTerm->pExpr->pRight; + if( sqlite3ExprIsInteger(pRight, &k) && k>=(-1) && k<=1 ){ + k = 10; + }else{ + k = 20; + } + if( iReducenOut > nRow-iReduce ) pLoop->nOut = nRow - iReduce; } +/* +** Adjust the cost C by the costMult facter T. This only occurs if +** compiled with -DSQLITE_ENABLE_COSTMULT +*/ +#ifdef SQLITE_ENABLE_COSTMULT +# define ApplyCostMultiplier(C,T) C += T +#else +# define ApplyCostMultiplier(C,T) +#endif + /* ** We have so far matched pBuilder->pNew->u.btree.nEq terms of the ** index pIndex. Try to match one more. @@ -4142,11 +4398,12 @@ static int whereLoopAddBtreeIndex( Bitmask saved_prereq; /* Original value of pNew->prereq */ u16 saved_nLTerm; /* Original value of pNew->nLTerm */ u16 saved_nEq; /* Original value of pNew->u.btree.nEq */ - u16 saved_nSkip; /* Original value of pNew->u.btree.nSkip */ + u16 saved_nSkip; /* Original value of pNew->nSkip */ u32 saved_wsFlags; /* Original value of pNew->wsFlags */ LogEst saved_nOut; /* Original value of pNew->nOut */ int iCol; /* Index of the column in the table */ int rc = SQLITE_OK; /* Return code */ + LogEst rSize; /* Number of rows in the table */ LogEst rLogSize; /* Logarithm of table size */ WhereTerm *pTop = 0, *pBtm = 0; /* Top and bottom range constraints */ @@ -4164,50 +4421,20 @@ static int whereLoopAddBtreeIndex( } if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE); - assert( pNew->u.btree.nEq<=pProbe->nKeyCol ); - if( pNew->u.btree.nEq < pProbe->nKeyCol ){ - iCol = pProbe->aiColumn[pNew->u.btree.nEq]; - }else{ - iCol = -1; - } + assert( pNew->u.btree.nEqnColumn ); + iCol = pProbe->aiColumn[pNew->u.btree.nEq]; + pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, iCol, opMask, pProbe); saved_nEq = pNew->u.btree.nEq; - saved_nSkip = pNew->u.btree.nSkip; + saved_nSkip = pNew->nSkip; saved_nLTerm = pNew->nLTerm; saved_wsFlags = pNew->wsFlags; saved_prereq = pNew->prereq; saved_nOut = pNew->nOut; pNew->rSetup = 0; - rLogSize = estLog(pProbe->aiRowLogEst[0]); - - /* Consider using a skip-scan if there are no WHERE clause constraints - ** available for the left-most terms of the index, and if the average - ** number of repeats in the left-most terms is at least 18. - ** - ** The magic number 18 is selected on the basis that scanning 17 rows - ** is almost always quicker than an index seek (even though if the index - ** contains fewer than 2^17 rows we assume otherwise in other parts of - ** the code). And, even if it is not, it should not be too much slower. - ** On the other hand, the extra seeks could end up being significantly - ** more expensive. */ - assert( 42==sqlite3LogEst(18) ); - if( pTerm==0 - && saved_nEq==saved_nSkip - && saved_nEq+1nKeyCol - && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */ - && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK - ){ - LogEst nIter; - pNew->u.btree.nEq++; - pNew->u.btree.nSkip++; - pNew->aLTerm[pNew->nLTerm++] = 0; - pNew->wsFlags |= WHERE_SKIPSCAN; - nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1]; - pNew->nOut -= nIter; - whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul); - pNew->nOut = saved_nOut; - } + rSize = pProbe->aiRowLogEst[0]; + rLogSize = estLog(rSize); for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ u16 eOp = pTerm->eOperator; /* Shorthand for pTerm->eOperator */ LogEst rCostIdx; @@ -4252,7 +4479,7 @@ static int whereLoopAddBtreeIndex( }else if( eOp & (WO_EQ) ){ pNew->wsFlags |= WHERE_COLUMN_EQ; if( iCol<0 || (nInMul==0 && pNew->u.btree.nEq==pProbe->nKeyCol-1) ){ - if( iCol>=0 && pProbe->onError==OE_None ){ + if( iCol>=0 && !IsUniqueIndex(pProbe) ){ pNew->wsFlags |= WHERE_UNQ_WANTED; }else{ pNew->wsFlags |= WHERE_ONEROW; @@ -4302,7 +4529,6 @@ static int whereLoopAddBtreeIndex( if( nInMul==0 && pProbe->nSample && pNew->u.btree.nEq<=pProbe->nSampleCol - && OptimizationEnabled(db, SQLITE_Stat3) && ((eOp & WO_IN)==0 || !ExprHasProperty(pTerm->pExpr, EP_xIsSelect)) ){ Expr *pExpr = pTerm->pExpr; @@ -4313,7 +4539,6 @@ static int whereLoopAddBtreeIndex( }else{ rc = whereInScanEst(pParse, pBuilder, pExpr->x.pList, &nOut); } - assert( rc!=SQLITE_OK || nOut>0 ); if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK; if( rc!=SQLITE_OK ) break; /* Jump out of the pTerm loop */ if( nOut ){ @@ -4345,11 +4570,12 @@ static int whereLoopAddBtreeIndex( if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){ pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16); } + ApplyCostMultiplier(pNew->rRun, pProbe->pTable->costMult); nOutUnadjusted = pNew->nOut; pNew->rRun += nInMul + nIn; pNew->nOut += nInMul + nIn; - whereLoopOutputAdjust(pBuilder->pWC, pNew); + whereLoopOutputAdjust(pBuilder->pWC, pNew, rSize); rc = whereLoopInsert(pBuilder, pNew); if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ @@ -4359,7 +4585,7 @@ static int whereLoopAddBtreeIndex( } if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 - && pNew->u.btree.nEq<(pProbe->nKeyCol + (pProbe->zName!=0)) + && pNew->u.btree.nEqnColumn ){ whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul+nIn); } @@ -4370,10 +4596,45 @@ static int whereLoopAddBtreeIndex( } pNew->prereq = saved_prereq; pNew->u.btree.nEq = saved_nEq; - pNew->u.btree.nSkip = saved_nSkip; + pNew->nSkip = saved_nSkip; pNew->wsFlags = saved_wsFlags; pNew->nOut = saved_nOut; pNew->nLTerm = saved_nLTerm; + + /* Consider using a skip-scan if there are no WHERE clause constraints + ** available for the left-most terms of the index, and if the average + ** number of repeats in the left-most terms is at least 18. + ** + ** The magic number 18 is selected on the basis that scanning 17 rows + ** is almost always quicker than an index seek (even though if the index + ** contains fewer than 2^17 rows we assume otherwise in other parts of + ** the code). And, even if it is not, it should not be too much slower. + ** On the other hand, the extra seeks could end up being significantly + ** more expensive. */ + assert( 42==sqlite3LogEst(18) ); + if( saved_nEq==saved_nSkip + && saved_nEq+1nKeyCol + && pProbe->noSkipScan==0 + && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */ + && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK + ){ + LogEst nIter; + pNew->u.btree.nEq++; + pNew->nSkip++; + pNew->aLTerm[pNew->nLTerm++] = 0; + pNew->wsFlags |= WHERE_SKIPSCAN; + nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1]; + pNew->nOut -= nIter; + /* TUNING: Because uncertainties in the estimates for skip-scan queries, + ** add a 1.375 fudge factor to make skip-scan slightly less likely. */ + nIter += 5; + whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul); + pNew->nOut = saved_nOut; + pNew->u.btree.nEq = saved_nEq; + pNew->nSkip = saved_nSkip; + pNew->wsFlags = saved_wsFlags; + } + return rc; } @@ -4399,6 +4660,7 @@ static int indexMightHelpWithOrderBy( Expr *pExpr = sqlite3ExprSkipCollate(pOB->a[ii].pExpr); if( pExpr->op!=TK_COLUMN ) return 0; if( pExpr->iTable==iCursor ){ + if( pExpr->iColumn<0 ) return 1; for(jj=0; jjnKeyCol; jj++){ if( pExpr->iColumn==pIndex->aiColumn[jj] ) return 1; } @@ -4464,6 +4726,14 @@ static int whereUsablePartialIndex(int iTab, WhereClause *pWC, Expr *pWhere){ ** Normally, nSeek is 1. nSeek values greater than 1 come about if the ** WHERE clause includes "x IN (....)" terms used in place of "x=?". Or when ** implicit "x IN (SELECT x FROM tbl)" terms are added for skip-scans. +** +** The estimated values (nRow, nVisit, nSeek) often contain a large amount +** of uncertainty. For this reason, scoring is designed to pick plans that +** "do the least harm" if the estimates are inaccurate. For example, a +** log(nRow) factor is omitted from a non-covering index scan in order to +** bias the scoring in favor of using an index, since the worst-case +** performance of using an index is far better than the worst-case performance +** of a full table scan. */ static int whereLoopAddBtree( WhereLoopBuilder *pBuilder, /* WHERE clause information */ @@ -4506,6 +4776,7 @@ static int whereLoopAddBtree( Index *pFirst; /* First of real indices on the table */ memset(&sPk, 0, sizeof(Index)); sPk.nKeyCol = 1; + sPk.nColumn = 1; sPk.aiColumn = &aiColumnPk; sPk.aiRowLogEst = aiRowEstPk; sPk.onError = OE_Replace; @@ -4542,17 +4813,26 @@ static int whereLoopAddBtree( if( pTerm->prereqRight & pNew->maskSelf ) continue; if( termCanDriveIndex(pTerm, pSrc, 0) ){ pNew->u.btree.nEq = 1; - pNew->u.btree.nSkip = 0; + pNew->nSkip = 0; pNew->u.btree.pIndex = 0; pNew->nLTerm = 1; pNew->aLTerm[0] = pTerm; /* TUNING: One-time cost for computing the automatic index is - ** approximately 7*N*log2(N) where N is the number of rows in - ** the table being indexed. */ - pNew->rSetup = rLogSize + rSize + 28; assert( 28==sqlite3LogEst(7) ); + ** estimated to be X*N*log2(N) where N is the number of rows in + ** the table being indexed and where X is 7 (LogEst=28) for normal + ** tables or 1.375 (LogEst=4) for views and subqueries. The value + ** of X is smaller for views and subqueries so that the query planner + ** will be more aggressive about generating automatic indexes for + ** those objects, since there is no opportunity to add schema + ** indexes on subqueries and views. */ + pNew->rSetup = rLogSize + rSize + 4; + if( pTab->pSelect==0 && (pTab->tabFlags & TF_Ephemeral)==0 ){ + pNew->rSetup += 24; + } + ApplyCostMultiplier(pNew->rSetup, pTab->costMult); /* TUNING: Each index lookup yields 20 rows in the table. This ** is more than the usual guess of 10 rows, since we have no way - ** of knowning how selective the index will ultimately be. It would + ** of knowing how selective the index will ultimately be. It would ** not be unreasonable to make this value much larger. */ pNew->nOut = 43; assert( 43==sqlite3LogEst(20) ); pNew->rRun = sqlite3LogEstAdd(rLogSize,pNew->nOut); @@ -4568,12 +4848,13 @@ static int whereLoopAddBtree( */ for(; rc==SQLITE_OK && pProbe; pProbe=pProbe->pNext, iSortIdx++){ if( pProbe->pPartIdxWhere!=0 - && !whereUsablePartialIndex(pNew->iTab, pWC, pProbe->pPartIdxWhere) ){ + && !whereUsablePartialIndex(pSrc->iCursor, pWC, pProbe->pPartIdxWhere) ){ + testcase( pNew->iTab!=pSrc->iCursor ); /* See ticket [98d973b8f5] */ continue; /* Partial index inappropriate for this query */ } rSize = pProbe->aiRowLogEst[0]; pNew->u.btree.nEq = 0; - pNew->u.btree.nSkip = 0; + pNew->nSkip = 0; pNew->nLTerm = 0; pNew->iSortIdx = 0; pNew->rSetup = 0; @@ -4591,7 +4872,8 @@ static int whereLoopAddBtree( pNew->iSortIdx = b ? iSortIdx : 0; /* TUNING: Cost of full table scan is (N*3.0). */ pNew->rRun = rSize + 16; - whereLoopOutputAdjust(pWC, pNew); + ApplyCostMultiplier(pNew->rRun, pTab->costMult); + whereLoopOutputAdjust(pWC, pNew, rSize); rc = whereLoopInsert(pBuilder, pNew); pNew->nOut = rSize; if( rc ) break; @@ -4626,8 +4908,8 @@ static int whereLoopAddBtree( if( m!=0 ){ pNew->rRun = sqlite3LogEstAdd(pNew->rRun, rSize+16); } - - whereLoopOutputAdjust(pWC, pNew); + ApplyCostMultiplier(pNew->rRun, pTab->costMult); + whereLoopOutputAdjust(pWC, pNew, rSize); rc = whereLoopInsert(pBuilder, pNew); pNew->nOut = rSize; if( rc ) break; @@ -4834,7 +5116,6 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ struct SrcList_item *pItem; pWC = pBuilder->pWC; - if( pWInfo->wctrlFlags & WHERE_AND_ONLY ) return SQLITE_OK; pWCEnd = pWC->a + pWC->nTerm; pNew = pBuilder->pNew; memset(&sSum, 0, sizeof(sSum)); @@ -4855,6 +5136,7 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ sSubBuild.pOrderBy = 0; sSubBuild.pOrSet = &sCur; + WHERETRACE(0x200, ("Begin processing OR-clause %p\n", pTerm)); for(pOrTerm=pOrWC->a; pOrTermeOperator & WO_AND)!=0 ){ sSubBuild.pWC = &pOrTerm->u.pAndInfo->wc; @@ -4869,6 +5151,15 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ continue; } sCur.n = 0; +#ifdef WHERETRACE_ENABLED + WHERETRACE(0x200, ("OR-term %d of %p has %d subterms:\n", + (int)(pOrTerm-pOrWC->a), pTerm, sSubBuild.pWC->nTerm)); + if( sqlite3WhereTrace & 0x400 ){ + for(i=0; inTerm; i++){ + whereTermPrint(&sSubBuild.pWC->a[i], i); + } + } +#endif #ifndef SQLITE_OMIT_VIRTUALTABLE if( IsVirtual(pItem->pTab) ){ rc = whereLoopAddVirtual(&sSubBuild, mExtra); @@ -4877,6 +5168,9 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ { rc = whereLoopAddBtree(&sSubBuild, mExtra); } + if( rc==SQLITE_OK ){ + rc = whereLoopAddOr(&sSubBuild, mExtra); + } assert( rc==SQLITE_OK || sCur.n==0 ); if( sCur.n==0 ){ sSum.n = 0; @@ -4921,6 +5215,7 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ pNew->prereq = sSum.a[i].prereq; rc = whereLoopInsert(pBuilder, pNew); } + WHERETRACE(0x200, ("End processing OR-clause %p\n", pTerm)); } } return rc; @@ -4980,7 +5275,7 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ ** strict. With GROUP BY and DISTINCT the only requirement is that ** equivalent rows appear immediately adjacent to one another. GROUP BY ** and DISTINCT do not require rows to appear in any particular order as long -** as equivelent rows are grouped together. Thus for GROUP BY and DISTINCT +** as equivalent rows are grouped together. Thus for GROUP BY and DISTINCT ** the pOrderBy terms can be matched in any order. With ORDER BY, the ** pOrderBy terms must be matched in strict left-to-right order. */ @@ -5096,7 +5391,7 @@ static i8 wherePathSatisfiesOrderBy( nColumn = pIndex->nColumn; assert( nColumn==nKeyCol+1 || !HasRowid(pIndex->pTable) ); assert( pIndex->aiColumn[nColumn-1]==(-1) || !HasRowid(pIndex->pTable)); - isOrderDistinct = pIndex->onError!=OE_None; + isOrderDistinct = IsUniqueIndex(pIndex); } /* Loop through all columns of the index and deal with the ones @@ -5109,7 +5404,7 @@ static i8 wherePathSatisfiesOrderBy( /* Skip over == and IS NULL terms */ if( ju.btree.nEq - && pLoop->u.btree.nSkip==0 + && pLoop->nSkip==0 && ((i = pLoop->aLTerm[j]->eOperator) & (WO_EQ|WO_ISNULL))!=0 ){ if( i & WO_ISNULL ){ @@ -5164,7 +5459,7 @@ static i8 wherePathSatisfiesOrderBy( isMatch = 1; break; } - if( isMatch && (pWInfo->wctrlFlags & WHERE_GROUPBY)==0 ){ + if( isMatch && (wctrlFlags & WHERE_GROUPBY)==0 ){ /* Make sure the sort order is compatible in an ORDER BY clause. ** Sort order is irrelevant for a GROUP BY clause. */ if( revSet ){ @@ -5265,6 +5560,45 @@ static const char *wherePathName(WherePath *pPath, int nLoop, WhereLoop *pLast){ } #endif +/* +** Return the cost of sorting nRow rows, assuming that the keys have +** nOrderby columns and that the first nSorted columns are already in +** order. +*/ +static LogEst whereSortingCost( + WhereInfo *pWInfo, + LogEst nRow, + int nOrderBy, + int nSorted +){ + /* TUNING: Estimated cost of a full external sort, where N is + ** the number of rows to sort is: + ** + ** cost = (3.0 * N * log(N)). + ** + ** Or, if the order-by clause has X terms but only the last Y + ** terms are out of order, then block-sorting will reduce the + ** sorting cost to: + ** + ** cost = (3.0 * N * log(N)) * (Y/X) + ** + ** The (Y/X) term is implemented using stack variable rScale + ** below. */ + LogEst rScale, rSortCost; + assert( nOrderBy>0 && 66==sqlite3LogEst(100) ); + rScale = sqlite3LogEst((nOrderBy-nSorted)*100/nOrderBy) - 66; + rSortCost = nRow + estLog(nRow) + rScale + 16; + + /* TUNING: The cost of implementing DISTINCT using a B-TREE is + ** similar but with a larger constant of proportionality. + ** Multiply by an additional factor of 3.0. */ + if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){ + rSortCost += 16; + } + + return rSortCost; +} + /* ** Given the list of WhereLoop objects at pWInfo->pLoops, this routine ** attempts to find the lowest cost path that visits each WhereLoop @@ -5286,9 +5620,8 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ int ii, jj; /* Loop counters */ int mxI = 0; /* Index of next entry to replace */ int nOrderBy; /* Number of ORDER BY clause terms */ - LogEst rCost; /* Cost of a path */ - LogEst nOut; /* Number of outputs */ LogEst mxCost = 0; /* Maximum cost of a set of paths */ + LogEst mxUnsorted = 0; /* Maximum unsorted cost of a set of path */ int nTo, nFrom; /* Number of valid entries in aTo[] and aFrom[] */ WherePath *aFrom; /* All nFrom paths at the previous level */ WherePath *aTo; /* The nTo best paths at the current level */ @@ -5296,7 +5629,9 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ WherePath *pTo; /* An element of aTo[] that we are working on */ WhereLoop *pWLoop; /* One of the WhereLoop objects */ WhereLoop **pX; /* Used to divy up the pSpace memory */ + LogEst *aSortCost = 0; /* Sorting and partial sorting costs */ char *pSpace; /* Temporary memory used by this routine */ + int nSpace; /* Bytes of space allocated at pSpace */ pParse = pWInfo->pParse; db = pParse->db; @@ -5306,11 +5641,23 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ ** For joins of 3 or more tables, track the 10 best paths */ mxChoice = (nLoop<=1) ? 1 : (nLoop==2 ? 5 : 10); assert( nLoop<=pWInfo->pTabList->nSrc ); - WHERETRACE(0x002, ("---- begin solver\n")); + WHERETRACE(0x002, ("---- begin solver. (nRowEst=%d)\n", nRowEst)); - /* Allocate and initialize space for aTo and aFrom */ - ii = (sizeof(WherePath)+sizeof(WhereLoop*)*nLoop)*mxChoice*2; - pSpace = sqlite3DbMallocRaw(db, ii); + /* If nRowEst is zero and there is an ORDER BY clause, ignore it. In this + ** case the purpose of this call is to estimate the number of rows returned + ** by the overall query. Once this estimate has been obtained, the caller + ** will invoke this function a second time, passing the estimate as the + ** nRowEst parameter. */ + if( pWInfo->pOrderBy==0 || nRowEst==0 ){ + nOrderBy = 0; + }else{ + nOrderBy = pWInfo->pOrderBy->nExpr; + } + + /* Allocate and initialize space for aTo, aFrom and aSortCost[] */ + nSpace = (sizeof(WherePath)+sizeof(WhereLoop*)*nLoop)*mxChoice*2; + nSpace += sizeof(LogEst) * nOrderBy; + pSpace = sqlite3DbMallocRaw(db, nSpace); if( pSpace==0 ) return SQLITE_NOMEM; aTo = (WherePath*)pSpace; aFrom = aTo+mxChoice; @@ -5319,6 +5666,18 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ for(ii=mxChoice*2, pFrom=aTo; ii>0; ii--, pFrom++, pX += nLoop){ pFrom->aLoop = pX; } + if( nOrderBy ){ + /* If there is an ORDER BY clause and it is not being ignored, set up + ** space for the aSortCost[] array. Each element of the aSortCost array + ** is either zero - meaning it has not yet been initialized - or the + ** cost of sorting nRowEst rows of data where the first X terms of + ** the ORDER BY clause are already in order, where X is the array + ** index. */ + aSortCost = (LogEst*)pX; + memset(aSortCost, 0, sizeof(LogEst) * nOrderBy); + } + assert( aSortCost==0 || &pSpace[nSpace]==(char*)&aSortCost[nOrderBy] ); + assert( aSortCost!=0 || &pSpace[nSpace]==(char*)pX ); /* Seed the search with a single WherePath containing zero WhereLoops. ** @@ -5327,15 +5686,15 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ ** rows, then do not use the automatic index. */ aFrom[0].nRow = MIN(pParse->nQueryLoop, 46); assert( 46==sqlite3LogEst(25) ); nFrom = 1; - - /* Precompute the cost of sorting the final result set, if the caller - ** to sqlite3WhereBegin() was concerned about sorting */ - if( pWInfo->pOrderBy==0 || nRowEst==0 ){ - aFrom[0].isOrdered = 0; - nOrderBy = 0; - }else{ - aFrom[0].isOrdered = nLoop>0 ? -1 : 1; - nOrderBy = pWInfo->pOrderBy->nExpr; + assert( aFrom[0].isOrdered==0 ); + if( nOrderBy ){ + /* If nLoop is zero, then there are no FROM terms in the query. Since + ** in this case the query may return a maximum of one row, the results + ** are already in the requested order. Set isOrdered to nOrderBy to + ** indicate this. Or, if nLoop is greater than zero, set isOrdered to + ** -1, indicating that the result set may or may not be ordered, + ** depending on the loops added to the current plan. */ + aFrom[0].isOrdered = nLoop>0 ? -1 : nOrderBy; } /* Compute successively longer WherePaths using the previous generation @@ -5345,66 +5704,71 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ nTo = 0; for(ii=0, pFrom=aFrom; iipLoops; pWLoop; pWLoop=pWLoop->pNextLoop){ - Bitmask maskNew; - Bitmask revMask = 0; - i8 isOrdered = pFrom->isOrdered; + LogEst nOut; /* Rows visited by (pFrom+pWLoop) */ + LogEst rCost; /* Cost of path (pFrom+pWLoop) */ + LogEst rUnsorted; /* Unsorted cost of (pFrom+pWLoop) */ + i8 isOrdered = pFrom->isOrdered; /* isOrdered for (pFrom+pWLoop) */ + Bitmask maskNew; /* Mask of src visited by (..) */ + Bitmask revMask = 0; /* Mask of rev-order loops for (..) */ + if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue; if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue; /* At this point, pWLoop is a candidate to be the next loop. ** Compute its cost */ - rCost = sqlite3LogEstAdd(pWLoop->rSetup,pWLoop->rRun + pFrom->nRow); - rCost = sqlite3LogEstAdd(rCost, pFrom->rCost); + rUnsorted = sqlite3LogEstAdd(pWLoop->rSetup,pWLoop->rRun + pFrom->nRow); + rUnsorted = sqlite3LogEstAdd(rUnsorted, pFrom->rUnsorted); nOut = pFrom->nRow + pWLoop->nOut; maskNew = pFrom->maskLoop | pWLoop->maskSelf; if( isOrdered<0 ){ isOrdered = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags, iLoop, pWLoop, &revMask); - if( isOrdered>=0 && isOrdered0 && 66==sqlite3LogEst(100) ); - rScale = sqlite3LogEst((nOrderBy-isOrdered)*100/nOrderBy) - 66; - rSortCost = nRowEst + estLog(nRowEst) + rScale + 16; - - /* TUNING: The cost of implementing DISTINCT using a B-TREE is - ** similar but with a larger constant of proportionality. - ** Multiply by an additional factor of 3.0. */ - if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){ - rSortCost += 16; - } - WHERETRACE(0x002, - ("---- sort cost=%-3d (%d/%d) increases cost %3d to %-3d\n", - rSortCost, (nOrderBy-isOrdered), nOrderBy, rCost, - sqlite3LogEstAdd(rCost,rSortCost))); - rCost = sqlite3LogEstAdd(rCost, rSortCost); - } }else{ revMask = pFrom->revLoop; } - /* Check to see if pWLoop should be added to the mxChoice best so far */ + if( isOrdered>=0 && isOrderedisOrdered^isOrdered)&0x80)==0" is equivalent + ** to (pTo->isOrdered==(-1))==(isOrdered==(-1))" for the range + ** of legal values for isOrdered, -1..64. + */ for(jj=0, pTo=aTo; jjmaskLoop==maskNew - && ((pTo->isOrdered^isOrdered)&80)==0 + && ((pTo->isOrdered^isOrdered)&0x80)==0 ){ testcase( jj==nTo-1 ); break; } } if( jj>=nTo ){ - if( nTo>=mxChoice && rCost>=mxCost ){ + /* None of the existing best-so-far paths match the candidate. */ + if( nTo>=mxChoice + && (rCost>mxCost || (rCost==mxCost && rUnsorted>=mxUnsorted)) + ){ + /* The current candidate is no better than any of the mxChoice + ** paths currently in the best-so-far buffer. So discard + ** this candidate as not viable. */ #ifdef WHERETRACE_ENABLED /* 0x4 */ if( sqlite3WhereTrace&0x4 ){ sqlite3DebugPrintf("Skip %s cost=%-3d,%3d order=%c\n", @@ -5414,7 +5778,8 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ #endif continue; } - /* Add a new Path to the aTo[] set */ + /* If we reach this points it means that the new candidate path + ** needs to be added to the set of best-so-far paths. */ if( nTorCost<=rCost ){ + /* Control reaches here if best-so-far path pTo=aTo[jj] covers the + ** same set of loops and has the sam isOrdered setting as the + ** candidate path. Check to see if the candidate should replace + ** pTo or if the candidate should be skipped */ + if( pTo->rCostrCost==rCost && pTo->nRow<=nOut) ){ #ifdef WHERETRACE_ENABLED /* 0x4 */ if( sqlite3WhereTrace&0x4 ){ sqlite3DebugPrintf( @@ -5443,11 +5812,13 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?'); } #endif + /* Discard the candidate path from further consideration */ testcase( pTo->rCost==rCost ); continue; } testcase( pTo->rCost==rCost+1 ); - /* A new and better score for a previously created equivalent path */ + /* Control reaches here if the candidate path is better than the + ** pTo path. Replace pTo with the candidate. */ #ifdef WHERETRACE_ENABLED /* 0x4 */ if( sqlite3WhereTrace&0x4 ){ sqlite3DebugPrintf( @@ -5465,15 +5836,20 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ pTo->revLoop = revMask; pTo->nRow = nOut; pTo->rCost = rCost; + pTo->rUnsorted = rUnsorted; pTo->isOrdered = isOrdered; memcpy(pTo->aLoop, pFrom->aLoop, sizeof(WhereLoop*)*iLoop); pTo->aLoop[iLoop] = pWLoop; if( nTo>=mxChoice ){ mxI = 0; mxCost = aTo[0].rCost; + mxUnsorted = aTo[0].nRow; for(jj=1, pTo=&aTo[1]; jjrCost>mxCost ){ + if( pTo->rCost>mxCost + || (pTo->rCost==mxCost && pTo->rUnsorted>mxUnsorted) + ){ mxCost = pTo->rCost; + mxUnsorted = pTo->rUnsorted; mxI = jj; } } @@ -5482,7 +5858,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ } #ifdef WHERETRACE_ENABLED /* >=2 */ - if( sqlite3WhereTrace>=2 ){ + if( sqlite3WhereTrace & 0x02 ){ sqlite3DebugPrintf("---- after round %d ----\n", iLoop); for(ii=0, pTo=aTo; iiwctrlFlags & WHERE_SORTBYGROUP) && pWInfo->nOBSat==pWInfo->pOrderBy->nExpr ){ - Bitmask notUsed = 0; + Bitmask revMask = 0; int nOrder = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, - pFrom, 0, nLoop-1, pFrom->aLoop[nLoop-1], ¬Used + pFrom, 0, nLoop-1, pFrom->aLoop[nLoop-1], &revMask ); assert( pWInfo->sorted==0 ); - pWInfo->sorted = (nOrder==pWInfo->pOrderBy->nExpr); + if( nOrder==pWInfo->pOrderBy->nExpr ){ + pWInfo->sorted = 1; + pWInfo->revMask = revMask; + } } } @@ -5598,7 +5977,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ pWC = &pWInfo->sWC; pLoop = pBuilder->pNew; pLoop->wsFlags = 0; - pLoop->u.btree.nSkip = 0; + pLoop->nSkip = 0; pTerm = findTerm(pWC, iCur, -1, 0, WO_EQ, 0); if( pTerm ){ pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_IPK|WHERE_ONEROW; @@ -5610,8 +5989,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ }else{ for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ assert( pLoop->aLTermSpace==pLoop->aLTerm ); - assert( ArraySize(pLoop->aLTermSpace)==4 ); - if( pIdx->onError==OE_None + if( !IsUniqueIndex(pIdx) || pIdx->pPartIdxWhere!=0 || pIdx->nKeyCol>ArraySize(pLoop->aLTermSpace) ) continue; @@ -5906,23 +6284,16 @@ WhereInfo *sqlite3WhereBegin( /* Construct the WhereLoop objects */ WHERETRACE(0xffff,("*** Optimizer Start ***\n")); +#if defined(WHERETRACE_ENABLED) /* Display all terms of the WHERE clause */ -#if defined(WHERETRACE_ENABLED) && defined(SQLITE_ENABLE_TREE_EXPLAIN) if( sqlite3WhereTrace & 0x100 ){ int i; - Vdbe *v = pParse->pVdbe; - sqlite3ExplainBegin(v); for(i=0; inTerm; i++){ - sqlite3ExplainPrintf(v, "#%-2d ", i); - sqlite3ExplainPush(v); - whereExplainTerm(v, &sWLB.pWC->a[i]); - sqlite3ExplainPop(v); - sqlite3ExplainNL(v); + whereTermPrint(&sWLB.pWC->a[i], i); } - sqlite3ExplainFinish(v); - sqlite3DebugPrintf("%s", sqlite3VdbeExplanation(v)); } #endif + if( nTabList!=1 || whereShortCut(&sWLB)==0 ){ rc = whereLoopAddAll(&sWLB); if( rc ) goto whereBeginError; @@ -6101,6 +6472,7 @@ WhereInfo *sqlite3WhereBegin( pWInfo->aiCurOnePass[1] = iIndexCur; }else if( iIdxCur && (wctrlFlags & WHERE_ONETABLE_ONLY)!=0 ){ iIndexCur = iIdxCur; + if( wctrlFlags & WHERE_REOPEN_IDX ) op = OP_ReopenIdx; }else{ iIndexCur = pParse->nTab++; } @@ -6125,7 +6497,10 @@ WhereInfo *sqlite3WhereBegin( */ notReady = ~(Bitmask)0; for(ii=0; iia[ii]; + wsFlags = pLevel->pWLoop->wsFlags; #ifndef SQLITE_OMIT_AUTOMATIC_INDEX if( (pLevel->pWLoop->wsFlags & WHERE_AUTO_INDEX)!=0 ){ constructAutomaticIndex(pParse, &pWInfo->sWC, @@ -6133,10 +6508,15 @@ WhereInfo *sqlite3WhereBegin( if( db->mallocFailed ) goto whereBeginError; } #endif - explainOneScan(pParse, pTabList, pLevel, ii, pLevel->iFrom, wctrlFlags); + addrExplain = explainOneScan( + pParse, pTabList, pLevel, ii, pLevel->iFrom, wctrlFlags + ); pLevel->addrBody = sqlite3VdbeCurrentAddr(v); notReady = codeOneLoopStart(pWInfo, ii, notReady); pWInfo->iContinue = pLevel->addrCont; + if( (wsFlags&WHERE_MULTI_OR)==0 && (wctrlFlags&WHERE_ONETABLE_ONLY)==0 ){ + addScanStatus(v, pTabList, pLevel, addrExplain); + } } /* Done. */ diff --git a/src/whereInt.h b/src/whereInt.h index 72e7530db9..2ccc6ec064 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -85,6 +85,9 @@ struct WhereLevel { } u; struct WhereLoop *pWLoop; /* The selected WhereLoop object */ Bitmask notReady; /* FROM entries not usable at this level */ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int addrVisit; /* Address at which row is visited */ +#endif }; /* @@ -115,7 +118,6 @@ struct WhereLoop { union { struct { /* Information for internal btree tables */ u16 nEq; /* Number of equality constraints */ - u16 nSkip; /* Number of initial index columns to skip */ Index *pIndex; /* Index used, or NULL */ } btree; struct { /* Information for virtual tables */ @@ -128,12 +130,13 @@ struct WhereLoop { } u; u32 wsFlags; /* WHERE_* flags describing the plan */ u16 nLTerm; /* Number of entries in aLTerm[] */ + u16 nSkip; /* Number of NULL aLTerm[] entries */ /**** whereLoopXfer() copies fields above ***********************/ # define WHERE_LOOP_XFER_SZ offsetof(WhereLoop,nLSlot) u16 nLSlot; /* Number of slots allocated for aLTerm[] */ WhereTerm **aLTerm; /* WhereTerms used */ WhereLoop *pNextLoop; /* Next WhereLoop object in the WhereClause */ - WhereTerm *aLTermSpace[4]; /* Initial aLTerm[] space */ + WhereTerm *aLTermSpace[3]; /* Initial aLTerm[] space */ }; /* This object holds the prerequisites and the cost of running a @@ -176,13 +179,14 @@ static int whereLoopResize(sqlite3*, WhereLoop*, int); ** 1. Then using those as a basis to compute the N best WherePath objects ** of length 2. And so forth until the length of WherePaths equals the ** number of nodes in the FROM clause. The best (lowest cost) WherePath -** at the end is the choosen query plan. +** at the end is the chosen query plan. */ struct WherePath { Bitmask maskLoop; /* Bitmask of all WhereLoop objects in this path */ Bitmask revLoop; /* aLoop[]s that should be reversed for ORDER BY */ LogEst nRow; /* Estimated number of rows generated by this path */ LogEst rCost; /* Total cost of this path */ + LogEst rUnsorted; /* Total cost of this path ignoring sorting costs */ i8 isOrdered; /* No. of ORDER BY terms satisfied. -1 for unknown */ WhereLoop **aLoop; /* Array of WhereLoop objects implementing this path */ }; @@ -458,3 +462,4 @@ struct WhereInfo { #define WHERE_AUTO_INDEX 0x00004000 /* Uses an ephemeral index */ #define WHERE_SKIPSCAN 0x00008000 /* Uses the skip-scan algorithm */ #define WHERE_UNQ_WANTED 0x00010000 /* WHERE_ONEROW would have been helpful*/ +#define WHERE_PARTIALIDX 0x00020000 /* The automatic index is partial */ diff --git a/test/aggnested.test b/test/aggnested.test index 6e2fd6554b..a87c751eda 100644 --- a/test/aggnested.test +++ b/test/aggnested.test @@ -156,8 +156,14 @@ do_test aggnested-3.2 { (SELECT value1 as xyz, max(x1) AS pqr FROM t1 GROUP BY id1); + SELECT + (SELECT sum(value2<>xyz) FROM t2) + FROM + (SELECT value1 as xyz, max(x1) AS pqr + FROM t1 + GROUP BY id1); } -} {0} +} {1 0} do_test aggnested-3.3 { db eval { DROP TABLE IF EXISTS t1; diff --git a/test/alter4.test b/test/alter4.test index eaad37e001..ac39d614a5 100644 --- a/test/alter4.test +++ b/test/alter4.test @@ -145,7 +145,7 @@ do_test alter4-2.6 { } {1 {Cannot add a column with non-constant default}} do_test alter4-2.7 { catchsql { - alter table t1 add column d default (-+1); + alter table t1 add column d default (-5+1); } } {1 {Cannot add a column with non-constant default}} do_test alter4-2.99 { diff --git a/test/analyze6.test b/test/analyze6.test index f77f05324a..31ace8eda5 100644 --- a/test/analyze6.test +++ b/test/analyze6.test @@ -91,7 +91,7 @@ do_test analyze6-2.3 { } {0 0 0 {SEARCH TABLE t201 USING INTEGER PRIMARY KEY (rowid=?)}} do_test analyze6-2.4 { execsql { - INSERT INTO t201 VALUES(1,2,3); + INSERT INTO t201 VALUES(1,2,3),(2,3,4),(3,4,5); ANALYZE t201; } eqp {SELECT * FROM t201 WHERE z=5} diff --git a/test/analyze8.test b/test/analyze8.test index 4384c39676..1079e68080 100644 --- a/test/analyze8.test +++ b/test/analyze8.test @@ -86,23 +86,23 @@ do_test 2.1 { # range. # # Test 3.2 is a little unstable. It depends on the planner estimating -# that (b BETWEEN 50 AND 54) will match more rows than (c BETWEEN +# that (b BETWEEN 30 AND 34) will match more rows than (c BETWEEN # 800000 AND 900000). Which is a pretty close call (50 vs. 32), so # the planner could get it wrong with an unlucky set of samples. This # case happens to work, but others ("b BETWEEN 40 AND 44" for example) # will fail. # do_execsql_test 3.0 { - SELECT count(*) FROM t1 WHERE b BETWEEN 50 AND 54; + SELECT count(*) FROM t1 WHERE b BETWEEN 30 AND 34; SELECT count(*) FROM t1 WHERE c BETWEEN 0 AND 100000; SELECT count(*) FROM t1 WHERE c BETWEEN 800000 AND 900000; } {50 376 32} do_test 3.1 { - eqp {SELECT * FROM t1 WHERE b BETWEEN 50 AND 54 AND c BETWEEN 0 AND 100000} + eqp {SELECT * FROM t1 WHERE b BETWEEN 30 AND 34 AND c BETWEEN 0 AND 100000} } {0 0 0 {SEARCH TABLE t1 USING INDEX t1b (b>? AND b? AND c45 AND x<96) THEN 'B' ELSE 'A' END, /* Column "a" */ + x, /* Column "b" */ + CASE WHEN (x<51) THEN 'one' ELSE 'two' END, /* Column "c" */ + x /* Column "d" */ + FROM r; + + CREATE INDEX i3 ON t3(c); + CREATE INDEX i4 ON t3(d); + ANALYZE; +} + +# Expression (c='one' AND a='B') matches 5 table rows. But (c='one' AND a=A') +# matches 45. Expression (d? AND b=? term. Better than + # (a<20) but not as good as (a<10). + do_eqp_test 25.4.1 { + SELECT * FROM t6 WHERE a < 10 AND (b BETWEEN ? AND 60) + } { + 0 0 0 {SEARCH TABLE t6 USING INDEX aa (a? AND b? AND c? AND b? AND b? AND c? AND c. AND a<...*/} +do_execsql_test 1.2 { + SELECT c FROM t1 ORDER BY a; +} {3 111 6 12 9 12} +do_execsql_test 1.3 { + EXPLAIN QUERY PLAN + SELECT c FROM t1 ORDER BY a; +} {/.*SCAN TABLE t1 USING INDEX t1a.*/} +do_execsql_test 1.3x { + EXPLAIN QUERY PLAN + SELECT c FROM t1 ORDER BY a; +} {~/.*B-TREE FOR ORDER BY.*/} + +# Now mark the t1a index as "unordered". Range queries and ORDER BY no +# longer use the index, but equality queries do. +# +do_execsql_test 2.0 { + UPDATE sqlite_stat1 SET stat='12345 2 unordered' WHERE idx='t1a'; + ANALYZE sqlite_master; + SELECT *, '#' FROM t1 WHERE a BETWEEN 3 AND 8 ORDER BY c; +} {4 5 6 # 7 8 9 # 4 8 12 #} +do_execsql_test 2.1 { + EXPLAIN QUERY PLAN + SELECT *, '#' FROM t1 WHERE a BETWEEN 3 AND 8 ORDER BY c; +} {~/.*USING INDEX.*/} +do_execsql_test 2.2 { + SELECT c FROM t1 ORDER BY a; +} {3 111 6 12 9 12} +do_execsql_test 2.3 { + EXPLAIN QUERY PLAN + SELECT c FROM t1 ORDER BY a; +} {~/.*USING INDEX.*/} +do_execsql_test 2.3x { + EXPLAIN QUERY PLAN + SELECT c FROM t1 ORDER BY a; +} {/.*B-TREE FOR ORDER BY.*/} + +# Ignore extraneous text parameters in the sqlite_stat1.stat field. +# +do_execsql_test 3.0 { + UPDATE sqlite_stat1 SET stat='12345 2 whatever=5 unordered xyzzy=11' + WHERE idx='t1a'; + ANALYZE sqlite_master; + SELECT *, '#' FROM t1 WHERE a BETWEEN 3 AND 8 ORDER BY c; +} {4 5 6 # 7 8 9 # 4 8 12 #} +do_execsql_test 3.1 { + EXPLAIN QUERY PLAN + SELECT *, '#' FROM t1 WHERE a BETWEEN 3 AND 8 ORDER BY c; +} {~/.*USING INDEX.*/} +do_execsql_test 3.2 { + SELECT c FROM t1 ORDER BY a; +} {3 111 6 12 9 12} +do_execsql_test 3.3 { + EXPLAIN QUERY PLAN + SELECT c FROM t1 ORDER BY a; +} {~/.*USING INDEX.*/} +do_execsql_test 3.3x { + EXPLAIN QUERY PLAN + SELECT c FROM t1 ORDER BY a; +} {/.*B-TREE FOR ORDER BY.*/} + +# The sz=NNN parameter determines which index to scan +# +do_execsql_test 4.0 { + DROP INDEX t1a; + CREATE INDEX t1ab ON t1(a,b); + CREATE INDEX t1ca ON t1(c,a); + DELETE FROM sqlite_stat1; + INSERT INTO sqlite_stat1(tbl,idx,stat) + VALUES('t1','t1ab','12345 3 2 sz=10'),('t1','t1ca','12345 3 2 sz=20'); + ANALYZE sqlite_master; + SELECT count(a) FROM t1; +} {6} +do_execsql_test 4.1 { + EXPLAIN QUERY PLAN + SELECT count(a) FROM t1; +} {/.*INDEX t1ab.*/} +do_execsql_test 4.2 { + DELETE FROM sqlite_stat1; + INSERT INTO sqlite_stat1(tbl,idx,stat) + VALUES('t1','t1ab','12345 3 2 sz=20'),('t1','t1ca','12345 3 2 sz=10'); + ANALYZE sqlite_master; + SELECT count(a) FROM t1; +} {6} +do_execsql_test 4.3 { + EXPLAIN QUERY PLAN + SELECT count(a) FROM t1; +} {/.*INDEX t1ca.*/} + + +# The sz=NNN parameter works even if there is other extraneous text +# in the sqlite_stat1.stat column. +# +do_execsql_test 5.0 { + DELETE FROM sqlite_stat1; + INSERT INTO sqlite_stat1(tbl,idx,stat) + VALUES('t1','t1ab','12345 3 2 x=5 sz=10 y=10'), + ('t1','t1ca','12345 3 2 whatever sz=20 junk'); + ANALYZE sqlite_master; + SELECT count(a) FROM t1; +} {6} +do_execsql_test 5.1 { + EXPLAIN QUERY PLAN + SELECT count(a) FROM t1; +} {/.*INDEX t1ab.*/} +do_execsql_test 5.2 { + DELETE FROM sqlite_stat1; + INSERT INTO sqlite_stat1(tbl,idx,stat) + VALUES('t1','t1ca','12345 3 2 x=5 sz=10 y=10'), + ('t1','t1ab','12345 3 2 whatever sz=20 junk'); + ANALYZE sqlite_master; + SELECT count(a) FROM t1; +} {6} +do_execsql_test 5.3 { + EXPLAIN QUERY PLAN + SELECT count(a) FROM t1; +} {/.*INDEX t1ca.*/} + + + + +finish_test diff --git a/test/analyzeD.test b/test/analyzeD.test new file mode 100644 index 0000000000..4d46be6c64 --- /dev/null +++ b/test/analyzeD.test @@ -0,0 +1,117 @@ +# 2005 July 22 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# This file implements tests for the ANALYZE command. +# +# $Id: analyze.test,v 1.9 2008/08/11 18:44:58 drh Exp $ + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix analyzeD + +ifcapable {!stat4} { + finish_test + return +} + + +# Set up a table with the following properties: +# +# * Contains 1000 rows. +# * Column a contains even integers between 0 and 18, inclusive (so that +# a=? for any such integer matches 100 rows). +# * Column b contains integers between 0 and 9, inclusive. +# * Column c contains integers between 0 and 199, inclusive (so that +# for any such integer, c=? matches 5 rows). +# * Then add 7 rows with a new value for "a" - 3001. The stat4 table will +# not contain any samples with a=3001. +# +do_execsql_test 1.0 { + CREATE TABLE t1(a, b, c); +} +do_test 1.1 { + for {set i 1} {$i < 1000} {incr i} { + set c [expr $i % 200] + execsql { INSERT INTO t1(a, b, c) VALUES( 2*($i/100), $i%10, $c ) } + } + + execsql { + INSERT INTO t1 VALUES(3001, 3001, 3001); + INSERT INTO t1 VALUES(3001, 3001, 3002); + INSERT INTO t1 VALUES(3001, 3001, 3003); + INSERT INTO t1 VALUES(3001, 3001, 3004); + INSERT INTO t1 VALUES(3001, 3001, 3005); + INSERT INTO t1 VALUES(3001, 3001, 3006); + INSERT INTO t1 VALUES(3001, 3001, 3007); + + CREATE INDEX t1_ab ON t1(a, b); + CREATE INDEX t1_c ON t1(c); + + ANALYZE; + } +} {} + +# With full ANALYZE data, SQLite sees that c=150 (5 rows) is better than +# a=3001 (7 rows). +# +do_eqp_test 1.2 { + SELECT * FROM t1 WHERE a=3001 AND c=150; +} { + 0 0 0 {SEARCH TABLE t1 USING INDEX t1_c (c=?)} +} + +do_test 1.3 { + execsql { DELETE FROM sqlite_stat1 } + db close + sqlite3 db test.db +} {} + +# Without stat1, because 3001 is larger than all samples in the stat4 +# table, SQLite things that a=3001 matches just 1 row. So it (incorrectly) +# chooses it over the c=150 index (5 rows). Even with stat1 data, things +# worked this way before commit [e6f7f97dbc]. +# +do_eqp_test 1.4 { + SELECT * FROM t1 WHERE a=3001 AND c=150; +} { + 0 0 0 {SEARCH TABLE t1 USING INDEX t1_ab (a=?)} +} + +do_test 1.5 { + execsql { + UPDATE t1 SET a=13 WHERE a = 3001; + ANALYZE; + } +} {} + +do_eqp_test 1.6 { + SELECT * FROM t1 WHERE a=13 AND c=150; +} { + 0 0 0 {SEARCH TABLE t1 USING INDEX t1_c (c=?)} +} + +do_test 1.7 { + execsql { DELETE FROM sqlite_stat1 } + db close + sqlite3 db test.db +} {} + +# Same test as 1.4, except this time the 7 rows that match the a=? condition +# do not feature larger values than all rows in the stat4 table. So SQLite +# gets this right, even without stat1 data. +do_eqp_test 1.8 { + SELECT * FROM t1 WHERE a=13 AND c=150; +} { + 0 0 0 {SEARCH TABLE t1 USING INDEX t1_c (c=?)} +} + +finish_test + diff --git a/test/analyzeE.test b/test/analyzeE.test new file mode 100644 index 0000000000..66db1e122c --- /dev/null +++ b/test/analyzeE.test @@ -0,0 +1,242 @@ +# 2014-10-08 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements tests for using STAT4 information +# on a descending index in a range query. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix analyzeE + +ifcapable {!stat4} { + finish_test + return +} + +# Verify that range queries on an ASCENDING index will use the +# index only if the range covers only a small fraction of the +# entries. +# +do_execsql_test analyzeE-1.0 { + CREATE TABLE t1(a,b); + WITH RECURSIVE + cnt(x) AS (VALUES(1000) UNION ALL SELECT x+1 FROM cnt WHERE x<2000) + INSERT INTO t1(a,b) SELECT x, x FROM cnt; + CREATE INDEX t1a ON t1(a); + ANALYZE; +} {} +do_execsql_test analyzeE-1.1 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a BETWEEN 500 AND 2500; +} {/SCAN TABLE t1/} +do_execsql_test analyzeE-1.2 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a BETWEEN 2900 AND 3000; +} {/SEARCH TABLE t1 USING INDEX t1a/} +do_execsql_test analyzeE-1.3 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a BETWEEN 1700 AND 1750; +} {/SEARCH TABLE t1 USING INDEX t1a/} +do_execsql_test analyzeE-1.4 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a BETWEEN 1 AND 500 +} {/SEARCH TABLE t1 USING INDEX t1a/} +do_execsql_test analyzeE-1.5 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a BETWEEN 3000 AND 3000000 +} {/SEARCH TABLE t1 USING INDEX t1a/} +do_execsql_test analyzeE-1.6 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a<500 +} {/SEARCH TABLE t1 USING INDEX t1a/} +do_execsql_test analyzeE-1.7 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a>2500 +} {/SEARCH TABLE t1 USING INDEX t1a/} +do_execsql_test analyzeE-1.8 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a>1900 +} {/SEARCH TABLE t1 USING INDEX t1a/} +do_execsql_test analyzeE-1.9 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a>1100 +} {/SCAN TABLE t1/} +do_execsql_test analyzeE-1.10 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a<1100 +} {/SEARCH TABLE t1 USING INDEX t1a/} +do_execsql_test analyzeE-1.11 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a<1900 +} {/SCAN TABLE t1/} + +# Verify that everything works the same on a DESCENDING index. +# +do_execsql_test analyzeE-2.0 { + DROP INDEX t1a; + CREATE INDEX t1a ON t1(a DESC); + ANALYZE; +} {} +do_execsql_test analyzeE-2.1 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a BETWEEN 500 AND 2500; +} {/SCAN TABLE t1/} +do_execsql_test analyzeE-2.2 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a BETWEEN 2900 AND 3000; +} {/SEARCH TABLE t1 USING INDEX t1a/} +do_execsql_test analyzeE-2.3 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a BETWEEN 1700 AND 1750; +} {/SEARCH TABLE t1 USING INDEX t1a/} +do_execsql_test analyzeE-2.4 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a BETWEEN 1 AND 500 +} {/SEARCH TABLE t1 USING INDEX t1a/} +do_execsql_test analyzeE-2.5 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a BETWEEN 3000 AND 3000000 +} {/SEARCH TABLE t1 USING INDEX t1a/} +do_execsql_test analyzeE-2.6 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a<500 +} {/SEARCH TABLE t1 USING INDEX t1a/} +do_execsql_test analyzeE-2.7 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a>2500 +} {/SEARCH TABLE t1 USING INDEX t1a/} +do_execsql_test analyzeE-2.8 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a>1900 +} {/SEARCH TABLE t1 USING INDEX t1a/} +do_execsql_test analyzeE-2.9 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a>1100 +} {/SCAN TABLE t1/} +do_execsql_test analyzeE-2.10 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a<1100 +} {/SEARCH TABLE t1 USING INDEX t1a/} +do_execsql_test analyzeE-2.11 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a<1900 +} {/SCAN TABLE t1/} + +# Now do a range query on the second term of an ASCENDING index +# where the first term is constrained by equality. +# +do_execsql_test analyzeE-3.0 { + DROP TABLE t1; + CREATE TABLE t1(a,b,c); + WITH RECURSIVE + cnt(x) AS (VALUES(1000) UNION ALL SELECT x+1 FROM cnt WHERE x<2000) + INSERT INTO t1(a,b,c) SELECT x, x, 123 FROM cnt; + CREATE INDEX t1ca ON t1(c,a); + ANALYZE; +} {} +do_execsql_test analyzeE-3.1 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a BETWEEN 500 AND 2500 AND c=123; +} {/SCAN TABLE t1/} +do_execsql_test analyzeE-3.2 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a BETWEEN 2900 AND 3000 AND c=123; +} {/SEARCH TABLE t1 USING INDEX t1ca/} +do_execsql_test analyzeE-3.3 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a BETWEEN 1700 AND 1750 AND c=123; +} {/SEARCH TABLE t1 USING INDEX t1ca/} +do_execsql_test analyzeE-3.4 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a BETWEEN 1 AND 500 AND c=123 +} {/SEARCH TABLE t1 USING INDEX t1ca/} +do_execsql_test analyzeE-3.5 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a BETWEEN 3000 AND 3000000 AND c=123 +} {/SEARCH TABLE t1 USING INDEX t1ca/} +do_execsql_test analyzeE-3.6 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a<500 AND c=123 +} {/SEARCH TABLE t1 USING INDEX t1ca/} +do_execsql_test analyzeE-3.7 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a>2500 AND c=123 +} {/SEARCH TABLE t1 USING INDEX t1ca/} +do_execsql_test analyzeE-3.8 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a>1900 AND c=123 +} {/SEARCH TABLE t1 USING INDEX t1ca/} +do_execsql_test analyzeE-3.9 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a>1100 AND c=123 +} {/SCAN TABLE t1/} +do_execsql_test analyzeE-3.10 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a<1100 AND c=123 +} {/SEARCH TABLE t1 USING INDEX t1ca/} +do_execsql_test analyzeE-3.11 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a<1900 AND c=123 +} {/SCAN TABLE t1/} + +# Repeat the 3.x tests using a DESCENDING index +# +do_execsql_test analyzeE-4.0 { + DROP INDEX t1ca; + CREATE INDEX t1ca ON t1(c ASC,a DESC); + ANALYZE; +} {} +do_execsql_test analyzeE-4.1 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a BETWEEN 500 AND 2500 AND c=123; +} {/SCAN TABLE t1/} +do_execsql_test analyzeE-4.2 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a BETWEEN 2900 AND 3000 AND c=123; +} {/SEARCH TABLE t1 USING INDEX t1ca/} +do_execsql_test analyzeE-4.3 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a BETWEEN 1700 AND 1750 AND c=123; +} {/SEARCH TABLE t1 USING INDEX t1ca/} +do_execsql_test analyzeE-4.4 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a BETWEEN 1 AND 500 AND c=123 +} {/SEARCH TABLE t1 USING INDEX t1ca/} +do_execsql_test analyzeE-4.5 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a BETWEEN 3000 AND 3000000 AND c=123 +} {/SEARCH TABLE t1 USING INDEX t1ca/} +do_execsql_test analyzeE-4.6 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a<500 AND c=123 +} {/SEARCH TABLE t1 USING INDEX t1ca/} +do_execsql_test analyzeE-4.7 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a>2500 AND c=123 +} {/SEARCH TABLE t1 USING INDEX t1ca/} +do_execsql_test analyzeE-4.8 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a>1900 AND c=123 +} {/SEARCH TABLE t1 USING INDEX t1ca/} +do_execsql_test analyzeE-4.9 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a>1100 AND c=123 +} {/SCAN TABLE t1/} +do_execsql_test analyzeE-4.10 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a<1100 AND c=123 +} {/SEARCH TABLE t1 USING INDEX t1ca/} +do_execsql_test analyzeE-4.11 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a<1900 AND c=123 +} {/SCAN TABLE t1/} + +finish_test diff --git a/test/auth.test b/test/auth.test index 43e53ef2e3..f3c2fa79e8 100644 --- a/test/auth.test +++ b/test/auth.test @@ -36,7 +36,7 @@ proc_real proc {name arguments script} { do_test auth-1.1.1 { db close set ::DB [sqlite3 db test.db] - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_INSERT" && $arg1=="sqlite_master"} { return SQLITE_DENY } @@ -61,7 +61,7 @@ do_test auth-1.2 { execsql {SELECT name FROM sqlite_master} } {} do_test auth-1.3.1 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_TABLE"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -82,7 +82,7 @@ do_test auth-1.4 { ifcapable tempdb { do_test auth-1.5 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_INSERT" && $arg1=="sqlite_temp_master"} { return SQLITE_DENY } @@ -94,7 +94,7 @@ ifcapable tempdb { execsql {SELECT name FROM sqlite_temp_master} } {} do_test auth-1.7.1 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_TEMP_TABLE"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -112,7 +112,7 @@ ifcapable tempdb { } do_test auth-1.9 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_INSERT" && $arg1=="sqlite_master"} { return SQLITE_IGNORE } @@ -124,7 +124,7 @@ do_test auth-1.10 { execsql {SELECT name FROM sqlite_master} } {} do_test auth-1.11 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_TABLE"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -139,7 +139,7 @@ do_test auth-1.12 { ifcapable tempdb { do_test auth-1.13 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_INSERT" && $arg1=="sqlite_temp_master"} { return SQLITE_IGNORE } @@ -151,7 +151,7 @@ ifcapable tempdb { execsql {SELECT name FROM sqlite_temp_master} } {} do_test auth-1.15 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_TEMP_TABLE"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -165,7 +165,7 @@ ifcapable tempdb { } {} do_test auth-1.17 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_TABLE"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -181,7 +181,7 @@ ifcapable tempdb { do_test auth-1.19.1 { set ::authargs {} - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_TEMP_TABLE"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -198,7 +198,7 @@ do_test auth-1.20 { } {t2} do_test auth-1.21.1 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_TABLE"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -214,7 +214,7 @@ do_test auth-1.22 { execsql {SELECT name FROM sqlite_master} } {t2} do_test auth-1.23.1 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_TABLE"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -232,7 +232,7 @@ do_test auth-1.24 { ifcapable tempdb { do_test auth-1.25 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_TEMP_TABLE"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -245,7 +245,7 @@ ifcapable tempdb { execsql {SELECT name FROM sqlite_temp_master} } {t1} do_test auth-1.27 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_TEMP_TABLE"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -260,7 +260,7 @@ ifcapable tempdb { } do_test auth-1.29 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_INSERT" && $arg1=="t2"} { return SQLITE_DENY } @@ -272,7 +272,7 @@ do_test auth-1.30 { execsql {SELECT * FROM t2} } {} do_test auth-1.31 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_INSERT" && $arg1=="t2"} { return SQLITE_IGNORE } @@ -284,7 +284,7 @@ do_test auth-1.32 { execsql {SELECT * FROM t2} } {} do_test auth-1.33 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_INSERT" && $arg1=="t1"} { return SQLITE_IGNORE } @@ -297,7 +297,7 @@ do_test auth-1.34 { } {1 2 3} do_test auth-1.35.1 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_READ" && $arg1=="t2" && $arg2=="b"} { return SQLITE_DENY } @@ -313,7 +313,7 @@ ifcapable attach { execsql {DETACH DATABASE two} } do_test auth-1.36 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_READ" && $arg1=="t2" && $arg2=="b"} { return SQLITE_IGNORE } @@ -322,7 +322,7 @@ do_test auth-1.36 { catchsql {SELECT * FROM t2} } {0 {1 {} 3}} do_test auth-1.37 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_READ" && $arg1=="t2" && $arg2=="b"} { return SQLITE_IGNORE } @@ -331,7 +331,7 @@ do_test auth-1.37 { catchsql {SELECT * FROM t2 WHERE b=2} } {0 {}} do_test auth-1.38 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_READ" && $arg1=="t2" && $arg2=="a"} { return SQLITE_IGNORE } @@ -340,7 +340,7 @@ do_test auth-1.38 { catchsql {SELECT * FROM t2 WHERE b=2} } {0 {{} 2 3}} do_test auth-1.39 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_READ" && $arg1=="t2" && $arg2=="b"} { return SQLITE_IGNORE } @@ -349,7 +349,7 @@ do_test auth-1.39 { catchsql {SELECT * FROM t2 WHERE b IS NULL} } {0 {1 {} 3}} do_test auth-1.40 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_READ" && $arg1=="t2" && $arg2=="b"} { return SQLITE_DENY } @@ -359,7 +359,7 @@ do_test auth-1.40 { } {1 {access to t2.b is prohibited}} do_test auth-1.41 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_UPDATE" && $arg1=="t2" && $arg2=="b"} { return SQLITE_DENY } @@ -371,7 +371,7 @@ do_test auth-1.42 { execsql {SELECT * FROM t2} } {11 2 3} do_test auth-1.43 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_UPDATE" && $arg1=="t2" && $arg2=="b"} { return SQLITE_DENY } @@ -383,7 +383,7 @@ do_test auth-1.44 { execsql {SELECT * FROM t2} } {11 2 3} do_test auth-1.45 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_UPDATE" && $arg1=="t2" && $arg2=="b"} { return SQLITE_IGNORE } @@ -396,7 +396,7 @@ do_test auth-1.46 { } {11 2 33} do_test auth-1.47 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="t2"} { return SQLITE_DENY } @@ -408,7 +408,7 @@ do_test auth-1.48 { execsql {SELECT * FROM t2} } {11 2 33} do_test auth-1.49 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="t2"} { return SQLITE_IGNORE } @@ -424,7 +424,7 @@ do_test auth-1.50.2 { } {} do_test auth-1.51 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_SELECT"} { return SQLITE_DENY } @@ -433,7 +433,7 @@ do_test auth-1.51 { catchsql {SELECT * FROM t2} } {1 {not authorized}} do_test auth-1.52 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_SELECT"} { return SQLITE_IGNORE } @@ -442,7 +442,7 @@ do_test auth-1.52 { catchsql {SELECT * FROM t2} } {0 {}} do_test auth-1.53 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_SELECT"} { return SQLITE_OK } @@ -462,7 +462,7 @@ do_test auth-1.55 { } {11 2 33 7 8 9} do_test auth-1.63 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="sqlite_master"} { return SQLITE_DENY } @@ -474,7 +474,7 @@ do_test auth-1.64 { execsql {SELECT name FROM sqlite_master} } {t2} do_test auth-1.65 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="t2"} { return SQLITE_DENY } @@ -488,7 +488,7 @@ do_test auth-1.66 { ifcapable tempdb { do_test auth-1.67 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="sqlite_temp_master"} { return SQLITE_DENY } @@ -500,7 +500,7 @@ ifcapable tempdb { execsql {SELECT name FROM sqlite_temp_master} } {t1} do_test auth-1.69 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="t1"} { return SQLITE_DENY } @@ -514,7 +514,7 @@ ifcapable tempdb { } do_test auth-1.71 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="sqlite_master"} { return SQLITE_IGNORE } @@ -526,7 +526,7 @@ do_test auth-1.72 { execsql {SELECT name FROM sqlite_master} } {t2} do_test auth-1.73 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="t2"} { return SQLITE_IGNORE } @@ -540,7 +540,7 @@ do_test auth-1.74 { ifcapable tempdb { do_test auth-1.75 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="sqlite_temp_master"} { return SQLITE_IGNORE } @@ -552,7 +552,7 @@ ifcapable tempdb { execsql {SELECT name FROM sqlite_temp_master} } {t1} do_test auth-1.77 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="t1"} { return SQLITE_IGNORE } @@ -569,7 +569,7 @@ ifcapable tempdb { # Omit these if the library was compiled with views omitted. ifcapable view { do_test auth-1.79 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_VIEW"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -585,7 +585,7 @@ do_test auth-1.81 { execsql {SELECT name FROM sqlite_master} } {t2} do_test auth-1.82 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_VIEW"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -603,7 +603,7 @@ do_test auth-1.84 { ifcapable tempdb { do_test auth-1.85 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_TEMP_VIEW"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -619,7 +619,7 @@ ifcapable tempdb { execsql {SELECT name FROM sqlite_temp_master} } {t1} do_test auth-1.88 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_TEMP_VIEW"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -637,7 +637,7 @@ ifcapable tempdb { } do_test auth-1.91 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_INSERT" && $arg1=="sqlite_master"} { return SQLITE_DENY } @@ -649,7 +649,7 @@ do_test auth-1.92 { execsql {SELECT name FROM sqlite_master} } {t2} do_test auth-1.93 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_INSERT" && $arg1=="sqlite_master"} { return SQLITE_IGNORE } @@ -663,7 +663,7 @@ do_test auth-1.94 { ifcapable tempdb { do_test auth-1.95 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_INSERT" && $arg1=="sqlite_temp_master"} { return SQLITE_DENY } @@ -675,7 +675,7 @@ ifcapable tempdb { execsql {SELECT name FROM sqlite_temp_master} } {t1} do_test auth-1.97 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_INSERT" && $arg1=="sqlite_temp_master"} { return SQLITE_IGNORE } @@ -689,7 +689,7 @@ ifcapable tempdb { } do_test auth-1.99 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="sqlite_master"} { return SQLITE_DENY } @@ -704,7 +704,7 @@ do_test auth-1.100 { execsql {SELECT name FROM sqlite_master} } {t2 v2} do_test auth-1.101 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_VIEW"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -720,7 +720,7 @@ do_test auth-1.103 { execsql {SELECT name FROM sqlite_master} } {t2 v2} do_test auth-1.104 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="sqlite_master"} { return SQLITE_IGNORE } @@ -732,7 +732,7 @@ do_test auth-1.105 { execsql {SELECT name FROM sqlite_master} } {t2 v2} do_test auth-1.106 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_VIEW"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -748,7 +748,7 @@ do_test auth-1.108 { execsql {SELECT name FROM sqlite_master} } {t2 v2} do_test auth-1.109 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_VIEW"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_OK @@ -767,7 +767,7 @@ do_test auth-1.111 { ifcapable tempdb { do_test auth-1.112 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="sqlite_temp_master"} { return SQLITE_DENY } @@ -782,7 +782,7 @@ ifcapable tempdb { execsql {SELECT name FROM sqlite_temp_master} } {t1 v1} do_test auth-1.114 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_TEMP_VIEW"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -798,7 +798,7 @@ ifcapable tempdb { execsql {SELECT name FROM sqlite_temp_master} } {t1 v1} do_test auth-1.117 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="sqlite_temp_master"} { return SQLITE_IGNORE } @@ -810,7 +810,7 @@ ifcapable tempdb { execsql {SELECT name FROM sqlite_temp_master} } {t1 v1} do_test auth-1.119 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_TEMP_VIEW"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -826,7 +826,7 @@ ifcapable tempdb { execsql {SELECT name FROM sqlite_temp_master} } {t1 v1} do_test auth-1.122 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_TEMP_VIEW"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_OK @@ -849,7 +849,7 @@ ifcapable tempdb { # ifcapable trigger&&tempdb { do_test auth-1.125 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_TRIGGER"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -869,7 +869,7 @@ do_test auth-1.127 { execsql {SELECT name FROM sqlite_master} } {t2} do_test auth-1.128 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_INSERT" && $arg1=="sqlite_master"} { return SQLITE_DENY } @@ -885,7 +885,7 @@ do_test auth-1.129 { execsql {SELECT name FROM sqlite_master} } {t2} do_test auth-1.130 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_TRIGGER"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -905,7 +905,7 @@ do_test auth-1.132 { execsql {SELECT name FROM sqlite_master} } {t2} do_test auth-1.133 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_INSERT" && $arg1=="sqlite_master"} { return SQLITE_IGNORE } @@ -921,7 +921,7 @@ do_test auth-1.134 { execsql {SELECT name FROM sqlite_master} } {t2} do_test auth-1.135 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_TRIGGER"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_OK @@ -944,7 +944,7 @@ do_test auth-1.136.2 { } } {r2} do_test auth-1.136.3 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { lappend ::authargs $code $arg1 $arg2 $arg3 $arg4 return SQLITE_OK } @@ -963,7 +963,7 @@ do_test auth-1.137 { execsql {SELECT name FROM sqlite_master} } {t2 tx r2} do_test auth-1.138 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_TEMP_TRIGGER"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -983,7 +983,7 @@ do_test auth-1.140 { execsql {SELECT name FROM sqlite_temp_master} } {t1} do_test auth-1.141 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_INSERT" && $arg1=="sqlite_temp_master"} { return SQLITE_DENY } @@ -999,7 +999,7 @@ do_test auth-1.142 { execsql {SELECT name FROM sqlite_temp_master} } {t1} do_test auth-1.143 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_TEMP_TRIGGER"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -1019,7 +1019,7 @@ do_test auth-1.145 { execsql {SELECT name FROM sqlite_temp_master} } {t1} do_test auth-1.146 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_INSERT" && $arg1=="sqlite_temp_master"} { return SQLITE_IGNORE } @@ -1035,7 +1035,7 @@ do_test auth-1.147 { execsql {SELECT name FROM sqlite_temp_master} } {t1} do_test auth-1.148 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_TEMP_TRIGGER"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_OK @@ -1056,7 +1056,7 @@ do_test auth-1.150 { } {t1 r1} do_test auth-1.151 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="sqlite_master"} { return SQLITE_DENY } @@ -1068,7 +1068,7 @@ do_test auth-1.152 { execsql {SELECT name FROM sqlite_master} } {t2 tx r2} do_test auth-1.153 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_TRIGGER"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -1084,7 +1084,7 @@ do_test auth-1.155 { execsql {SELECT name FROM sqlite_master} } {t2 tx r2} do_test auth-1.156 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="sqlite_master"} { return SQLITE_IGNORE } @@ -1096,7 +1096,7 @@ do_test auth-1.157 { execsql {SELECT name FROM sqlite_master} } {t2 tx r2} do_test auth-1.158 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_TRIGGER"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -1112,7 +1112,7 @@ do_test auth-1.160 { execsql {SELECT name FROM sqlite_master} } {t2 tx r2} do_test auth-1.161 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_TRIGGER"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_OK @@ -1133,7 +1133,7 @@ do_test auth-1.163 { } {t2} do_test auth-1.164 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="sqlite_temp_master"} { return SQLITE_DENY } @@ -1145,7 +1145,7 @@ do_test auth-1.165 { execsql {SELECT name FROM sqlite_temp_master} } {t1 r1} do_test auth-1.166 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_TEMP_TRIGGER"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -1161,7 +1161,7 @@ do_test auth-1.168 { execsql {SELECT name FROM sqlite_temp_master} } {t1 r1} do_test auth-1.169 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="sqlite_temp_master"} { return SQLITE_IGNORE } @@ -1173,7 +1173,7 @@ do_test auth-1.170 { execsql {SELECT name FROM sqlite_temp_master} } {t1 r1} do_test auth-1.171 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_TEMP_TRIGGER"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -1189,7 +1189,7 @@ do_test auth-1.173 { execsql {SELECT name FROM sqlite_temp_master} } {t1 r1} do_test auth-1.174 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_TEMP_TRIGGER"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_OK @@ -1207,7 +1207,7 @@ do_test auth-1.176 { } ;# ifcapable trigger do_test auth-1.177 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_INDEX"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -1223,7 +1223,7 @@ do_test auth-1.179 { execsql {SELECT name FROM sqlite_master} } {t2} do_test auth-1.180 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_INSERT" && $arg1=="sqlite_master"} { return SQLITE_DENY } @@ -1235,7 +1235,7 @@ do_test auth-1.181 { execsql {SELECT name FROM sqlite_master} } {t2} do_test auth-1.182 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_INDEX"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -1251,7 +1251,7 @@ do_test auth-1.184 { execsql {SELECT name FROM sqlite_master} } {t2} do_test auth-1.185 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_INSERT" && $arg1=="sqlite_master"} { return SQLITE_IGNORE } @@ -1263,7 +1263,7 @@ do_test auth-1.186 { execsql {SELECT name FROM sqlite_master} } {t2} do_test auth-1.187 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_INDEX"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_OK @@ -1281,7 +1281,7 @@ do_test auth-1.189 { ifcapable tempdb { do_test auth-1.190 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_TEMP_INDEX"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -1297,7 +1297,7 @@ ifcapable tempdb { execsql {SELECT name FROM sqlite_temp_master} } {t1} do_test auth-1.193 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_INSERT" && $arg1=="sqlite_temp_master"} { return SQLITE_DENY } @@ -1309,7 +1309,7 @@ ifcapable tempdb { execsql {SELECT name FROM sqlite_temp_master} } {t1} do_test auth-1.195 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_TEMP_INDEX"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -1325,7 +1325,7 @@ ifcapable tempdb { execsql {SELECT name FROM sqlite_temp_master} } {t1} do_test auth-1.198 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_INSERT" && $arg1=="sqlite_temp_master"} { return SQLITE_IGNORE } @@ -1337,7 +1337,7 @@ ifcapable tempdb { execsql {SELECT name FROM sqlite_temp_master} } {t1} do_test auth-1.200 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_TEMP_INDEX"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_OK @@ -1355,7 +1355,7 @@ ifcapable tempdb { } do_test auth-1.203 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="sqlite_master"} { return SQLITE_DENY } @@ -1367,7 +1367,7 @@ do_test auth-1.204 { execsql {SELECT name FROM sqlite_master} } {t2 i2} do_test auth-1.205 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_INDEX"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -1383,7 +1383,7 @@ do_test auth-1.207 { execsql {SELECT name FROM sqlite_master} } {t2 i2} do_test auth-1.208 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="sqlite_master"} { return SQLITE_IGNORE } @@ -1395,7 +1395,7 @@ do_test auth-1.209 { execsql {SELECT name FROM sqlite_master} } {t2 i2} do_test auth-1.210 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_INDEX"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -1411,7 +1411,7 @@ do_test auth-1.212 { execsql {SELECT name FROM sqlite_master} } {t2 i2} do_test auth-1.213 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_INDEX"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_OK @@ -1429,7 +1429,7 @@ do_test auth-1.215 { ifcapable tempdb { do_test auth-1.216 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="sqlite_temp_master"} { return SQLITE_DENY } @@ -1441,7 +1441,7 @@ ifcapable tempdb { execsql {SELECT name FROM sqlite_temp_master} } {t1 i1} do_test auth-1.218 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_TEMP_INDEX"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -1457,7 +1457,7 @@ ifcapable tempdb { execsql {SELECT name FROM sqlite_temp_master} } {t1 i1} do_test auth-1.221 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="sqlite_temp_master"} { return SQLITE_IGNORE } @@ -1469,7 +1469,7 @@ ifcapable tempdb { execsql {SELECT name FROM sqlite_temp_master} } {t1 i1} do_test auth-1.223 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_TEMP_INDEX"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -1485,7 +1485,7 @@ ifcapable tempdb { execsql {SELECT name FROM sqlite_temp_master} } {t1 i1} do_test auth-1.226 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_TEMP_INDEX"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_OK @@ -1503,7 +1503,7 @@ ifcapable tempdb { } do_test auth-1.229 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_PRAGMA"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -1519,7 +1519,7 @@ do_test auth-1.231 { execsql2 {SELECT a FROM t2} } {a 11 a 7} do_test auth-1.232 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_PRAGMA"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -1535,7 +1535,7 @@ do_test auth-1.234 { execsql2 {SELECT a FROM t2} } {a 11 a 7} do_test auth-1.235 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_PRAGMA"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_OK @@ -1548,7 +1548,7 @@ do_test auth-1.236 { execsql2 {SELECT a FROM t2} } {t2.a 11 t2.a 7} do_test auth-1.237 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_PRAGMA"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_OK @@ -1565,7 +1565,7 @@ do_test auth-1.239 { } {a 11 a 7} do_test auth-1.240 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_TRANSACTION"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -1578,7 +1578,7 @@ do_test auth-1.241 { set ::authargs } {BEGIN {} {} {}} do_test auth-1.242 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_TRANSACTION" && $arg1!="BEGIN"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -1618,7 +1618,7 @@ do_test auth-1.250 { ifcapable attach { do_test auth-1.251 { db authorizer ::auth - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_ATTACH"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] } @@ -1644,7 +1644,7 @@ ifcapable attach { } {{} {} {} {}} do_test auth-1.253 { catchsql {DETACH DATABASE test1} - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_ATTACH"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -1660,7 +1660,7 @@ ifcapable attach { } {} do_test auth-1.255 { catchsql {DETACH DATABASE test1} - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_ATTACH"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -1675,7 +1675,7 @@ ifcapable attach { lindex [execsql {PRAGMA database_list}] 7 } {} do_test auth-1.257 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DETACH"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_OK @@ -1692,7 +1692,7 @@ ifcapable attach { } {} do_test auth-1.259 { execsql {ATTACH DATABASE ':memory:' AS test1} - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DETACH"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -1710,7 +1710,7 @@ ifcapable attach { } {test1} } ;# ifcapable schema_pragmas do_test auth-1.261 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DETACH"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -1735,7 +1735,7 @@ ifcapable attach { ifcapable altertable { do_test auth-1.263 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_ALTER_TABLE"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_OK @@ -1753,7 +1753,7 @@ ifcapable attach { set authargs } {temp t1 {} {}} do_test auth-1.266 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_ALTER_TABLE"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -1771,7 +1771,7 @@ ifcapable attach { set authargs } {temp t1x {} {}} do_test auth-1.269 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_ALTER_TABLE"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -1804,7 +1804,7 @@ db authorizer {} catchsql {ALTER TABLE t1x RENAME TO t1} db authorizer ::auth do_test auth-1.272 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_ALTER_TABLE"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_OK @@ -1822,7 +1822,7 @@ do_test auth-1.274 { set authargs } {main t2 {} {}} do_test auth-1.275 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_ALTER_TABLE"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -1840,7 +1840,7 @@ do_test auth-1.277 { set authargs } {main t2x {} {}} do_test auth-1.278 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_ALTER_TABLE"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -1867,7 +1867,7 @@ ifcapable reindex { proc auth {code args} { if {$code=="SQLITE_REINDEX"} { - set ::authargs [concat $::authargs $args] + set ::authargs [concat $::authargs [lrange $args 0 3]] } return SQLITE_OK } @@ -1950,7 +1950,7 @@ ifcapable tempdb { } {t3_idx2 {} temp {} t3_idx1 {} temp {} sqlite_autoindex_t3_1 {} temp {}} proc auth {code args} { if {$code=="SQLITE_REINDEX"} { - set ::authargs [concat $::authargs $args] + set ::authargs [concat $::authargs [lrange $args 0 3]] return SQLITE_DENY } return SQLITE_OK @@ -1973,7 +1973,7 @@ ifcapable tempdb { ifcapable analyze { proc auth {code args} { if {$code=="SQLITE_ANALYZE"} { - set ::authargs [concat $::authargs $args] + set ::authargs [concat $::authargs [lrange $args 0 3]] } return SQLITE_OK } @@ -2020,7 +2020,7 @@ ifcapable analyze { ifcapable {altertable} { do_test auth-1.300 { execsql {CREATE TABLE t5(x)} - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_ALTER_TABLE"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_OK @@ -2039,7 +2039,7 @@ ifcapable {altertable} { set authargs } {main t5 {} {}} do_test auth-1.303 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_ALTER_TABLE"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -2058,7 +2058,7 @@ ifcapable {altertable} { set authargs } {main t5 {} {}} do_test auth-1.306 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_ALTER_TABLE"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -2082,7 +2082,7 @@ ifcapable {altertable} { ifcapable {cte} { do_test auth-1.310 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_RECURSIVE"} { return SQLITE_DENY } @@ -2117,7 +2117,7 @@ ifcapable {cte} { } ;# ifcapable cte do_test auth-2.1 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_READ" && $arg1=="t3" && $arg2=="x"} { return SQLITE_DENY } @@ -2137,7 +2137,7 @@ do_test auth-2.3 { catchsql {SELECT OID,y,z FROM t3} } {1 {access to t3.x is prohibited}} do_test auth-2.4 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_READ" && $arg1=="t3" && $arg2=="x"} { return SQLITE_IGNORE } @@ -2150,7 +2150,7 @@ do_test auth-2.5 { catchsql {SELECT rowid,y,z FROM t3} } {0 {{} 55 66}} do_test auth-2.6 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_READ" && $arg1=="t3" && $arg2=="ROWID"} { return SQLITE_IGNORE } @@ -2162,7 +2162,7 @@ do_test auth-2.7 { catchsql {SELECT ROWID,y,z FROM t3} } {0 {44 55 66}} do_test auth-2.8 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_READ" && $arg1=="t2" && $arg2=="ROWID"} { return SQLITE_IGNORE } @@ -2181,7 +2181,7 @@ do_test auth-2.9.1 { # db cache flush - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_READ" && $arg1=="t2" && $arg2=="ROWID"} { return bogus } @@ -2193,7 +2193,7 @@ do_test auth-2.9.2 { db errorcode } {1} do_test auth-2.10 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_SELECT"} { return bogus } @@ -2202,7 +2202,7 @@ do_test auth-2.10 { catchsql {SELECT ROWID,b,c FROM t2} } {1 {authorizer malfunction}} do_test auth-2.11.1 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_READ" && $arg2=="a"} { return SQLITE_IGNORE } @@ -2211,7 +2211,7 @@ do_test auth-2.11.1 { catchsql {SELECT * FROM t2, t3} } {0 {{} 2 33 44 55 66 {} 8 9 44 55 66}} do_test auth-2.11.2 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_READ" && $arg2=="x"} { return SQLITE_IGNORE } @@ -2224,7 +2224,7 @@ do_test auth-2.11.2 { # ifcapable trigger { do_test auth-3.1 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { return SQLITE_OK } execsql { @@ -2237,7 +2237,7 @@ ifcapable trigger { } } {11 12 2 2 33 33 7 8 8 8 9 9} do_test auth-3.2 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_READ" && $arg1=="t2" && $arg2=="c"} { return SQLITE_IGNORE } @@ -2255,7 +2255,7 @@ ifcapable trigger { # ifcapable trigger { do_test auth-4.1 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { lappend ::authargs $code $arg1 $arg2 $arg3 $arg4 return SQLITE_OK } @@ -2340,7 +2340,7 @@ do_test auth-4.5 { # clause. # do_test auth-5.1 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { return SQLITE_OK } execsql { @@ -2393,7 +2393,7 @@ ifcapable trigger { } {} set ::authargs [list] proc auth {args} { - eval lappend ::authargs $args + eval lappend ::authargs [lrange $args 0 4] return SQLITE_OK } do_test auth-5.3.2 { @@ -2419,7 +2419,7 @@ do_test auth-6.1 { } {} set ::authargs [list] proc auth {args} { - eval lappend ::authargs $args + eval lappend ::authargs [lrange $args 0 4] return SQLITE_OK } do_test auth-6.2 { diff --git a/test/auth2.test b/test/auth2.test index 65e0591249..a9d64d08af 100644 --- a/test/auth2.test +++ b/test/auth2.test @@ -31,7 +31,7 @@ do_test auth2-1.1 { INSERT INTO t1 VALUES(1,2,3); } set ::flist {} - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_FUNCTION"} { lappend ::flist $arg2 if {$arg2=="max"} { @@ -80,7 +80,7 @@ sqlite3 db test.db sqlite3 db2 test.db proc auth {args} { global authargs - append authargs $args\n + append authargs [lrange $args 0 4]\n return SQLITE_OK } db auth auth diff --git a/test/auth3.test b/test/auth3.test index 21e2b3b65d..eef10b398f 100644 --- a/test/auth3.test +++ b/test/auth3.test @@ -30,7 +30,7 @@ if {[catch {db auth {}} msg]} { db cache size 0 db authorizer ::auth -proc auth {code arg1 arg2 arg3 arg4} { +proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE"} { return $::authcode } diff --git a/test/autoindex1.test b/test/autoindex1.test index 6cb0ab146a..bcde5bc2e7 100644 --- a/test/autoindex1.test +++ b/test/autoindex1.test @@ -413,4 +413,101 @@ do_execsql_test autoindex1-801 { WHERE mimetypes._id=10 AND data14 IS NOT NULL; } {/SEARCH TABLE data .*SEARCH TABLE raw_contacts/} +# Another test case from an important user of SQLite. The key feature of +# this test is that the "aggindex" subquery should make use of an +# automatic index. If it does, the query is fast. If it does not, the +# query is deathly slow. It worked OK in 3.7.17 but started going slow +# with version 3.8.0. The problem was fixed for 3.8.7 by reducing the +# cost estimate for automatic indexes on views and subqueries. +# +db close +forcedelete test.db +sqlite3 db test.db +do_execsql_test autoindex1-900 { + CREATE TABLE messages (ROWID INTEGER PRIMARY KEY AUTOINCREMENT, message_id, document_id BLOB, in_reply_to, remote_id INTEGER, sender INTEGER, subject_prefix, subject INTEGER, date_sent INTEGER, date_received INTEGER, date_created INTEGER, date_last_viewed INTEGER, mailbox INTEGER, remote_mailbox INTEGER, original_mailbox INTEGER, flags INTEGER, read, flagged, size INTEGER, color, encoding, type INTEGER, pad, conversation_id INTEGER DEFAULT -1, snippet TEXT DEFAULT NULL, fuzzy_ancestor INTEGER DEFAULT NULL, automated_conversation INTEGER DEFAULT 0, root_status INTEGER DEFAULT -1, conversation_position INTEGER DEFAULT -1); + CREATE INDEX date_index ON messages(date_received); + CREATE INDEX date_last_viewed_index ON messages(date_last_viewed); + CREATE INDEX date_created_index ON messages(date_created); + CREATE INDEX message_message_id_mailbox_index ON messages(message_id, mailbox); + CREATE INDEX message_document_id_index ON messages(document_id); + CREATE INDEX message_read_index ON messages(read); + CREATE INDEX message_flagged_index ON messages(flagged); + CREATE INDEX message_mailbox_index ON messages(mailbox, date_received); + CREATE INDEX message_remote_mailbox_index ON messages(remote_mailbox, remote_id); + CREATE INDEX message_type_index ON messages(type); + CREATE INDEX message_conversation_id_conversation_position_index ON messages(conversation_id, conversation_position); + CREATE INDEX message_fuzzy_ancestor_index ON messages(fuzzy_ancestor); + CREATE INDEX message_subject_fuzzy_ancestor_index ON messages(subject, fuzzy_ancestor); + CREATE INDEX message_sender_subject_automated_conversation_index ON messages(sender, subject, automated_conversation); + CREATE INDEX message_sender_index ON messages(sender); + CREATE INDEX message_root_status ON messages(root_status); + CREATE TABLE subjects (ROWID INTEGER PRIMARY KEY, subject COLLATE RTRIM, normalized_subject COLLATE RTRIM); + CREATE INDEX subject_subject_index ON subjects(subject); + CREATE INDEX subject_normalized_subject_index ON subjects(normalized_subject); + CREATE TABLE addresses (ROWID INTEGER PRIMARY KEY, address COLLATE NOCASE, comment, UNIQUE(address, comment)); + CREATE INDEX addresses_address_index ON addresses(address); + CREATE TABLE mailboxes (ROWID INTEGER PRIMARY KEY, url UNIQUE, total_count INTEGER DEFAULT 0, unread_count INTEGER DEFAULT 0, unseen_count INTEGER DEFAULT 0, deleted_count INTEGER DEFAULT 0, unread_count_adjusted_for_duplicates INTEGER DEFAULT 0, change_identifier, source INTEGER, alleged_change_identifier); + CREATE INDEX mailboxes_source_index ON mailboxes(source); + CREATE TABLE labels (ROWID INTEGER PRIMARY KEY, message_id INTEGER NOT NULL, mailbox_id INTEGER NOT NULL, UNIQUE(message_id, mailbox_id)); + CREATE INDEX labels_message_id_mailbox_id_index ON labels(message_id, mailbox_id); + CREATE INDEX labels_mailbox_id_index ON labels(mailbox_id); + + explain query plan + SELECT messages.ROWID, + messages.message_id, + messages.remote_id, + messages.date_received, + messages.date_sent, + messages.flags, + messages.size, + messages.color, + messages.date_last_viewed, + messages.subject_prefix, + subjects.subject, + sender.comment, + sender.address, + NULL, + messages.mailbox, + messages.original_mailbox, + NULL, + NULL, + messages.type, + messages.document_id, + sender, + NULL, + messages.conversation_id, + messages.conversation_position, + agglabels.labels + FROM mailboxes AS mailbox + JOIN messages ON mailbox.ROWID = messages.mailbox + LEFT OUTER JOIN subjects ON messages.subject = subjects.ROWID + LEFT OUTER JOIN addresses AS sender ON messages.sender = sender.ROWID + LEFT OUTER JOIN ( + SELECT message_id, group_concat(mailbox_id) as labels + FROM labels GROUP BY message_id + ) AS agglabels ON messages.ROWID = agglabels.message_id + WHERE (mailbox.url = 'imap://email.app@imap.gmail.com/%5BGmail%5D/All%20Mail') + AND (messages.ROWID IN ( + SELECT labels.message_id + FROM labels JOIN mailboxes ON labels.mailbox_id = mailboxes.ROWID + WHERE mailboxes.url = 'imap://email.app@imap.gmail.com/INBOX')) + AND messages.mailbox in (6,12,18,24,30,36,42,1,7,13,19,25,31,37,43,2,8, + 14,20,26,32,38,3,9,15,21,27,33,39,4,10,16,22,28, + 34,40,5,11,17,23,35,41) + ORDER BY date_received DESC; +} {/agglabels USING AUTOMATIC COVERING INDEX/} + +# A test case for VIEWs +# +do_execsql_test autoindex1-901 { + CREATE TABLE t1(x INTEGER PRIMARY KEY, y, z); + CREATE TABLE t2(a, b); + CREATE VIEW agg2 AS SELECT a, sum(b) AS m FROM t2 GROUP BY a; + EXPLAIN QUERY PLAN + SELECT t1.z, agg2.m + FROM t1 JOIN agg2 ON t1.y=agg2.m + WHERE t1.x IN (1,2,3); +} {/USING AUTOMATIC COVERING INDEX/} + + finish_test diff --git a/test/autoindex2.test b/test/autoindex2.test index f4da416745..c8abdf410a 100644 --- a/test/autoindex2.test +++ b/test/autoindex2.test @@ -224,48 +224,4 @@ do_execsql_test autoindex2-120 { # on t3 and reordering the tables so that t3 was in the outer loop and # implementing the ORDER BY clause using a B-Tree. -do_execsql_test autoindex2-120 { - EXPLAIN QUERY PLAN - SELECT - t1_id, - t1.did, - param2, - param3, - t1.ptime, - t1.trange, - t1.exmass, - t1.mass, - t1.vstatus, - type, - subtype, - t1.deviation, - t1.formula, - dparam1, - reserve1, - reserve2, - param4, - t1.last_operation, - t1.admin_uuid, - t1.previous_value, - t1.job_id, - client_did, - t1.last_t1, - t1.data_t1, - t1.previous_date, - param5, - param6, - mgr_uuid - FROM - t3, - t2, - t1 - WHERE - t1.ptime > 1393520400 - AND param3<>9001 - AND t3.flg7 = 1 - AND t1.did = t2.did - AND t2.uid = t3.uid - ORDER BY t1.ptime desc LIMIT 500; -} {0 0 2 {SEARCH TABLE t1 USING INDEX t1x1 (ptime>?)} 0 1 1 {SEARCH TABLE t2 USING INDEX t2x0 (did=?)} 0 2 0 {SEARCH TABLE t3 USING INDEX t3x0 (uid=?)}} - finish_test diff --git a/test/autoindex3.test b/test/autoindex3.test index 33053bba67..c99a175c6d 100644 --- a/test/autoindex3.test +++ b/test/autoindex3.test @@ -17,6 +17,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +set testprefix autoindex3 # The t1b and t2d indexes are not very selective. It used to be that # the autoindex mechanism would create automatic indexes on t1(b) or @@ -54,5 +55,38 @@ do_execsql_test autoindex3-140 { EXPLAIN QUERY PLAN SELECT * FROM t1, t2 WHERE d IN (5,b) AND x=y; } {/AUTO/} +reset_db +do_execsql_test 210 { + CREATE TABLE v(b, d, e); + CREATE TABLE u(a, b, c); + ANALYZE sqlite_master; + INSERT INTO "sqlite_stat1" VALUES('u','uab','40000 400 1'); + INSERT INTO "sqlite_stat1" VALUES('v','vbde','40000 400 1 1'); + INSERT INTO "sqlite_stat1" VALUES('v','ve','40000 21'); + + CREATE INDEX uab on u(a, b); + CREATE INDEX ve on v(e); + CREATE INDEX vbde on v(b,d,e); + + DROP TABLE IF EXISTS sqlite_stat4; + ANALYZE sqlite_master; +} + +# At one point, SQLite was using the inferior plan: +# +# 0|0|1|SEARCH TABLE v USING INDEX ve (e>?) +# 0|1|0|SEARCH TABLE u USING COVERING INDEX uab (ANY(a) AND b=?) +# +# on the basis that the real index "uab" must be better than the automatic +# index. This is not right - a skip-scan is not necessarily better than an +# automatic index scan. +# +do_eqp_test 220 { + select count(*) from u, v where u.b = v.b and v.e > 34; +} { + 0 0 1 {SEARCH TABLE v USING INDEX ve (e>?)} + 0 1 0 {SEARCH TABLE u USING AUTOMATIC COVERING INDEX (b=?)} +} + finish_test diff --git a/test/autoindex4.test b/test/autoindex4.test new file mode 100644 index 0000000000..6d0865bf72 --- /dev/null +++ b/test/autoindex4.test @@ -0,0 +1,52 @@ +# 2014-10-24 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# This file implements regression tests for SQLite library. The +# focus of this script is testing automatic index creation logic, +# and specifically creation of automatic partial indexes. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +do_execsql_test autoindex4-1.0 { + CREATE TABLE t1(a,b); + INSERT INTO t1 VALUES(123,'abc'),(234,'def'),(234,'ghi'),(345,'jkl'); + CREATE TABLE t2(x,y); + INSERT INTO t2 VALUES(987,'zyx'),(654,'wvu'),(987,'rqp'); + + SELECT *, '|' FROM t1, t2 WHERE a=234 AND x=987 ORDER BY +b; +} {234 def 987 rqp | 234 def 987 zyx | 234 ghi 987 rqp | 234 ghi 987 zyx |} +do_execsql_test autoindex4-1.1 { + SELECT *, '|' FROM t1, t2 WHERE a=234 AND x=555; +} {} + +do_execsql_test autoindex4-1.2 { + SELECT *, '|' FROM t1 LEFT JOIN t2 ON a=234 AND x=555; +} {123 abc {} {} | 234 def {} {} | 234 ghi {} {} | 345 jkl {} {} |} +do_execsql_test autoindex4-1.3 { + SELECT *, '|' FROM t1 LEFT JOIN t2 ON x=555 WHERE a=234; +} {234 def {} {} | 234 ghi {} {} |} +do_execsql_test autoindex4-1.4 { + SELECT *, '|' FROM t1 LEFT JOIN t2 WHERE a=234 AND x=555; +} {} + + +do_execsql_test autoindex4-2.0 { + CREATE TABLE t3(e,f); + INSERT INTO t3 VALUES(123,654),(555,444),(234,987); + + SELECT (SELECT count(*) FROM t1, t2 WHERE a=e AND x=f), e, f, '|' + FROM t3 + ORDER BY rowid; +} {1 123 654 | 0 555 444 | 4 234 987 |} + +finish_test diff --git a/test/backup.test b/test/backup.test index 444619c68c..3b1e1db9e5 100644 --- a/test/backup.test +++ b/test/backup.test @@ -217,6 +217,7 @@ foreach nPagePerStep {1 200} { INSERT INTO ${file_dest}.t1 VALUES(1, randstr(1000,1000)) " $db_dest } + execsql COMMIT $db_dest } # Backup the source database. diff --git a/test/backup5.test b/test/backup5.test new file mode 100644 index 0000000000..c789adfa69 --- /dev/null +++ b/test/backup5.test @@ -0,0 +1,65 @@ +# 2014 November 13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix backup5 + +forcedelete test2.db + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b); + CREATE TABLE t2(a, b); + INSERT INTO t2 VALUES(1, 1); + INSERT INTO t2 VALUES(2, 2); + INSERT INTO t2 VALUES(3, 3); +} + +do_test 1.1 { + forcecopy test.db test.db2 + db eval { + DROP TABLE t2; + INSERT INTO t1 VALUES(zeroblob(1000), zeroblob(1000)); + INSERT INTO t1 VALUES(randomblob(1000), randomblob(1000)); + } +} {} + +do_test 1.2 { + sqlite3 db2 test.db2 + set stmt [sqlite3_prepare_v2 db2 "SELECT * FROM t2" -1 dummy] + sqlite3_step $stmt +} {SQLITE_ROW} + +do_test 1.3 { + list [catch { sqlite3_backup B db2 main db main } msg] $msg +} {1 {sqlite3_backup_init() failed}} + +do_test 1.4 { + sqlite3_errmsg db2 +} {destination database is in use} + +do_test 1.5 { + sqlite3_reset $stmt + sqlite3_backup B db2 main db main + B step 200 + B finish +} {SQLITE_OK} + +do_test 1.6 { + list [sqlite3_step $stmt] [sqlite3_finalize $stmt] +} {SQLITE_ERROR SQLITE_ERROR} + +do_test 1.7 { + sqlite3_errmsg db2 +} {no such table: t2} + +finish_test diff --git a/test/bigsort.test b/test/bigsort.test new file mode 100644 index 0000000000..c711515973 --- /dev/null +++ b/test/bigsort.test @@ -0,0 +1,50 @@ +# 2014 November 26 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix bigsort + +#-------------------------------------------------------------------- +# At one point there was an overflow problem if the product of the +# cache-size and page-size was larger than 2^31. Causing an infinite +# loop if the product was also an integer multiple of 2^32, or +# inefficiency otherwise. +# +# This test causes thrashing on machines with smaller amounts of +# memory. Make sure the host has at least 8GB available before running +# this test. +# +if {[catch {exec free | grep Mem:} out] || [lindex $out 1]<8000000} { + finish_test + return +} + +do_execsql_test 1.0 { + PRAGMA page_size = 1024; + CREATE TABLE t1(a, b); + BEGIN; + WITH data(x,y) AS ( + SELECT 1, zeroblob(10000) + UNION ALL + SELECT x+1, y FROM data WHERE x < 300000 + ) + INSERT INTO t1 SELECT * FROM data; + COMMIT; +} +do_execsql_test 1.1 { + PRAGMA cache_size = 4194304; + CREATE INDEX i1 ON t1(a, b); +} + + +finish_test diff --git a/test/btree01.test b/test/btree01.test new file mode 100644 index 0000000000..25f2c6897b --- /dev/null +++ b/test/btree01.test @@ -0,0 +1,132 @@ +# 2014-11-27 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains test cases for b-tree logic. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix btree01 + +# The refactoring on the b-tree balance() routine in check-in +# http://www.sqlite.org/src/info/face33bea1ba3a (2014-10-27) +# caused the integrity_check on the following SQL to fail. +# +do_execsql_test btree01-1.1 { + PRAGMA page_size=65536; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b BLOB); + WITH RECURSIVE + c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<30) + INSERT INTO t1(a,b) SELECT i, zeroblob(6500) FROM c; + UPDATE t1 SET b=zeroblob(3000); + UPDATE t1 SET b=zeroblob(64000) WHERE a=2; + PRAGMA integrity_check; +} {ok} + +# The previous test is sufficient to prevent a regression. But we +# add a number of additional tests to stress the balancer in similar +# ways, looking for related problems. +# +for {set i 1} {$i<=30} {incr i} { + do_test btree01-1.2.$i { + db eval { + DELETE FROM t1; + WITH RECURSIVE + c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<30) + INSERT INTO t1(a,b) SELECT i, zeroblob(6500) FROM c; + UPDATE t1 SET b=zeroblob(3000); + UPDATE t1 SET b=zeroblob(64000) WHERE a=$::i; + PRAGMA integrity_check; + } + } {ok} +} +for {set i 1} {$i<=30} {incr i} { + do_test btree01-1.3.$i { + db eval { + DELETE FROM t1; + WITH RECURSIVE + c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<30) + INSERT INTO t1(a,b) SELECT i, zeroblob(6500) FROM c; + UPDATE t1 SET b=zeroblob(2000); + UPDATE t1 SET b=zeroblob(64000) WHERE a=$::i; + PRAGMA integrity_check; + } + } {ok} +} +for {set i 1} {$i<=30} {incr i} { + do_test btree01-1.4.$i { + db eval { + DELETE FROM t1; + WITH RECURSIVE + c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<30) + INSERT INTO t1(a,b) SELECT i, zeroblob(6500) FROM c; + UPDATE t1 SET b=zeroblob(6499) WHERE (a%3)==0; + UPDATE t1 SET b=zeroblob(6499) WHERE (a%3)==1; + UPDATE t1 SET b=zeroblob(6499) WHERE (a%3)==2; + UPDATE t1 SET b=zeroblob(64000) WHERE a=$::i; + PRAGMA integrity_check; + } + } {ok} +} +for {set i 1} {$i<=30} {incr i} { + do_test btree01-1.5.$i { + db eval { + DELETE FROM t1; + WITH RECURSIVE + c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<30) + INSERT INTO t1(a,b) SELECT i, zeroblob(6542) FROM c; + UPDATE t1 SET b=zeroblob(2331); + UPDATE t1 SET b=zeroblob(65496) WHERE a=$::i; + PRAGMA integrity_check; + } + } {ok} +} +for {set i 1} {$i<=30} {incr i} { + do_test btree01-1.6.$i { + db eval { + DELETE FROM t1; + WITH RECURSIVE + c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<30) + INSERT INTO t1(a,b) SELECT i, zeroblob(6542) FROM c; + UPDATE t1 SET b=zeroblob(2332); + UPDATE t1 SET b=zeroblob(65496) WHERE a=$::i; + PRAGMA integrity_check; + } + } {ok} +} +for {set i 1} {$i<=30} {incr i} { + do_test btree01-1.7.$i { + db eval { + DELETE FROM t1; + WITH RECURSIVE + c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<30) + INSERT INTO t1(a,b) SELECT i, zeroblob(6500) FROM c; + UPDATE t1 SET b=zeroblob(1); + UPDATE t1 SET b=zeroblob(65000) WHERE a=$::i; + PRAGMA integrity_check; + } + } {ok} +} +for {set i 1} {$i<=31} {incr i} { + do_test btree01-1.8.$i { + db eval { + DELETE FROM t1; + WITH RECURSIVE + c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<31) + INSERT INTO t1(a,b) SELECT i, zeroblob(6500) FROM c; + UPDATE t1 SET b=zeroblob(4000); + UPDATE t1 SET b=zeroblob(65000) WHERE a=$::i; + PRAGMA integrity_check; + } + } {ok} +} + +finish_test diff --git a/test/capi3.test b/test/capi3.test index cbaa5c5e5b..9f3d6f6916 100644 --- a/test/capi3.test +++ b/test/capi3.test @@ -912,7 +912,7 @@ do_test capi3-11.9.3 { } 1 do_test capi3-11.10 { sqlite3_step $STMT -} {SQLITE_ERROR} +} {SQLITE_ROW} ifcapable !autoreset { # If SQLITE_OMIT_AUTORESET is defined, then the statement must be # reset() before it can be passed to step() again. @@ -921,11 +921,11 @@ ifcapable !autoreset { } do_test capi3-11.11 { sqlite3_step $STMT -} {SQLITE_ROW} +} {SQLITE_DONE} do_test capi3-11.12 { sqlite3_step $STMT sqlite3_step $STMT -} {SQLITE_DONE} +} {SQLITE_ROW} do_test capi3-11.13 { sqlite3_finalize $STMT } {SQLITE_OK} diff --git a/test/capi3c.test b/test/capi3c.test index 6388717e00..6ab3bc24f6 100644 --- a/test/capi3c.test +++ b/test/capi3c.test @@ -863,7 +863,7 @@ do_test capi3c-11.9.3 { } 1 do_test capi3c-11.10 { sqlite3_step $STMT -} {SQLITE_ABORT} +} {SQLITE_ROW} ifcapable !autoreset { # If SQLITE_OMIT_AUTORESET is defined, then the statement must be # reset() before it can be passed to step() again. @@ -872,11 +872,11 @@ ifcapable !autoreset { } do_test capi3c-11.11 { sqlite3_step $STMT -} {SQLITE_ROW} +} {SQLITE_DONE} do_test capi3c-11.12 { sqlite3_step $STMT sqlite3_step $STMT -} {SQLITE_DONE} +} {SQLITE_ROW} do_test capi3c-11.13 { sqlite3_finalize $STMT } {SQLITE_OK} diff --git a/test/capi3d.test b/test/capi3d.test index ed3765b051..1459c5abfe 100644 --- a/test/capi3d.test +++ b/test/capi3d.test @@ -144,4 +144,40 @@ do_test capi3d-3.99 { sqlite3_stmt_busy 0 } {0} +#-------------------------------------------------------------------------- +# Test the sqlite3_stmt_busy() function with ROLLBACK statements. +# +reset_db + +do_execsql_test capi3d-4.1 { + CREATE TABLE t4(x,y); + BEGIN; +} + +do_test capi3d-4.2.1 { + set ::s1 [sqlite3_prepare_v2 db "ROLLBACK" -1 notused] + sqlite3_step $::s1 +} {SQLITE_DONE} + +do_test capi3d-4.2.2 { + sqlite3_stmt_busy $::s1 +} {1} + +do_catchsql_test capi3d-4.2.3 { + VACUUM +} {1 {cannot VACUUM - SQL statements in progress}} + +do_test capi3d-4.2.4 { + sqlite3_reset $::s1 +} {SQLITE_OK} + +do_catchsql_test capi3d-4.2.5 { + VACUUM +} {0 {}} + +do_test capi3d-4.2.6 { + sqlite3_finalize $::s1 +} {SQLITE_OK} + + finish_test diff --git a/test/colmeta.test b/test/colmeta.test index 3939f8228a..21b0e0f38a 100644 --- a/test/colmeta.test +++ b/test/colmeta.test @@ -17,17 +17,14 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -ifcapable !columnmetadata { - finish_test - return -} - # Set up a schema in the main and temp test databases. do_test colmeta-0 { execsql { CREATE TABLE abc(a, b, c); CREATE TABLE abc2(a PRIMARY KEY COLLATE NOCASE, b VARCHAR(32), c); CREATE TABLE abc3(a NOT NULL, b INTEGER PRIMARY KEY, c); + CREATE TABLE abc5(w,x,y,z,PRIMARY KEY(x,z)) WITHOUT ROWID; + CREATE TABLE abc6(rowid TEXT COLLATE rtrim, oid REAL, _rowid_ BLOB); } ifcapable autoinc { execsql { @@ -57,19 +54,27 @@ set tests { 13 {main abc rowid} {0 {INTEGER BINARY 0 1 0}} 14 {main abc3 rowid} {0 {INTEGER BINARY 0 1 0}} 16 {main abc d} {1 {no such table column: abc.d}} + 20 {main abc5 w} {0 {{} BINARY 0 0 0}} + 21 {main abc5 x} {0 {{} BINARY 1 1 0}} + 22 {main abc5 y} {0 {{} BINARY 0 0 0}} + 23 {main abc5 z} {0 {{} BINARY 1 1 0}} + 24 {main abc5 rowid} {1 {no such table column: abc5.rowid}} + 30 {main abc6 rowid} {0 {TEXT rtrim 0 0 0}} + 31 {main abc6 oid} {0 {REAL BINARY 0 0 0}} + 32 {main abc6 _rowid_} {0 {BLOB BINARY 0 0 0}} } -ifcapable view { +ifcapable autoinc { set tests [concat $tests { - 8 {{} abc4 b} {0 {INTEGER BINARY 0 1 1}} - 15 {main abc4 rowid} {0 {INTEGER BINARY 0 1 1}} + 100 {{} abc4 b} {0 {INTEGER BINARY 0 1 1}} + 101 {main abc4 rowid} {0 {INTEGER BINARY 0 1 1}} }] } ifcapable view { set tests [concat $tests { - 9 {{} v1 a} {1 {no such table column: v1.a}} - 10 {main v1 b} {1 {no such table column: v1.b}} - 11 {main v1 badname} {1 {no such table column: v1.badname}} - 12 {main v1 rowid} {1 {no such table column: v1.rowid}} + 200 {{} v1 a} {1 {no such table column: v1.a}} + 201 {main v1 b} {1 {no such table column: v1.b}} + 202 {main v1 badname} {1 {no such table column: v1.badname}} + 203 {main v1 rowid} {1 {no such table column: v1.rowid}} }] } @@ -91,4 +96,15 @@ foreach {tn params results} $tests { } $results } +# Calling sqlite3_table_column_metadata with a NULL column name merely +# checks for the existance of the table. +# +do_test colmeta-300 { + catch {sqlite3_table_column_metadata $::DB main xyzzy} res +} {1} +do_test colmeta-301 { + catch {sqlite3_table_column_metadata $::DB main abc} res +} {0} + + finish_test diff --git a/test/corruptH.test b/test/corruptH.test index ee2bb1ee48..5c83cb3b90 100644 --- a/test/corruptH.test +++ b/test/corruptH.test @@ -64,7 +64,6 @@ do_test 1.2 { } {} do_test 1.3 { -breakpoint db eval { PRAGMA secure_delete=1 } list [catch { db eval { SELECT * FROM t1 WHERE a IN (1, 2) } { diff --git a/test/corruptI.test b/test/corruptI.test index ed34c0f8c3..c8d0176236 100644 --- a/test/corruptI.test +++ b/test/corruptI.test @@ -75,5 +75,34 @@ do_test 2.2 { catchsql { SELECT * FROM r WHERE x >= 10 } } {1 {database disk image is malformed}} - +if {[db one {SELECT sqlite_compileoption_used('ENABLE_OVERSIZE_CELL_CHECK')}]} { + # The following tests only work if OVERSIZE_CELL_CHECK is disabled +} else { + reset_db + do_execsql_test 3.1 { + PRAGMA auto_vacuum=0; + PRAGMA page_size = 512; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + WITH s(a, b) AS ( + SELECT 2, 'abcdefghij' + UNION ALL + SELECT a+2, b FROM s WHERe a < 40 + ) + INSERT INTO t1 SELECT * FROM s; + } {} + + do_test 3.2 { + hexio_write test.db [expr 512+3] 0054 + db close + sqlite3 db test.db + execsql { INSERT INTO t1 VALUES(5, 'klmnopqrst') } + execsql { INSERT INTO t1 VALUES(7, 'klmnopqrst') } + } {} + + db close + sqlite3 db test.db + do_catchsql_test 3.3 { + INSERT INTO t1 VALUES(9, 'klmnopqrst'); + } {1 {database disk image is malformed}} +} ;# end-if !defined(ENABLE_OVERSIZE_CELL_CHECK) finish_test diff --git a/test/default.test b/test/default.test index d6b6f97d98..406eb53677 100644 --- a/test/default.test +++ b/test/default.test @@ -99,4 +99,32 @@ do_execsql_test default-3.3 { SELECT * FROM t300; } {2147483647 2147483648 9223372036854775807 -2147483647 -2147483648 -9223372036854775808 9.22337203685478e+18 9223372036854775807} +# Do now allow bound parameters in new DEFAULT values. +# Silently convert bound parameters to NULL in DEFAULT causes +# in the sqlite_master table, for backwards compatibility. +# +db close +forcedelete test.db +sqlite3 db test.db +do_execsql_test default-4.0 { + CREATE TABLE t1(a TEXT, b TEXT DEFAULT(99)); + PRAGMA writable_schema=ON; + UPDATE sqlite_master SET sql='CREATE TABLE t1(a TEXT, b TEXT DEFAULT(:xyz))'; +} {} +db close +sqlite3 db test.db +do_execsql_test default-4.1 { + INSERT INTO t1(a) VALUES('xyzzy'); + SELECT a, quote(b) FROM t1; +} {xyzzy NULL} +do_catchsql_test default-4.2 { + CREATE TABLE t2(a TEXT, b TEXT DEFAULT(:xyz)); +} {1 {default value of column [b] is not constant}} +do_catchsql_test default-4.3 { + CREATE TABLE t2(a TEXT, b TEXT DEFAULT(abs(:xyz))); +} {1 {default value of column [b] is not constant}} +do_catchsql_test default-4.4 { + CREATE TABLE t2(a TEXT, b TEXT DEFAULT(98+coalesce(5,:xyz))); +} {1 {default value of column [b] is not constant}} + finish_test diff --git a/test/distinct.test b/test/distinct.test index 78c2c1df36..2fb90dc3e3 100644 --- a/test/distinct.test +++ b/test/distinct.test @@ -222,4 +222,34 @@ do_execsql_test 4.1 { SELECT quote(x) FROM t2 ORDER BY 1; } {'xyzzy' X'0000000000'} +#---------------------------------------------------------------------------- +# Ticket [c5ea805691bfc4204b1cb9e9aa0103bd48bc7d34] (2014-12-04) +# Make sure that DISTINCT works together with ORDER BY and descending +# indexes. +# +do_execsql_test 5.1 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(x); + INSERT INTO t1(x) VALUES(3),(1),(5),(2),(6),(4),(5),(1),(3); + CREATE INDEX t1x ON t1(x DESC); + SELECT DISTINCT x FROM t1 ORDER BY x ASC; +} {1 2 3 4 5 6} +do_execsql_test 5.2 { + SELECT DISTINCT x FROM t1 ORDER BY x DESC; +} {6 5 4 3 2 1} +do_execsql_test 5.3 { + SELECT DISTINCT x FROM t1 ORDER BY x; +} {1 2 3 4 5 6} +do_execsql_test 5.4 { + DROP INDEX t1x; + CREATE INDEX t1x ON t1(x ASC); + SELECT DISTINCT x FROM t1 ORDER BY x ASC; +} {1 2 3 4 5 6} +do_execsql_test 5.5 { + SELECT DISTINCT x FROM t1 ORDER BY x DESC; +} {6 5 4 3 2 1} +do_execsql_test 5.6 { + SELECT DISTINCT x FROM t1 ORDER BY x; +} {1 2 3 4 5 6} + finish_test diff --git a/test/e_blobbytes.test b/test/e_blobbytes.test new file mode 100644 index 0000000000..a6283ab852 --- /dev/null +++ b/test/e_blobbytes.test @@ -0,0 +1,76 @@ +# 2014 October 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix e_blobbytes + +do_execsql_test 1.0 { + CREATE TABLE q1(r INTEGER PRIMARY KEY, s TEXT); + WITH d(a, b) AS ( + SELECT 0, '' + UNION ALL + SELECT a+1, b||'.' FROM d WHERE a<10000 + ) + INSERT INTO q1 SELECT * FROM d; +} + + +# EVIDENCE-OF: R-07796-55423 Returns the size in bytes of the BLOB +# accessible via the successfully opened BLOB handle in its only +# argument. +# +proc check_blob_size {tn rowid bytes} { + uplevel [list do_test $tn [subst -nocommands { + sqlite3_blob_open db main q1 s $rowid 0 B + set res [sqlite3_blob_bytes [set B]] + sqlite3_blob_close [set B] + set res + }] $bytes] +} +check_blob_size 1.1 43 43 +check_blob_size 1.2 391 391 +check_blob_size 1.3 6349 6349 +check_blob_size 1.4 2621 2621 +check_blob_size 1.5 7771 7771 +check_blob_size 1.6 7949 7949 +check_blob_size 1.7 4374 4374 +check_blob_size 1.8 2578 2578 +check_blob_size 1.9 7004 7004 +check_blob_size 1.10 2180 2180 +check_blob_size 1.11 3796 3796 +check_blob_size 1.12 7101 7101 +check_blob_size 1.13 7449 7449 +check_blob_size 1.14 7224 7224 +check_blob_size 1.15 3038 3038 +check_blob_size 1.16 1083 1083 +check_blob_size 1.17 5157 5157 +check_blob_size 1.18 6686 6686 +check_blob_size 1.19 6592 6592 +check_blob_size 1.20 0 0 + + +# EVIDENCE-OF: R-53088-19343 The incremental blob I/O routines can only +# read or overwriting existing blob content; they cannot change the size +# of a blob. +# +# Also demonstrated in other e_blobXXX.test files. +# +do_test 2.1 { + sqlite3_blob_open db main q1 s 86 1 B + list [catch { sqlite3_blob_write $B 86 "1" 1 } msg] $msg +} {1 SQLITE_ERROR} +sqlite3_blob_close $B + +finish_test + + diff --git a/test/e_blobclose.test b/test/e_blobclose.test new file mode 100644 index 0000000000..a5d432d3b5 --- /dev/null +++ b/test/e_blobclose.test @@ -0,0 +1,171 @@ +# 2014 October 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix e_blobclose + +set dots [string repeat . 40] +do_execsql_test 1.0 { + CREATE TABLE x1(a INTEGER PRIMARY KEY, b DOTS); + INSERT INTO x1 VALUES(-1, $dots); + INSERT INTO x1 VALUES(-10, $dots); + INSERT INTO x1 VALUES(-100, $dots); + INSERT INTO x1 VALUES(-1000, $dots); + INSERT INTO x1 VALUES(-10000, $dots); +} + +# EVIDENCE-OF: R-03145-46390 This function closes an open BLOB handle. +# +# It's not clear how to test that a blob handle really is closed. +# Attempting to use a closed blob handle will likely crash the process. +# Assume here that if the SHARED lock on the db file is released, +# the blob handle has been closed. +# +do_execsql_test 1.1 { PRAGMA lock_status } {main unlocked temp closed} +sqlite3_blob_open db main x1 b -1 0 B +do_execsql_test 1.2 { PRAGMA lock_status } {main shared temp closed} +sqlite3_blob_close $B +do_execsql_test 1.3 { PRAGMA lock_status } {main unlocked temp closed} + + +# EVIDENCE-OF: R-34027-00617 If the blob handle being closed was opened +# for read-write access, and if the database is in auto-commit mode and +# there are no other open read-write blob handles or active write +# statements, the current transaction is committed. +# +# 2.1.*: Transaction is not committed if there are other open +# read-write blob handles. +# +# 2.2.*: Transaction is not committed if not in auto-commit mode. +# +# 2.3.*: Active write statements. +# +do_test 2.1.1 { + sqlite3_blob_open db main x1 b -100 1 B1 + sqlite3_blob_open db main x1 b -1000 1 B2 + sqlite3_blob_open db main x1 b -10000 1 B3 + sqlite3_blob_open db main x1 b -10000 0 B4 ;# B4 is read-only! + execsql { PRAGMA lock_status } +} {main reserved temp closed} +do_test 2.1.2 { + sqlite3_blob_close $B1 + execsql { PRAGMA lock_status } +} {main reserved temp closed} +do_test 2.1.3 { + sqlite3_blob_close $B2 + execsql { PRAGMA lock_status } +} {main reserved temp closed} +do_test 2.1.4 { + sqlite3_blob_close $B3 + execsql { PRAGMA lock_status } +} {main shared temp closed} +do_test 2.1.5 { + sqlite3_blob_close $B4 + execsql { PRAGMA lock_status } +} {main unlocked temp closed} + +do_test 2.2.1 { + sqlite3_blob_open db main x1 b -100 1 B1 + execsql { PRAGMA lock_status } +} {main reserved temp closed} +do_test 2.2.2 { + execsql { BEGIN } + sqlite3_blob_close $B1 + execsql { PRAGMA lock_status } +} {main reserved temp closed} +do_test 2.2.3 { + execsql { COMMIT } + execsql { PRAGMA lock_status } +} {main unlocked temp closed} + +proc val {} { + sqlite3_blob_close $::B + db eval { PRAGMA lock_status } +} +db func val val +do_test 2.3.1 { + sqlite3_blob_open db main x1 b -100 1 B + execsql { PRAGMA lock_status } +} {main reserved temp closed} +do_test 2.3.2 { + execsql { INSERT INTO x1 VALUES(15, val()) } + execsql { PRAGMA lock_status } +} {main unlocked temp closed} +do_test 2.3.3 { + execsql { SELECT * FROM x1 WHERE a = 15 } +} {15 {main reserved temp closed}} + +# A reader does not inhibit commit. +do_test 2.3.4 { + sqlite3_blob_open db main x1 b -100 1 B + execsql { PRAGMA lock_status } +} {main reserved temp closed} +do_test 2.3.5 { + execsql { SELECT a, val() FROM x1 LIMIT 1 } +} {-10000 {main shared temp closed}} + + +do_test 3.1 { + sqlite3_blob_open db main x1 b -10 1 B + execsql { + INSERT INTO x1 VALUES(1, 'abc'); + SELECT * FROM x1 WHERE a=1; + } +} {1 abc} +do_test 3.2 { + sqlite3_blob_write $B 0 "abcdefghij" 10 + execsql { SELECT * FROM x1 WHERE a=-10 } +} {-10 abcdefghij..............................} + +do_test 3.3 { + sqlite3 db2 test.db + execsql { BEGIN ; SELECT * FROM x1 } db2 + sqlite3_blob_close $B +} {SQLITE_BUSY} + +# EVIDENCE-OF: R-41959-38737 Otherwise, if this function is passed a +# valid open blob handle, the values returned by the sqlite3_errcode() +# and sqlite3_errmsg() functions are set before returning. +# +do_test 3.4 { + list [sqlite3_errcode db] [sqlite3_errmsg db] +} {SQLITE_BUSY {database is locked}} + +# EVIDENCE-OF: R-37801-37633 The BLOB handle is closed unconditionally. +# Even if this routine returns an error code, the handle is still +# closed. +# +# Test that the lock has been released. Assume this means the handle +# is closed, even though blob_close() returned SQLITE_BUSY. +# +do_execsql_test 3.4 { PRAGMA lock_status } {main unlocked temp closed} + +# EVIDENCE-OF: R-35111-05628 If an error occurs while committing the +# transaction, an error code is returned and the transaction rolled +# back. +# +# Row 1 is removed (it was inserted this transaction) and row -10 +# is restored to its original state. Transaction has been rolled back. +# +do_execsql_test 3.5 { + SELECT * FROM x1 WHERE a IN (1, -10); +} {-10 ........................................} + +# EVIDENCE-OF: R-25894-51060 Calling this routine with a null pointer +# (such as would be returned by a failed call to sqlite3_blob_open()) is +# a harmless no-op. +# +do_test 4.0 { sqlite3_blob_close 0 } {} + +finish_test + diff --git a/test/e_blobopen.test b/test/e_blobopen.test new file mode 100644 index 0000000000..01f62cdd78 --- /dev/null +++ b/test/e_blobopen.test @@ -0,0 +1,549 @@ +# 2014 October 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix e_blobopen + +forcedelete test.db2 + +do_execsql_test 1.0 { + ATTACH 'test.db2' AS aux; + + CREATE TABLE main.t1(a INTEGER PRIMARY KEY, b TEXT, c BLOB); + CREATE TEMP TABLE t1(a INTEGER PRIMARY KEY, b TEXT, c BLOB); + CREATE TABLE aux.t1(a INTEGER PRIMARY KEY, b TEXT, c BLOB); + + CREATE TABLE main.x1(a INTEGER PRIMARY KEY, b TEXT, c BLOB); + CREATE TEMP TABLE x2(a INTEGER PRIMARY KEY, b TEXT, c BLOB); + CREATE TABLE aux.x3(a INTEGER PRIMARY KEY, b TEXT, c BLOB); + + INSERT INTO main.t1 VALUES(1, 'main one', X'0101'); + INSERT INTO main.t1 VALUES(2, 'main two', X'0102'); + INSERT INTO main.t1 VALUES(3, 'main three', X'0103'); + INSERT INTO main.t1 VALUES(4, 'main four', X'0104'); + INSERT INTO main.t1 VALUES(5, 'main five', X'0105'); + + INSERT INTO main.x1 VALUES(1, 'x main one', X'000101'); + INSERT INTO main.x1 VALUES(2, 'x main two', X'000102'); + INSERT INTO main.x1 VALUES(3, 'x main three', X'000103'); + INSERT INTO main.x1 VALUES(4, 'x main four', X'000104'); + INSERT INTO main.x1 VALUES(5, 'x main five', X'000105'); + + INSERT INTO temp.t1 VALUES(1, 'temp one', X'0201'); + INSERT INTO temp.t1 VALUES(2, 'temp two', X'0202'); + INSERT INTO temp.t1 VALUES(3, 'temp three', X'0203'); + INSERT INTO temp.t1 VALUES(4, 'temp four', X'0204'); + INSERT INTO temp.t1 VALUES(5, 'temp five', X'0205'); + + INSERT INTO temp.x2 VALUES(1, 'x temp one', X'000201'); + INSERT INTO temp.x2 VALUES(2, 'x temp two', X'000202'); + INSERT INTO temp.x2 VALUES(3, 'x temp three', X'000203'); + INSERT INTO temp.x2 VALUES(4, 'x temp four', X'000204'); + INSERT INTO temp.x2 VALUES(5, 'x temp five', X'000205'); + + INSERT INTO aux.t1 VALUES(1, 'aux one', X'0301'); + INSERT INTO aux.t1 VALUES(2, 'aux two', X'0302'); + INSERT INTO aux.t1 VALUES(3, 'aux three', X'0303'); + INSERT INTO aux.t1 VALUES(4, 'aux four', X'0304'); + INSERT INTO aux.t1 VALUES(5, 'aux five', X'0305'); + + INSERT INTO aux.x3 VALUES(1, 'x aux one', X'000301'); + INSERT INTO aux.x3 VALUES(2, 'x aux two', X'000302'); + INSERT INTO aux.x3 VALUES(3, 'x aux three', X'000303'); + INSERT INTO aux.x3 VALUES(4, 'x aux four', X'000304'); + INSERT INTO aux.x3 VALUES(5, 'x aux five', X'000305'); +} + +#------------------------------------------------------------------------- +# EVIDENCE-OF: R-37639-55938 This interfaces opens a handle to the BLOB +# located in row iRow, column zColumn, table zTable in database zDb; in +# other words, the same BLOB that would be selected by: SELECT zColumn +# FROM zDb.zTable WHERE rowid = iRow; +# +proc read_blob {zDb zTab zCol iRow} { + sqlite3_blob_open db $zDb $zTab $zCol $iRow 0 B + set nByte [sqlite3_blob_bytes $B] + set data [sqlite3_blob_read $B 0 $nByte] + sqlite3_blob_close $B + return $data +} + +do_test 1.1.1 { read_blob main t1 b 1 } "main one" +do_test 1.1.2 { read_blob main t1 c 1 } "\01\01" +do_test 1.1.3 { read_blob temp t1 b 1 } "temp one" +do_test 1.1.4 { read_blob temp t1 c 1 } "\02\01" +do_test 1.1.6 { read_blob aux t1 b 1 } "aux one" +do_test 1.1.7 { read_blob aux t1 c 1 } "\03\01" + +do_test 1.2.1 { read_blob main t1 b 4 } "main four" +do_test 1.2.2 { read_blob main t1 c 4 } "\01\04" +do_test 1.2.3 { read_blob temp t1 b 4 } "temp four" +do_test 1.2.4 { read_blob temp t1 c 4 } "\02\04" +do_test 1.2.6 { read_blob aux t1 b 4 } "aux four" +do_test 1.2.7 { read_blob aux t1 c 4 } "\03\04" + +do_test 1.3.1 { read_blob main x1 b 2 } "x main two" +do_test 1.3.2 { read_blob main x1 c 2 } "\00\01\02" +do_test 1.3.3 { read_blob temp x2 b 2 } "x temp two" +do_test 1.3.4 { read_blob temp x2 c 2 } "\00\02\02" +do_test 1.3.6 { read_blob aux x3 b 2 } "x aux two" +do_test 1.3.7 { read_blob aux x3 c 2 } "\00\03\02" + +#------------------------------------------------------------------------- +# EVIDENCE-OF: R-27234-05761 Parameter zDb is not the filename that +# contains the database, but rather the symbolic name of the database. +# For attached databases, this is the name that appears after the AS +# keyword in the ATTACH statement. For the main database file, the +# database name is "main". For TEMP tables, the database name is "temp". +# +# The test cases immediately above demonstrate that the database name +# for the main db, for TEMP tables and for those in attached databases +# is correct. The following tests check that filenames cannot be +# used as well. +# +do_test 2.1 { + list [catch { sqlite3_blob_open db "test.db" t1 b 1 0 B } msg] $msg +} {1 SQLITE_ERROR} +do_test 2.2 { + list [catch { sqlite3_blob_open db "test.db2" t1 b 1 0 B } msg] $msg +} {1 SQLITE_ERROR} + +#------------------------------------------------------------------------- +# EVIDENCE-OF: R-50854-53979 If the flags parameter is non-zero, then +# the BLOB is opened for read and write access. +# +# EVIDENCE-OF: R-03922-41160 If the flags parameter is zero, the BLOB is +# opened for read-only access. +# +foreach {tn iRow flags} { + 1 1 0 + 2 2 1 + 3 3 -1 + 4 4 2147483647 + 5 5 -2147483648 +} { + do_test 3.$tn.1 { + sqlite3_blob_open db main x1 c $iRow $flags B + set n [sqlite3_blob_bytes $B] + sqlite3_blob_read $B 0 $n + } [binary format ccc 0 1 $iRow] + + if {$flags==0} { + # Blob was opened for read-only access - writing returns an error. + do_test 3.$tn.2 { + list [catch { sqlite3_blob_write $B 0 xxx 3 } msg] $msg + } {1 SQLITE_READONLY} + + do_execsql_test 3.$tn.3 { + SELECT c FROM x1 WHERE a=$iRow; + } [binary format ccc 0 1 $iRow] + } else { + # Blob was opened for read/write access - writing succeeds + do_test 3.$tn.4 { + list [catch { sqlite3_blob_write $B 0 xxx 3 } msg] $msg + } {0 {}} + + do_execsql_test 3.$tn.5 { + SELECT c FROM x1 WHERE a=$iRow; + } {xxx} + } + + sqlite3_blob_close $B +} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 4.0 { + CREATE TABLE t1(x, y); + INSERT INTO t1 VALUES('abcd', 152); + INSERT INTO t1 VALUES(NULL, X'00010203'); + INSERT INTO t1 VALUES('', 154.2); + + CREATE TABLE t2(x PRIMARY KEY, y) WITHOUT ROWID; + INSERT INTO t2 VALUES(1, 'blob'); + + CREATE TABLE t3(a PRIMARY KEY, b, c, d, e, f, UNIQUE(e, f)); + INSERT INTO t3 VALUES('aaaa', 'bbbb', 'cccc', 'dddd', 'eeee', 'ffff'); + CREATE INDEX t3b ON t3(b); + + CREATE TABLE p1(x PRIMARY KEY); + INSERT INTO p1 VALUES('abc'); + + CREATE TABLE c1(a INTEGER PRIMARY KEY, b REFERENCES p1); + INSERT INTO c1 VALUES(45, 'abc'); +} + +proc test_blob_open {tn zDb zTab zCol iRow flags errcode errmsg} { + global B + set B "0x1234" + + if {$errcode=="SQLITE_OK"} { + set expected "0 {}" + } else { + set expected "1 $errcode" + } + + set ::res [list [ + catch { sqlite3_blob_open db $zDb $zTab $zCol $iRow $flags B } msg + ] $msg] + do_test 4.$tn.1 { set ::res } $expected + + # EVIDENCE-OF: R-08940-21305 Unless it returns SQLITE_MISUSE, this + # function sets the database connection error code and message + # accessible via sqlite3_errcode() and sqlite3_errmsg() and related + # functions. + # + # This proc (test_blob_open) is used below to test various error and + # non-error conditions. But never SQLITE_MISUSE conditions. So these + # test cases are considered as partly verifying the requirement above. + # See below for a test of the SQLITE_MISUSE case. + # + do_test 4.$tn.2 { + sqlite3_errcode db + } $errcode + do_test 4.$tn.3 { + sqlite3_errmsg db + } $errmsg + + # EVIDENCE-OF: R-31086-35521 On success, SQLITE_OK is returned and the + # new BLOB handle is stored in *ppBlob. Otherwise an error code is + # returned and, unless the error code is SQLITE_MISUSE, *ppBlob is set + # to NULL. + # + do_test 4.$tn.4 { + expr {$B == "0"} + } [expr {$errcode != "SQLITE_OK"}] + + # EVIDENCE-OF: R-63421-15521 This means that, provided the API is not + # misused, it is always safe to call sqlite3_blob_close() on *ppBlob + # after this function it returns. + do_test 4.$tn.5 { + sqlite3_blob_close $B + } {} +} + +# EVIDENCE-OF: R-31204-44780 Database zDb does not exist +test_blob_open 1 nosuchdb t1 x 1 0 SQLITE_ERROR "no such table: nosuchdb.t1" + +# EVIDENCE-OF: R-28676-08005 Table zTable does not exist within database zDb +test_blob_open 2 main tt1 x 1 0 SQLITE_ERROR "no such table: main.tt1" + +# EVIDENCE-OF: R-40134-30296 Table zTable is a WITHOUT ROWID table +test_blob_open 3 main t2 y 1 0 SQLITE_ERROR \ + "cannot open table without rowid: t2" + +# EVIDENCE-OF: R-56376-21261 Column zColumn does not exist +test_blob_open 4 main t1 z 2 0 SQLITE_ERROR "no such column: \"z\"" + +# EVIDENCE-OF: R-28258-23166 Row iRow is not present in the table +test_blob_open 5 main t1 y 6 0 SQLITE_ERROR "no such rowid: 6" + +# EVIDENCE-OF: R-11683-62380 The specified column of row iRow contains a +# value that is not a TEXT or BLOB value +test_blob_open 6 main t1 x 2 0 SQLITE_ERROR "cannot open value of type null" +test_blob_open 7 main t1 y 1 0 SQLITE_ERROR "cannot open value of type integer" +test_blob_open 8 main t1 y 3 0 SQLITE_ERROR "cannot open value of type real" + +# EVIDENCE-OF: R-34146-30782 Column zColumn is part of an index, PRIMARY +# KEY or UNIQUE constraint and the blob is being opened for read/write +# access +# +# Test cases 8.1.* show that such columns can be opened for read-access. +# Tests 8.2.* show that read-write access is different. Columns "c" and "c" +# are not part of an index, PK or UNIQUE constraint, so they work in both +# cases. +# +test_blob_open 8.1.1 main t3 a 1 0 SQLITE_OK "not an error" +test_blob_open 8.1.2 main t3 b 1 0 SQLITE_OK "not an error" +test_blob_open 8.1.3 main t3 c 1 0 SQLITE_OK "not an error" +test_blob_open 8.1.4 main t3 d 1 0 SQLITE_OK "not an error" +test_blob_open 8.1.5 main t3 e 1 0 SQLITE_OK "not an error" +test_blob_open 8.1.6 main t3 f 1 0 SQLITE_OK "not an error" + +set cannot "cannot open indexed column for writing" +test_blob_open 8.2.1 main t3 a 1 8 SQLITE_ERROR $cannot +test_blob_open 8.2.2 main t3 b 1 8 SQLITE_ERROR $cannot +test_blob_open 8.2.3 main t3 c 1 8 SQLITE_OK "not an error" +test_blob_open 8.2.4 main t3 d 1 8 SQLITE_OK "not an error" +test_blob_open 8.2.5 main t3 e 1 8 SQLITE_ERROR $cannot +test_blob_open 8.2.6 main t3 f 1 8 SQLITE_ERROR $cannot + +# EVIDENCE-OF: R-50117-55204 Foreign key constraints are enabled, column +# zColumn is part of a child key definition and the blob is being opened +# for read/write access +# +# 9.1: FK disabled, read-only access. +# 9.2: FK disabled, read-only access. +# 9.3: FK enabled, read/write access. +# 9.4: FK enabled, read/write access. +# +test_blob_open 9.1 main c1 b 45 0 SQLITE_OK "not an error" +test_blob_open 9.2 main c1 b 45 1 SQLITE_OK "not an error" +execsql { PRAGMA foreign_keys = ON } +test_blob_open 9.3 main c1 b 45 0 SQLITE_OK "not an error" +test_blob_open 9.4 main c1 b 45 1 SQLITE_ERROR \ + "cannot open foreign key column for writing" + +#------------------------------------------------------------------------- +# EVIDENCE-OF: R-08940-21305 Unless it returns SQLITE_MISUSE, this +# function sets the database connection error code and message +# accessible via sqlite3_errcode() and sqlite3_errmsg() and related +# functions. +# +# This requirement is partially verified by the many uses of test +# command [test_blob_open] above. All that is left is to verify the +# SQLITE_MISUSE case. +# +# SQLITE_MISUSE is only returned if SQLITE_ENABLE_API_ARMOR is defined +# during compilation. +# +ifcapable api_armor { + sqlite3_blob_open db main t1 x 1 0 B + + do_test 10.1.1 { + list [catch {sqlite3_blob_open $B main t1 x 1 0 B2} msg] $msg + } {1 SQLITE_MISUSE} + do_test 10.1.2 { + list [sqlite3_errcode db] [sqlite3_errmsg db] + } {SQLITE_OK {not an error}} + sqlite3_blob_close $B + + do_test 10.2.1 { + list [catch {sqlite3_blob_open db main {} x 1 0 B} msg] $msg + } {1 SQLITE_MISUSE} + do_test 10.2.2 { + list [sqlite3_errcode db] [sqlite3_errmsg db] + } {SQLITE_OK {not an error}} +} + +#------------------------------------------------------------------------- +# EVIDENCE-OF: R-50542-62589 If the row that a BLOB handle points to is +# modified by an UPDATE, DELETE, or by ON CONFLICT side-effects then the +# BLOB handle is marked as "expired". This is true if any column of the +# row is changed, even a column other than the one the BLOB handle is +# open on. +# +# EVIDENCE-OF: R-48367-20048 Calls to sqlite3_blob_read() and +# sqlite3_blob_write() for an expired BLOB handle fail with a return +# code of SQLITE_ABORT. +# +# 11.2: read-only handle, DELETE. +# 11.3: read-only handle, UPDATE. +# 11.4: read-only handle, REPLACE. +# 11.5: read/write handle, DELETE. +# 11.6: read/write handle, UPDATE. +# 11.7: read/write handle, REPLACE. +# +do_execsql_test 11.1 { + CREATE TABLE b1(a INTEGER PRIMARY KEY, b, c UNIQUE); + INSERT INTO b1 VALUES(1, '1234567890', 1); + INSERT INTO b1 VALUES(2, '1234567890', 2); + INSERT INTO b1 VALUES(3, '1234567890', 3); + INSERT INTO b1 VALUES(4, '1234567890', 4); + INSERT INTO b1 VALUES(5, '1234567890', 5); + INSERT INTO b1 VALUES(6, '1234567890', 6); + + CREATE TABLE b2(a INTEGER PRIMARY KEY, b, c UNIQUE); + INSERT INTO b2 VALUES(1, '1234567890', 1); + INSERT INTO b2 VALUES(2, '1234567890', 2); + INSERT INTO b2 VALUES(3, '1234567890', 3); + INSERT INTO b2 VALUES(4, '1234567890', 4); + INSERT INTO b2 VALUES(5, '1234567890', 5); + INSERT INTO b2 VALUES(6, '1234567890', 6); +} + +do_test 11.2.1 { + sqlite3_blob_open db main b1 b 2 0 B + sqlite3_blob_read $B 0 10 +} {1234567890} +do_test 11.2.2 { + # Deleting a different row does not invalidate the blob handle. + execsql { DELETE FROM b1 WHERE a = 1 } + sqlite3_blob_read $B 0 10 +} {1234567890} +do_test 11.2.3 { + execsql { DELETE FROM b1 WHERE a = 2 } + list [catch { sqlite3_blob_read $B 0 10 } msg] $msg +} {1 SQLITE_ABORT} +do_test 11.2.4 { + sqlite3_blob_close $B +} {} + +do_test 11.3.1 { + sqlite3_blob_open db main b1 b 3 0 B + sqlite3_blob_read $B 0 10 +} {1234567890} +do_test 11.3.2 { + # Updating a different row + execsql { UPDATE b1 SET c = 42 WHERE a=4 } + sqlite3_blob_read $B 0 10 +} {1234567890} +do_test 11.3.3 { + execsql { UPDATE b1 SET c = 43 WHERE a=3 } + list [catch { sqlite3_blob_read $B 0 10 } msg] $msg +} {1 SQLITE_ABORT} +do_test 11.3.4 { + sqlite3_blob_close $B +} {} + +do_test 11.4.1 { + sqlite3_blob_open db main b1 b 6 0 B + sqlite3_blob_read $B 0 10 +} {1234567890} +do_test 11.4.2 { + # Replace a different row + execsql { INSERT OR REPLACE INTO b1 VALUES(10, 'abcdefghij', 5) } + sqlite3_blob_read $B 0 10 +} {1234567890} +do_test 11.4.3 { + execsql { INSERT OR REPLACE INTO b1 VALUES(11, 'abcdefghij', 6) } + list [catch { sqlite3_blob_read $B 0 10 } msg] $msg +} {1 SQLITE_ABORT} +do_test 11.4.4 { + sqlite3_blob_close $B +} {} + +do_test 11.4.1 { + sqlite3_blob_open db main b2 b 2 1 B + sqlite3_blob_write $B 0 "abcdefghij" +} {} +do_test 11.4.2 { + # Deleting a different row does not invalidate the blob handle. + execsql { DELETE FROM b2 WHERE a = 1 } + sqlite3_blob_write $B 0 "ABCDEFGHIJ" +} {} +do_test 11.4.3 { + execsql { DELETE FROM b2 WHERE a = 2 } + list [catch { sqlite3_blob_write $B 0 "0987654321" } msg] $msg +} {1 SQLITE_ABORT} +do_test 11.4.4 { + sqlite3_blob_close $B +} {} + +do_test 11.5.1 { + sqlite3_blob_open db main b2 b 3 1 B + sqlite3_blob_write $B 0 "abcdefghij" +} {} +do_test 11.5.2 { + # Updating a different row + execsql { UPDATE b2 SET c = 42 WHERE a=4 } + sqlite3_blob_write $B 0 "ABCDEFGHIJ" +} {} +do_test 11.5.3 { + execsql { UPDATE b2 SET c = 43 WHERE a=3 } + list [catch { sqlite3_blob_write $B 0 "0987654321" } msg] $msg +} {1 SQLITE_ABORT} +do_test 11.5.4 { + sqlite3_blob_close $B +} {} + +do_test 11.6.1 { + sqlite3_blob_open db main b2 b 6 1 B + sqlite3_blob_write $B 0 "abcdefghij" +} {} +do_test 11.6.2 { + # Replace a different row + execsql { INSERT OR REPLACE INTO b2 VALUES(10, 'abcdefghij', 5) } + sqlite3_blob_write $B 0 "ABCDEFGHIJ" +} {} +do_test 11.6.3 { + execsql { INSERT OR REPLACE INTO b2 VALUES(11, 'abcdefghij', 6) } + list [catch { sqlite3_blob_write $B 0 "0987654321" } msg] $msg +} {1 SQLITE_ABORT} +do_test 11.6.4 { + sqlite3_blob_close $B +} {} + +#------------------------------------------------------------------------- +# EVIDENCE-OF: R-45408-40694 Changes written into a BLOB prior to the +# BLOB expiring are not rolled back by the expiration of the BLOB. Such +# changes will eventually commit if the transaction continues to +# completion. +# +do_execsql_test 12.1 { + CREATE TABLE b3(x INTEGER PRIMARY KEY, y TEXT, z INTEGER); + INSERT INTO b3 VALUES(22, '..........', NULL); +} +do_test 12.2 { + sqlite3_blob_open db main b3 y 22 1 B + sqlite3_blob_write $B 0 "xxxxx" 5 +} {} +do_execsql_test 12.3 { + UPDATE b3 SET z = 'not null'; +} +do_test 12.4 { + list [catch {sqlite3_blob_write $B 5 "xxxxx" 5} msg] $msg +} {1 SQLITE_ABORT} +do_execsql_test 12.5 { + SELECT * FROM b3; +} {22 xxxxx..... {not null}} +do_test 12.5 { + sqlite3_blob_close $B +} {} +do_execsql_test 12.6 { + SELECT * FROM b3; +} {22 xxxxx..... {not null}} + +#------------------------------------------------------------------------- +# EVIDENCE-OF: R-58813-55036 The sqlite3_bind_zeroblob() and +# sqlite3_result_zeroblob() interfaces and the built-in zeroblob SQL +# function may be used to create a zero-filled blob to read or write +# using the incremental-blob interface. +# +do_execsql_test 13.1 { + CREATE TABLE c2(i INTEGER PRIMARY KEY, j); + INSERT INTO c2 VALUES(10, zeroblob(24)); +} + +do_test 13.2 { + set stmt [sqlite3_prepare_v2 db "INSERT INTO c2 VALUES(11, ?)" -1] + sqlite3_bind_zeroblob $stmt 1 45 + sqlite3_step $stmt + sqlite3_finalize $stmt +} {SQLITE_OK} + +# The blobs can be read: +# +do_test 13.3.1 { + sqlite3_blob_open db main c2 j 10 1 B + sqlite3_blob_open db main c2 j 11 1 B2 + list [sqlite3_blob_bytes $B] [sqlite3_blob_bytes $B2] +} {24 45} +do_test 13.3.2 { + sqlite3_blob_read $B 0 24 +} [string repeat [binary format c 0] 24] +do_test 13.3.3 { + sqlite3_blob_read $B2 0 45 +} [string repeat [binary format c 0] 45] + +# And also written: +# +do_test 13.4.1 { + sqlite3_blob_write $B 0 [string repeat [binary format c 1] 24] +} {} +do_test 13.4.2 { + sqlite3_blob_write $B2 0 [string repeat [binary format c 1] 45] +} {} +do_test 13.5 { + sqlite3_blob_close $B + sqlite3_blob_close $B2 + execsql { SELECT j FROM c2 } +} [list \ + [string repeat [binary format c 1] 24] \ + [string repeat [binary format c 1] 45] \ +] + + +finish_test + diff --git a/test/e_blobwrite.test b/test/e_blobwrite.test new file mode 100644 index 0000000000..a0d33336de --- /dev/null +++ b/test/e_blobwrite.test @@ -0,0 +1,204 @@ +# 2014 October 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix e_blobwrite + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-62898-22698 This function is used to write data into an +# open BLOB handle from a caller-supplied buffer. N bytes of data are +# copied from the buffer Z into the open BLOB, starting at offset +# iOffset. +# +set dots [string repeat . 40] +do_execsql_test 1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, t TEXT); + INSERT INTO t1 VALUES(-1, $dots); + INSERT INTO t1 VALUES(-2, $dots); + INSERT INTO t1 VALUES(-3, $dots); + INSERT INTO t1 VALUES(-4, $dots); + INSERT INTO t1 VALUES(-5, $dots); + INSERT INTO t1 VALUES(-6, $dots); +} + +proc blob_write_test {tn id iOffset blob nData final} { + sqlite3_blob_open db main t1 t $id 1 B + + # EVIDENCE-OF: R-45864-01884 On success, sqlite3_blob_write() returns + # SQLITE_OK. Otherwise, an error code or an extended error code is + # returned. + # + # This block tests the SQLITE_OK case in the requirement above (the + # Tcl sqlite3_blob_write() wrapper uses an empty string in place of + # "SQLITE_OK"). The error cases are tested by the "blob_write_error_test" + # tests below. + # + set res [sqlite3_blob_write $B $iOffset $blob $nData] + uplevel [list do_test $tn.1 [list set {} $res] {}] + + sqlite3_blob_close $B + uplevel [list do_execsql_test $tn.3 "SELECT t FROM t1 WHERE a=$id" $final] +} + +set blob "0123456789012345678901234567890123456789" +blob_write_test 1.1 -1 0 $blob 10 { 0123456789.............................. } +blob_write_test 1.2 -2 8 $blob 10 { ........0123456789...................... } +blob_write_test 1.3 -3 8 $blob 1 { ........0............................... } +blob_write_test 1.4 -4 18 $blob 22 { ..................0123456789012345678901 } +blob_write_test 1.5 -5 18 $blob 0 { ........................................ } +blob_write_test 1.6 -6 0 $blob 40 { 0123456789012345678901234567890123456789 } + + +proc blob_write_error_test {tn B iOffset blob nData errcode errmsg} { + + # In cases where the underlying sqlite3_blob_write() function returns + # SQLITE_OK, the Tcl wrapper returns an empty string. If the underlying + # function returns an error, the Tcl wrapper throws an exception with + # the error code as the Tcl exception message. + # + if {$errcode=="SQLITE_OK"} { + set ret "" + set isError 0 + } else { + set ret $errcode + set isError 1 + } + + set cmd [list sqlite3_blob_write $B $iOffset $blob $nData] + uplevel [list do_test $tn.1 [subst -nocommands { + list [catch {$cmd} msg] [set msg] + }] [list $isError $ret]] + + # EVIDENCE-OF: R-34782-18311 Unless SQLITE_MISUSE is returned, this + # function sets the database connection error code and message + # accessible via sqlite3_errcode() and sqlite3_errmsg() and related + # functions. + # + if {$errcode == "SQLITE_MISUSE"} { error "test proc misuse!" } + uplevel [list do_test $tn.2 [list sqlite3_errcode db] $errcode] + uplevel [list do_test $tn.3 [list sqlite3_errmsg db] $errmsg] +} + +do_execsql_test 2.0 { + CREATE TABLE t2(a TEXT, b INTEGER PRIMARY KEY); + INSERT INTO t2 VALUES($dots, 43); + INSERT INTO t2 VALUES($dots, 44); + INSERT INTO t2 VALUES($dots, 45); +} + +# EVIDENCE-OF: R-63341-57517 If the BLOB handle passed as the first +# argument was not opened for writing (the flags parameter to +# sqlite3_blob_open() was zero), this function returns SQLITE_READONLY. +# +sqlite3_blob_open db main t2 a 43 0 B +blob_write_error_test 2.1 $B 0 $blob 10 \ + SQLITE_READONLY {attempt to write a readonly database} +sqlite3_blob_close $B + +# EVIDENCE-OF: R-29804-27366 If offset iOffset is less than N bytes from +# the end of the BLOB, SQLITE_ERROR is returned and no data is written. +# +sqlite3_blob_open db main t2 a 44 3 B +blob_write_error_test 2.2.1 $B 31 $blob 10 \ + SQLITE_ERROR {SQL logic error or missing database} + +# Make a successful write to the blob handle. This shows that the +# sqlite3_errcode() and sqlite3_errmsg() values are set even if the +# blob_write() call succeeds (see requirement in the [blob_write_error_test] +# proc). +blob_write_error_test 2.2.1 $B 30 $blob 10 SQLITE_OK {not an error} + +# EVIDENCE-OF: R-58570-38916 If N or iOffset are less than zero +# SQLITE_ERROR is returned and no data is written. +# +blob_write_error_test 2.2.2 $B 31 $blob -1 \ + SQLITE_ERROR {SQL logic error or missing database} +blob_write_error_test 2.2.3 $B 20 $blob 10 SQLITE_OK {not an error} +blob_write_error_test 2.2.4 $B -1 $blob 10 \ + SQLITE_ERROR {SQL logic error or missing database} +sqlite3_blob_close $B + +# EVIDENCE-OF: R-20958-54138 An attempt to write to an expired BLOB +# handle fails with an error code of SQLITE_ABORT. +# +do_test 2.3 { + sqlite3_blob_open db main t2 a 43 0 B + execsql { DELETE FROM t2 WHERE b=43 } +} {} +blob_write_error_test 2.3.1 $B 5 $blob 5 \ + SQLITE_ABORT {callback requested query abort} +do_test 2.3.2 { + execsql { SELECT 1, 2, 3 } + sqlite3_errcode db +} {SQLITE_OK} +blob_write_error_test 2.3.3 $B 5 $blob 5 \ + SQLITE_ABORT {callback requested query abort} +sqlite3_blob_close $B + +# EVIDENCE-OF: R-08382-59936 Writes to the BLOB that occurred before the +# BLOB handle expired are not rolled back by the expiration of the +# handle, though of course those changes might have been overwritten by +# the statement that expired the BLOB handle or by other independent +# statements. +# +# 3.1.*: not rolled back, +# 3.2.*: overwritten. +# +do_execsql_test 3.0 { + CREATE TABLE t3(i INTEGER PRIMARY KEY, j TEXT, k TEXT); + INSERT INTO t3 VALUES(1, $dots, $dots); + INSERT INTO t3 VALUES(2, $dots, $dots); + SELECT * FROM t3 WHERE i=1; +} { + 1 + ........................................ + ........................................ +} +sqlite3_blob_open db main t3 j 1 1 B +blob_write_error_test 3.1.1 $B 5 $blob 10 SQLITE_OK {not an error} +do_execsql_test 3.1.2 { + UPDATE t3 SET k = 'xyz' WHERE i=1; + SELECT * FROM t3 WHERE i=1; +} { + 1 .....0123456789......................... xyz +} +blob_write_error_test 3.1.3 $B 15 $blob 10 \ + SQLITE_ABORT {callback requested query abort} +sqlite3_blob_close $B +do_execsql_test 3.1.4 { + SELECT * FROM t3 WHERE i=1; +} { + 1 .....0123456789......................... xyz +} + +sqlite3_blob_open db main t3 j 2 1 B +blob_write_error_test 3.2.1 $B 5 $blob 10 SQLITE_OK {not an error} +do_execsql_test 3.2.2 { + UPDATE t3 SET j = 'xyz' WHERE i=2; + SELECT * FROM t3 WHERE i=2; +} { + 2 xyz ........................................ +} +blob_write_error_test 3.2.3 $B 15 $blob 10 \ + SQLITE_ABORT {callback requested query abort} +sqlite3_blob_close $B +do_execsql_test 3.2.4 { + SELECT * FROM t3 WHERE i=2; +} { + 2 xyz ........................................ +} + + + +finish_test + diff --git a/test/e_changes.test b/test/e_changes.test new file mode 100644 index 0000000000..a77e22a2ee --- /dev/null +++ b/test/e_changes.test @@ -0,0 +1,441 @@ +# 2011 October 28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix e_changes + +# Like [do_execsql_test], except it appends the value returned by +# [db changes] to the result of executing the SQL script. +# +proc do_changes_test {tn sql res} { + uplevel [list \ + do_test $tn "concat \[execsql {$sql}\] \[db changes\]" $res + ] +} + + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-15996-49369 This function returns the number of rows +# modified, inserted or deleted by the most recently completed INSERT, +# UPDATE or DELETE statement on the database connection specified by the +# only parameter. +# +do_execsql_test 1.0 { + CREATE TABLE t1(a, b); + CREATE TABLE t2(x, y, PRIMARY KEY(x, y)) WITHOUT ROWID; + CREATE INDEX i1 ON t1(a); + CREATE INDEX i2 ON t2(y); +} +foreach {tn schema} { + 1 { + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(b); + } + 2 { + CREATE TABLE t1(a, b, PRIMARY KEY(a, b)) WITHOUT ROWID; + CREATE INDEX i1 ON t1(b); + } +} { + reset_db + execsql $schema + + # Insert 1 row. + do_changes_test 1.$tn.1 { INSERT INTO t1 VALUES(0, 0) } 1 + + # Insert 10 rows. + do_changes_test 1.$tn.2 { + WITH rows(i, j) AS ( + SELECT 1, 1 UNION ALL SELECT i+1, j+i FROM rows WHERE i<10 + ) + INSERT INTO t1 SELECT * FROM rows + } 10 + + # Modify 5 rows. + do_changes_test 1.$tn.3 { + UPDATE t1 SET b=b+1 WHERE a<5; + } 5 + + # Delete 4 rows + do_changes_test 1.$tn.4 { + DELETE FROM t1 WHERE a>6 + } 4 + + # Check the "on the database connecton specified" part of hte + # requirement - changes made by other connections do not show up in + # the return value of sqlite3_changes(). + do_test 1.$tn.5 { + sqlite3 db2 test.db + execsql { INSERT INTO t1 VALUES(-1, -1) } db2 + db2 changes + } 1 + do_test 1.$tn.6 { + db changes + } 4 + db2 close + + # Test that statements that modify no rows because they hit UNIQUE + # constraints set the sqlite3_changes() value to 0. Regardless of + # whether or not they are executed inside an explicit transaction. + # + # 1.$tn.8-9: outside of a transaction + # 1.$tn.10-12: inside a transaction + # + do_changes_test 1.$tn.7 { + CREATE UNIQUE INDEX i2 ON t1(a); + } 4 + do_catchsql_test 1.$tn.8 { + INSERT INTO t1 VALUES('a', 0), ('b', 0), ('c', 0), (0, 11); + } {1 {UNIQUE constraint failed: t1.a}} + do_test 1.$tn.9 { db changes } 0 + do_catchsql_test 1.$tn.10 { + BEGIN; + INSERT INTO t1 VALUES('a', 0), ('b', 0), ('c', 0), (0, 11); + } {1 {UNIQUE constraint failed: t1.a}} + do_test 1.$tn.11 { db changes } 0 + do_changes_test 1.$tn.12 COMMIT 0 + +} + + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-44877-05564 Executing any other type of SQL statement +# does not modify the value returned by this function. +# +reset_db +do_changes_test 2.1 { CREATE TABLE t1(x) } 0 +do_changes_test 2.2 { + WITH d(y) AS (SELECT 1 UNION ALL SELECT y+1 FROM d WHERE y<47) + INSERT INTO t1 SELECT y FROM d; +} 47 + +# The statement above set changes() to 47. Check that none of the following +# modify this. +do_changes_test 2.3 { SELECT count(x) FROM t1 } {47 47} +do_changes_test 2.4 { DROP TABLE t1 } 47 +do_changes_test 2.5 { CREATE TABLE t1(x) } 47 +do_changes_test 2.6 { ALTER TABLE t1 ADD COLUMN b } 47 + + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-53938-27527 Only changes made directly by the INSERT, +# UPDATE or DELETE statement are considered - auxiliary changes caused +# by triggers, foreign key actions or REPLACE constraint resolution are +# not counted. +# +# 3.1.*: triggers +# 3.2.*: foreign key actions +# 3.3.*: replace constraints +# +reset_db +do_execsql_test 3.1.0 { + CREATE TABLE log(x); + CREATE TABLE p1(one PRIMARY KEY, two); + + CREATE TRIGGER tr_ai AFTER INSERT ON p1 BEGIN + INSERT INTO log VALUES('insert'); + END; + CREATE TRIGGER tr_bd BEFORE DELETE ON p1 BEGIN + INSERT INTO log VALUES('delete'); + END; + CREATE TRIGGER tr_au AFTER UPDATE ON p1 BEGIN + INSERT INTO log VALUES('update'); + END; + +} + +do_changes_test 3.1.1 { + INSERT INTO p1 VALUES('a', 'A'), ('b', 'B'), ('c', 'C'); +} 3 +do_changes_test 3.1.2 { + UPDATE p1 SET two = two||two; +} 3 +do_changes_test 3.1.3 { + DELETE FROM p1 WHERE one IN ('a', 'c'); +} 2 +do_execsql_test 3.1.4 { + -- None of the inserts on table log were counted. + SELECT count(*) FROM log +} 8 + +do_execsql_test 3.2.0 { + DELETE FROM p1; + INSERT INTO p1 VALUES('a', 'A'), ('b', 'B'), ('c', 'C'); + + CREATE TABLE c1(a, b, FOREIGN KEY(a) REFERENCES p1 ON DELETE SET NULL); + CREATE TABLE c2(a, b, FOREIGN KEY(a) REFERENCES p1 ON DELETE SET DEFAULT); + CREATE TABLE c3(a, b, FOREIGN KEY(a) REFERENCES p1 ON DELETE CASCADE); + INSERT INTO c1 VALUES('a', 'aaa'); + INSERT INTO c2 VALUES('b', 'bbb'); + INSERT INTO c3 VALUES('c', 'ccc'); + + INSERT INTO p1 VALUES('d', 'D'), ('e', 'E'), ('f', 'F'); + CREATE TABLE c4(a, b, FOREIGN KEY(a) REFERENCES p1 ON UPDATE SET NULL); + CREATE TABLE c5(a, b, FOREIGN KEY(a) REFERENCES p1 ON UPDATE SET DEFAULT); + CREATE TABLE c6(a, b, FOREIGN KEY(a) REFERENCES p1 ON UPDATE CASCADE); + INSERT INTO c4 VALUES('d', 'aaa'); + INSERT INTO c5 VALUES('e', 'bbb'); + INSERT INTO c6 VALUES('f', 'ccc'); + + PRAGMA foreign_keys = ON; +} + +do_changes_test 3.2.1 { DELETE FROM p1 WHERE one = 'a' } 1 +do_changes_test 3.2.2 { DELETE FROM p1 WHERE one = 'b' } 1 +do_changes_test 3.2.3 { DELETE FROM p1 WHERE one = 'c' } 1 +do_execsql_test 3.2.4 { + SELECT * FROM c1; + SELECT * FROM c2; + SELECT * FROM c3; +} {{} aaa {} bbb} + +do_changes_test 3.2.5 { UPDATE p1 SET one = 'g' WHERE one = 'd' } 1 +do_changes_test 3.2.6 { UPDATE p1 SET one = 'h' WHERE one = 'e' } 1 +do_changes_test 3.2.7 { UPDATE p1 SET one = 'i' WHERE one = 'f' } 1 +do_execsql_test 3.2.8 { + SELECT * FROM c4; + SELECT * FROM c5; + SELECT * FROM c6; +} {{} aaa {} bbb i ccc} + +do_execsql_test 3.3.0 { + CREATE TABLE r1(a UNIQUE, b UNIQUE); + INSERT INTO r1 VALUES('i', 'i'); + INSERT INTO r1 VALUES('ii', 'ii'); + INSERT INTO r1 VALUES('iii', 'iii'); + INSERT INTO r1 VALUES('iv', 'iv'); + INSERT INTO r1 VALUES('v', 'v'); + INSERT INTO r1 VALUES('vi', 'vi'); + INSERT INTO r1 VALUES('vii', 'vii'); +} + +do_changes_test 3.3.1 { INSERT OR REPLACE INTO r1 VALUES('i', 1) } 1 +do_changes_test 3.3.2 { INSERT OR REPLACE INTO r1 VALUES('iv', 'v') } 1 +do_changes_test 3.3.3 { UPDATE OR REPLACE r1 SET b='v' WHERE a='iii' } 1 +do_changes_test 3.3.4 { UPDATE OR REPLACE r1 SET b='vi',a='vii' WHERE a='ii' } 1 +do_execsql_test 3.3.5 { + SELECT * FROM r1 ORDER BY a; +} {i 1 iii v vii vi} + + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-09813-48563 The value returned by sqlite3_changes() +# immediately after an INSERT, UPDATE or DELETE statement run on a view +# is always zero. +# +reset_db +do_execsql_test 4.1 { + CREATE TABLE log(log); + CREATE TABLE t1(x, y); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t1 VALUES(5, 6); + + CREATE VIEW v1 AS SELECT * FROM t1; + CREATE TRIGGER v1_i INSTEAD OF INSERT ON v1 BEGIN + INSERT INTO log VALUES('insert'); + END; + CREATE TRIGGER v1_u INSTEAD OF UPDATE ON v1 BEGIN + INSERT INTO log VALUES('update'), ('update'); + END; + CREATE TRIGGER v1_d INSTEAD OF DELETE ON v1 BEGIN + INSERT INTO log VALUES('delete'), ('delete'), ('delete'); + END; +} + +do_changes_test 4.2.1 { INSERT INTO t1 SELECT * FROM t1 } 3 +do_changes_test 4.2.2 { INSERT INTO v1 VALUES(1, 2) } 0 + +do_changes_test 4.3.1 { INSERT INTO t1 SELECT * FROM t1 } 6 +do_changes_test 4.3.2 { UPDATE v1 SET y='xyz' WHERE x=1 } 0 + +do_changes_test 4.4.1 { INSERT INTO t1 SELECT * FROM t1 } 12 +do_changes_test 4.4.2 { DELETE FROM v1 WHERE x=5 } 0 + + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-32918-61474 Before entering a trigger program the value +# returned by sqlite3_changes() function is saved. After the trigger +# program has finished, the original value is restored. +# +reset_db +db func my_changes my_changes +set ::changes [list] +proc my_changes {x} { + set res [db changes] + lappend ::changes $x $res + return $res +} + +do_execsql_test 5.1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + CREATE TABLE t2(x); + INSERT INTO t1 VALUES(1, NULL); + INSERT INTO t1 VALUES(2, NULL); + INSERT INTO t1 VALUES(3, NULL); + CREATE TRIGGER AFTER UPDATE ON t1 BEGIN + INSERT INTO t2 VALUES('a'), ('b'), ('c'); + SELECT my_changes('trigger'); + END; +} + +do_execsql_test 5.1.1 { + INSERT INTO t2 VALUES('a'), ('b'); + UPDATE t1 SET b = my_changes('update'); + SELECT * FROM t1; +} {1 2 2 2 3 2} + +# Value is being restored to "2" when the trigger program exits. +do_test 5.1.2 { + set ::changes +} {update 2 trigger 3 update 2 trigger 3 update 2 trigger 3} + + +reset_db +do_execsql_test 5.2.0 { + CREATE TABLE t1(a, b); + CREATE TABLE log(x); + INSERT INTO t1 VALUES(1, 0); + INSERT INTO t1 VALUES(2, 0); + INSERT INTO t1 VALUES(3, 0); + CREATE TRIGGER t1_a_u AFTER UPDATE ON t1 BEGIN + INSERT INTO log VALUES(old.b || ' -> ' || new.b || ' c = ' || changes() ); + END; + CREATE TABLE t2(a); + INSERT INTO t2 VALUES(1), (2), (3); + UPDATE t1 SET b = changes(); +} +do_execsql_test 5.2.1 { + SELECT * FROM t1; +} {1 3 2 3 3 3} +do_execsql_test 5.2.2 { + SELECT * FROM log; +} {{0 -> 3 c = 3} {0 -> 3 c = 3} {0 -> 3 c = 3}} + + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-17146-37073 Within a trigger program each INSERT, +# UPDATE and DELETE statement sets the value returned by +# sqlite3_changes() upon completion as normal. Of course, this value +# will not include any changes performed by sub-triggers, as the +# sqlite3_changes() value will be saved and restored after each +# sub-trigger has run. +reset_db +do_execsql_test 6.0 { + + CREATE TABLE t1(a, b); + CREATE TABLE t2(a, b); + CREATE TABLE t3(a, b); + CREATE TABLE log(x); + + CREATE TRIGGER t1_i BEFORE INSERT ON t1 BEGIN + INSERT INTO t2 VALUES(new.a, new.b), (new.a, new.b); + INSERT INTO log VALUES('t2->' || changes()); + END; + + CREATE TRIGGER t2_i AFTER INSERT ON t2 BEGIN + INSERT INTO t3 VALUES(new.a, new.b), (new.a, new.b), (new.a, new.b); + INSERT INTO log VALUES('t3->' || changes()); + END; + + CREATE TRIGGER t1_u AFTER UPDATE ON t1 BEGIN + UPDATE t2 SET b=new.b WHERE a=old.a; + INSERT INTO log VALUES('t2->' || changes()); + END; + + CREATE TRIGGER t2_u BEFORE UPDATE ON t2 BEGIN + UPDATE t3 SET b=new.b WHERE a=old.a; + INSERT INTO log VALUES('t3->' || changes()); + END; + + CREATE TRIGGER t1_d AFTER DELETE ON t1 BEGIN + DELETE FROM t2 WHERE a=old.a AND b=old.b; + INSERT INTO log VALUES('t2->' || changes()); + END; + + CREATE TRIGGER t2_d BEFORE DELETE ON t2 BEGIN + DELETE FROM t3 WHERE a=old.a AND b=old.b; + INSERT INTO log VALUES('t3->' || changes()); + END; +} + +do_changes_test 6.1 { + INSERT INTO t1 VALUES('+', 'o'); + SELECT * FROM log; +} {t3->3 t3->3 t2->2 1} + +do_changes_test 6.2 { + DELETE FROM log; + UPDATE t1 SET b='*'; + SELECT * FROM log; +} {t3->6 t3->6 t2->2 1} + +do_changes_test 6.3 { + DELETE FROM log; + DELETE FROM t1; + SELECT * FROM log; +} {t3->6 t3->0 t2->2 1} + + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-43399-09409 This means that if the changes() SQL +# function (or similar) is used by the first INSERT, UPDATE or DELETE +# statement within a trigger, it returns the value as set when the +# calling statement began executing. +# +# EVIDENCE-OF: R-53215-27584 If it is used by the second or subsequent +# such statement within a trigger program, the value returned reflects +# the number of rows modified by the previous INSERT, UPDATE or DELETE +# statement within the same trigger. +# +reset_db +do_execsql_test 7.1 { + CREATE TABLE q1(t); + CREATE TABLE q2(u, v); + CREATE TABLE q3(w); + + CREATE TRIGGER q2_insert BEFORE INSERT ON q2 BEGIN + + /* changes() returns value from previous I/U/D in callers context */ + INSERT INTO q1 VALUES('1:' || changes()); + + /* changes() returns value of previous I/U/D in this context */ + INSERT INTO q3 VALUES(changes()), (2), (3); + INSERT INTO q1 VALUES('2:' || changes()); + INSERT INTO q3 VALUES(changes() + 3), (changes()+4); + SELECT 'this does not affect things!'; + INSERT INTO q1 VALUES('3:' || changes()); + UPDATE q3 SET w = w+10 WHERE w%2; + INSERT INTO q1 VALUES('4:' || changes()); + DELETE FROM q3; + INSERT INTO q1 VALUES('5:' || changes()); + END; +} + +do_execsql_test 7.2 { + INSERT INTO q2 VALUES('x', 'y'); + SELECT * FROM q1; +} { + 1:0 2:3 3:2 4:3 5:5 +} + +do_execsql_test 7.3 { + DELETE FROM q1; + INSERT INTO q2 VALUES('x', 'y'); + SELECT * FROM q1; +} { + 1:5 2:3 3:2 4:3 5:5 +} + + + +finish_test diff --git a/test/e_createtable.test b/test/e_createtable.test index 42fc017e2a..2921d86c6f 100644 --- a/test/e_createtable.test +++ b/test/e_createtable.test @@ -862,11 +862,11 @@ do_createtable_tests 3.2.3 -query { 3 "INSERT INTO t1 DEFAULT VALUES" {NULL NULL NULL} } -# EVIDENCE-OF: R-62940-43005 An explicit DEFAULT clause may specify that +# EVIDENCE-OF: R-07343-35026 An explicit DEFAULT clause may specify that # the default value is NULL, a string constant, a blob constant, a -# signed-number, or any constant expression enclosed in parentheses. An -# explicit default value may also be one of the special case-independent -# keywords CURRENT_TIME, CURRENT_DATE or CURRENT_TIMESTAMP. +# signed-number, or any constant expression enclosed in parentheses. A +# default value may also be one of the special case-independent keywords +# CURRENT_TIME, CURRENT_DATE or CURRENT_TIMESTAMP. # do_execsql_test e_createtable-3.3.1 { CREATE TABLE t4( @@ -884,9 +884,9 @@ do_execsql_test e_createtable-3.3.1 { ); } {} -# EVIDENCE-OF: R-36381-62919 For the purposes of the DEFAULT clause, an -# expression is considered constant provided that it does not contain -# any sub-queries, column or table references, or string literals +# EVIDENCE-OF: R-18415-27776 For the purposes of the DEFAULT clause, an +# expression is considered constant if it does contains no sub-queries, +# column or table references, bound parameters, or string literals # enclosed in double-quotes instead of single-quotes. # do_createtable_tests 3.4.1 -error { @@ -896,6 +896,7 @@ do_createtable_tests 3.4.1 -error { 2 {CREATE TABLE t5(x DEFAULT ( "abc" ))} {} 3 {CREATE TABLE t5(x DEFAULT ( 1 IN (SELECT 1) ))} {} 4 {CREATE TABLE t5(x DEFAULT ( EXISTS (SELECT 1) ))} {} + 5 {CREATE TABLE t5(x DEFAULT ( x!=?1 ))} {} } do_createtable_tests 3.4.2 -repair { catchsql { DROP TABLE t5 } @@ -1175,9 +1176,9 @@ do_createtable_tests 4.2 -repair { # EVIDENCE-OF: R-59124-61339 Each row in a table with a primary key must # have a unique combination of values in its primary key columns. # -# EVIDENCE-OF: R-39102-06737 If an INSERT or UPDATE statement attempts -# to modify the table content so that two or more rows feature identical -# primary key values, it is a constraint violation. +# EVIDENCE-OF: R-06471-16287 If an INSERT or UPDATE statement attempts +# to modify the table content so that two or more rows have identical +# primary key values, that is a constraint violation. # drop_all_tables do_execsql_test 4.3.0 { diff --git a/test/e_expr.test b/test/e_expr.test index a743c0c390..271635f944 100644 --- a/test/e_expr.test +++ b/test/e_expr.test @@ -450,7 +450,7 @@ do_execsql_test e_expr-10.3.4 { SELECT typeof('isn''t') } {text} # containing hexadecimal data and preceded by a single "x" or "X" # character. # -# EVIDENCE-OF: R-39344-59787 For example: X'53514C697465' +# EVIDENCE-OF: R-19836-11244 Example: X'53514C697465' # do_execsql_test e_expr-10.4.1 { SELECT typeof(X'0123456789ABCDEF') } blob do_execsql_test e_expr-10.4.2 { SELECT typeof(x'0123456789ABCDEF') } blob @@ -1596,6 +1596,13 @@ do_expr_test e_expr-30.4.1 { CAST('' AS INTEGER) } integer 0 do_expr_test e_expr-30.4.2 { CAST('not a number' AS INTEGER) } integer 0 do_expr_test e_expr-30.4.3 { CAST('XXI' AS INTEGER) } integer 0 +# EVIDENCE-OF: R-08980-53124 The CAST operator understands decimal +# integers only — conversion of hexadecimal integers stops at +# the "x" in the "0x" prefix of the hexadecimal integer string and thus +# result of the CAST is always zero. +do_expr_test e_expr-30.5.1 { CAST('0x1234' AS INTEGER) } integer 0 +do_expr_test e_expr-30.5.2 { CAST('0X1234' AS INTEGER) } integer 0 + # EVIDENCE-OF: R-02752-50091 A cast of a REAL value into an INTEGER # results in the integer between the REAL value and zero that is closest # to the REAL value. diff --git a/test/e_totalchanges.test b/test/e_totalchanges.test new file mode 100644 index 0000000000..ee163c914f --- /dev/null +++ b/test/e_totalchanges.test @@ -0,0 +1,213 @@ +# 2011 May 06 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix e_totalchanges + +# Like [do_execsql_test], except it appends the value returned by +# [db total_changes] to the result of executing the SQL script. +# +proc do_tc_test {tn sql res} { + uplevel [list \ + do_test $tn "concat \[execsql {$sql}\] \[db total_changes\]" $res + ] +} + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b); + CREATE INDEX t1_b ON t1(b); + CREATE TABLE t2(x, y, PRIMARY KEY(x, y)) WITHOUT ROWID; + CREATE INDEX t2_y ON t2(y); +} + + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-65438-26258 This function returns the total number of +# rows inserted, modified or deleted by all INSERT, UPDATE or DELETE +# statements completed since the database connection was opened, +# including those executed as part of trigger programs. +# +# 1.1.*: different types of I/U/D statements, +# 1.2.*: trigger programs. +# +do_tc_test 1.1.1 { + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + UPDATE t1 SET a = a+1; + DELETE FROM t1; +} {6} +do_tc_test 1.1.2 { + DELETE FROM t1 +} {6} + +do_tc_test 1.1.3 { + WITH data(a,b) AS ( + SELECT 0, 0 UNION ALL SELECT a+1, b+1 FROM data WHERE a<99 + ) + INSERT INTO t1 SELECT * FROM data; +} {106} + +do_tc_test 1.1.4 { + INSERT INTO t2 SELECT * FROM t1 WHERE a<50; + UPDATE t2 SET y=y+1; +} {206} + +do_tc_test 1.1.5 { + DELETE FROM t2 WHERE y<=25 +} {231} + +do_execsql_test 1.2.1 { + DELETE FROM t1; + DELETE FROM t2; +} +sqlite3 db test.db ; # To reset total_changes +do_tc_test 1.2.2 { + CREATE TABLE log(detail); + CREATE TRIGGER t1_after_insert AFTER INSERT ON t1 BEGIN + INSERT INTO log VALUES('inserted into t1'); + END; + + CREATE TRIGGER t1_before_delete BEFORE DELETE ON t1 BEGIN + INSERT INTO log VALUES('deleting from t1'); + INSERT INTO log VALUES('here we go!'); + END; + + CREATE TRIGGER t1_after_update AFTER UPDATE ON t1 BEGIN + INSERT INTO log VALUES('update'); + DELETE FROM log; + END; + + INSERT INTO t1 VALUES('a', 'b'); -- 1 + 1 + UPDATE t1 SET b='c'; -- 1 + 1 + 2 + DELETE FROM t1; -- 1 + 1 + 1 +} {9} + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-61766-15253 Executing any other type of SQL statement +# does not affect the value returned by sqlite3_total_changes(). +do_tc_test 2.1 { + INSERT INTO t1 VALUES(1, 2), (3, 4); + INSERT INTO t2 VALUES(1, 2), (3, 4); +} {15} +do_tc_test 2.2 { + SELECT count(*) FROM t1; +} {2 15} +do_tc_test 2.3 { + CREATE TABLE t4(a, b); + ALTER TABLE t4 ADD COLUMN c; + CREATE INDEX i4 ON t4(c); + ALTER TABLE t4 RENAME TO t5; + ANALYZE; + BEGIN; + DROP TABLE t2; + ROLLBACK; + VACUUM; +} {15} + + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-36043-10590 Changes made as part of foreign key +# actions are included in the count, but those made as part of REPLACE +# constraint resolution are not. +# +# 3.1.*: foreign key actions +# 3.2.*: REPLACE constraints. +# +sqlite3 db test.db ; # To reset total_changes +do_tc_test 3.1.1 { + CREATE TABLE p1(c PRIMARY KEY, d); + CREATE TABLE c1(a, b, FOREIGN KEY(a) REFERENCES p1 ON DELETE SET NULL); + CREATE TABLE c2(a, b, FOREIGN KEY(a) REFERENCES p1 ON DELETE CASCADE); + CREATE TABLE c3(a, b, FOREIGN KEY(a) REFERENCES p1 ON DELETE SET DEFAULT); + + INSERT INTO p1 VALUES(1, 'one'); + INSERT INTO p1 VALUES(2, 'two'); + INSERT INTO p1 VALUES(3, 'three'); + INSERT INTO p1 VALUES(4, 'four'); + + INSERT INTO c1 VALUES(1, 'i'); + INSERT INTO c2 VALUES(2, 'ii'); + INSERT INTO c3 VALUES(3, 'iii'); + PRAGMA foreign_keys = ON; +} {7} + +do_tc_test 3.1.2 { DELETE FROM p1 WHERE c=1; } {9} +do_tc_test 3.1.3 { DELETE FROM p1 WHERE c=2; } {11} +do_tc_test 3.1.4 { DELETE FROM p1 WHERE c=3; } {13} +do_tc_test 3.1.5 { DELETE FROM p1 WHERE c=4; } {14} ; # only 1 this time. + +sqlite3 db test.db ; # To reset total_changes +do_tc_test 3.1.6 { + DROP TABLE c1; + DROP TABLE c2; + DROP TABLE c3; + CREATE TABLE c1(a, b, FOREIGN KEY(a) REFERENCES p1 ON UPDATE SET NULL); + CREATE TABLE c2(a, b, FOREIGN KEY(a) REFERENCES p1 ON UPDATE CASCADE); + CREATE TABLE c3(a, b, FOREIGN KEY(a) REFERENCES p1 ON UPDATE SET DEFAULT); + + INSERT INTO p1 VALUES(1, 'one'); + INSERT INTO p1 VALUES(2, 'two'); + INSERT INTO p1 VALUES(3, 'three'); + INSERT INTO p1 VALUES(4, 'four'); + + INSERT INTO c1 VALUES(1, 'i'); + INSERT INTO c2 VALUES(2, 'ii'); + INSERT INTO c3 VALUES(3, 'iii'); + PRAGMA foreign_keys = ON; +} {7} + +do_tc_test 3.1.7 { UPDATE p1 SET c=c+4 WHERE c=1; } {9} +do_tc_test 3.1.8 { UPDATE p1 SET c=c+4 WHERE c=2; } {11} +do_tc_test 3.1.9 { UPDATE p1 SET c=c+4 WHERE c=3; } {13} +do_tc_test 3.1.10 { UPDATE p1 SET c=c+4 WHERE c=4; } {14} ; # only 1 this time. + +sqlite3 db test.db ; # To reset total_changes +do_tc_test 3.2.1 { + CREATE TABLE t3(a UNIQUE, b UNIQUE); + INSERT INTO t3 VALUES('one', 'one'); + INSERT INTO t3 VALUES('two', 'two'); + INSERT OR REPLACE INTO t3 VALUES('one', 'two'); +} {3} + +do_tc_test 3.2.2 { + INSERT INTO t3 VALUES('three', 'one'); + UPDATE OR REPLACE t3 SET b='two' WHERE b='one'; + SELECT * FROM t3; +} {three two 5} + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-54872-08741 Changes to a view that are intercepted by +# INSTEAD OF triggers are not counted. +# +sqlite3 db test.db ; # To reset total_changes +do_tc_test 4.1 { + CREATE TABLE t6(x); + CREATE VIEW v1 AS SELECT * FROM t6; + CREATE TRIGGER v1_tr1 INSTEAD OF INSERT ON v1 BEGIN + SELECT 'no-op'; + END; + + INSERT INTO v1 VALUES('a'); + INSERT INTO v1 VALUES('b'); +} {0} +do_tc_test 4.2 { + CREATE TRIGGER v1_tr2 INSTEAD OF INSERT ON v1 BEGIN + INSERT INTO t6 VALUES(new.x); + END; + + INSERT INTO v1 VALUES('c'); + INSERT INTO v1 VALUES('d'); +} {2} + + +finish_test diff --git a/test/e_uri.test b/test/e_uri.test index a8865cad28..d1590e4108 100644 --- a/test/e_uri.test +++ b/test/e_uri.test @@ -125,6 +125,9 @@ if {$tcl_platform(platform) == "unix"} { sqlite3_shutdown sqlite3_config_uri 1 +# EVIDENCE-OF: R-06842-00595 If the URI contains an authority, then it +# must be either an empty string or the string "localhost". +# # EVIDENCE-OF: R-17482-00398 If the authority is not an empty string or # "localhost", an error is returned to the caller. # diff --git a/test/e_wal.test b/test/e_wal.test new file mode 100644 index 0000000000..a5e074f49b --- /dev/null +++ b/test/e_wal.test @@ -0,0 +1,229 @@ +# 2011 May 06 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix e_wal + +db close +testvfs oldvfs -iversion 1 + + +# EVIDENCE-OF: R-58297-14483 WAL databases can be created, read, and +# written even if shared memory is unavailable as long as the +# locking_mode is set to EXCLUSIVE before the first attempted access. +# +# EVIDENCE-OF: R-00449-33772 This feature allows WAL databases to be +# created, read, and written by legacy VFSes that lack the "version 2" +# shared-memory methods xShmMap, xShmLock, xShmBarrier, and xShmUnmap on +# the sqlite3_io_methods object. +# +# 1.1: "create" tests. +# 1.2: "read" tests. +# 1.3: "write" tests. +# +# All three done with VFS "oldvfs", which has iVersion==1 and so does +# not support shared memory. +# +sqlite3 db test.db -vfs oldvfs +do_execsql_test 1.1.1 { + PRAGMA journal_mode = WAL; +} {delete} +do_execsql_test 1.1.2 { + PRAGMA locking_mode = EXCLUSIVE; + PRAGMA journal_mode = WAL; +} {exclusive wal} +do_execsql_test 1.1.3 { + CREATE TABLE t1(x, y); + INSERT INTO t1 VALUES(1, 2); +} {} +do_test 1.1.4 { + list [file exists test.db-shm] [file exists test.db-wal] +} {0 1} + +do_test 1.2.1 { + db close + sqlite3 db test.db -vfs oldvfs + catchsql { SELECT * FROM t1 } +} {1 {unable to open database file}} +do_test 1.2.2 { + execsql { PRAGMA locking_mode = EXCLUSIVE } + execsql { SELECT * FROM t1 } +} {1 2} +do_test 1.2.3 { + list [file exists test.db-shm] [file exists test.db-wal] +} {0 1} + +do_test 1.3.1 { + db close + sqlite3 db test.db -vfs oldvfs + catchsql { INSERT INTO t1 VALUES(3, 4) } +} {1 {unable to open database file}} +do_test 1.3.2 { + execsql { PRAGMA locking_mode = EXCLUSIVE } + execsql { INSERT INTO t1 VALUES(3, 4) } + execsql { SELECT * FROM t1 } +} {1 2 3 4} +do_test 1.3.3 { + list [file exists test.db-shm] [file exists test.db-wal] +} {0 1} + +# EVIDENCE-OF: R-31969-57825 If EXCLUSIVE locking mode is set prior to +# the first WAL-mode database access, then SQLite never attempts to call +# any of the shared-memory methods and hence no shared-memory wal-index +# is ever created. +# +db close +sqlite3 db test.db +do_execsql_test 2.1.1 { + PRAGMA locking_mode = EXCLUSIVE; + SELECT * FROM t1; +} {exclusive 1 2 3 4} +do_test 2.1.2 { + list [file exists test.db-shm] [file exists test.db-wal] +} {0 1} + +# EVIDENCE-OF: R-36328-16367 In that case, the database connection +# remains in EXCLUSIVE mode as long as the journal mode is WAL; attempts +# to change the locking mode using "PRAGMA locking_mode=NORMAL;" are +# no-ops. +# +do_execsql_test 2.2.1 { + PRAGMA locking_mode = NORMAL; + SELECT * FROM t1; +} {exclusive 1 2 3 4} +do_test 2.2.2 { + sqlite3 db2 test.db + catchsql {SELECT * FROM t1} db2 +} {1 {database is locked}} +db2 close + +# EVIDENCE-OF: R-63522-46088 The only way to change out of EXCLUSIVE +# locking mode is to first change out of WAL journal mode. +# +do_execsql_test 2.3.1 { + PRAGMA journal_mode = DELETE; + SELECT * FROM t1; +} {delete 1 2 3 4} +do_test 2.3.2 { + sqlite3 db2 test.db + catchsql {SELECT * FROM t1} db2 +} {1 {database is locked}} +do_execsql_test 2.3.3 { + PRAGMA locking_mode = NORMAL; + SELECT * FROM t1; +} {normal 1 2 3 4} +do_test 2.3.4 { + sqlite3 db2 test.db + catchsql {SELECT * FROM t1} db2 +} {0 {1 2 3 4}} +db2 close +db close + + +# EVIDENCE-OF: R-57239-11845 If NORMAL locking mode is in effect for the +# first WAL-mode database access, then the shared-memory wal-index is +# created. +# +do_test 3.0 { + sqlite3 db test.db + execsql { PRAGMA journal_mode = WAL } + db close +} {} +do_test 3.1 { + sqlite3 db test.db + execsql { SELECT * FROM t1 } + list [file exists test.db-shm] [file exists test.db-wal] +} {1 1} + +# EVIDENCE-OF: R-13779-07711 As long as exactly one connection is using +# a shared-memory wal-index, the locking mode can be changed freely +# between NORMAL and EXCLUSIVE. +# +do_execsql_test 3.2.1 { + PRAGMA locking_mode = EXCLUSIVE; + PRAGMA locking_mode = NORMAL; + PRAGMA locking_mode = EXCLUSIVE; + INSERT INTO t1 VALUES(5, 6); +} {exclusive normal exclusive} +do_test 3.2.2 { + sqlite3 db2 test.db + catchsql { SELECT * FROM t1 } db2 +} {1 {database is locked}} + +# EVIDENCE-OF: R-10993-11647 It is only when the shared-memory wal-index +# is omitted, when the locking mode is EXCLUSIVE prior to the first +# WAL-mode database access, that the locking mode is stuck in EXCLUSIVE. +# +do_execsql_test 3.2.3 { + PRAGMA locking_mode = NORMAL; + SELECT * FROM t1; +} {normal 1 2 3 4 5 6} +do_test 3.2.4 { + catchsql { SELECT * FROM t1 } db2 +} {0 {1 2 3 4 5 6}} + +do_catchsql_test 3.2.5 { + PRAGMA locking_mode = EXCLUSIVE; + INSERT INTO t1 VALUES(7, 8); +} {1 {database is locked}} + +db2 close + +# EVIDENCE-OF: R-46197-42811 This means that the underlying VFS must +# support the "version 2" shared-memory. +# +# EVIDENCE-OF: R-55316-21772 If the VFS does not support shared-memory +# methods, then the attempt to open a database that is already in WAL +# mode, or the attempt convert a database into WAL mode, will fail. +# +db close +do_test 3.4.1 { + sqlite3 db test.db -vfs oldvfs + catchsql { SELECT * FROM t1 } +} {1 {unable to open database file}} +db close +do_test 3.4.2 { + forcedelete test.db2 + sqlite3 db test.db2 -vfs oldvfs + catchsql { PRAGMA journal_mode = WAL } +} {0 delete} +db close + + +# EVIDENCE-OF: R-22428-28959 To prevent older versions of SQLite from +# trying to recover a WAL-mode database (and making matters worse) the +# database file format version numbers (bytes 18 and 19 in the database +# header) are increased from 1 to 2 in WAL mode. +# +reset_db +do_execsql_test 4.1.1 { CREATE TABLE t1(x, y) } +do_test 4.1.2 { hexio_read test.db 18 2 } {0101} +do_execsql_test 4.1.3 { PRAGMA journal_mode = wAL } {wal} +do_test 4.1.4 { hexio_read test.db 18 2 } {0202} + + +# EVIDENCE-OF: R-02535-05811 One can explicitly change out of WAL mode +# using a pragma such as this: PRAGMA journal_mode=DELETE; +# +do_execsql_test 4.2.1 { INSERT INTO t1 VALUES(1, 1); } {} +do_test 4.2.2 { file exists test.db-wal } {1} +do_execsql_test 4.2.3 { PRAGMA journal_mode = delete } {delete} +do_test 4.2.4 { file exists test.db-wal } {0} + +# EVIDENCE-OF: R-60175-02388 Deliberately changing out of WAL mode +# changes the database file format version numbers back to 1 so that +# older versions of SQLite can once again access the database file. +# +do_test 4.3 { hexio_read test.db 18 2 } {0101} + +finish_test diff --git a/test/e_walauto.test b/test/e_walauto.test new file mode 100644 index 0000000000..239adc3b95 --- /dev/null +++ b/test/e_walauto.test @@ -0,0 +1,202 @@ +# 2014 December 04 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/wal_common.tcl +set testprefix e_walauto + + +proc read_nbackfill {} { + seek $::shmfd 96 + binary scan [read $::shmfd 4] i nBackfill + set nBackfill +} +proc read_mxframe {} { + seek $::shmfd 16 + binary scan [read $::shmfd 4] i mxFrame + set mxFrame +} + +# Assuming that the main db for database handle +# +proc do_autocommit_threshold_test {tn value} { + + set nBackfillSaved [read_nbackfill] + while {1} { + db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) } + if {[read_mxframe] >= $value} break + } + + set nBackfillNew [read_nbackfill] + uplevel [list do_test $tn "expr $nBackfillNew > $nBackfillSaved" 1] +} + +# EVIDENCE-OF: R-30135-06439 The wal_autocheckpoint pragma can be used +# to invoke this interface from SQL. +# +# All tests in this file are run twice - once using the +# sqlite3_wal_autocheckpoint() API, and once using "PRAGMA +# wal_autocheckpoint". +# +foreach {tn code} { + 1 { + proc autocheckpoint {db value} { + uplevel [list $db eval "PRAGMA wal_autocheckpoint = $value"] + } + } + + 2 { + proc autocheckpoint {db value} { + uplevel [list sqlite3_wal_autocheckpoint $db $value] + return $value + } + } +} { + + eval $code + + reset_db + execsql { PRAGMA auto_vacuum = 0 } + do_execsql_test 1.$tn.0 { PRAGMA journal_mode = WAL } {wal} + do_execsql_test 1.$tn.1 { CREATE TABLE t1(a, b) } + set shmfd [open "test.db-shm" rb] + + # EVIDENCE-OF: R-41531-51083 Every new database connection defaults to + # having the auto-checkpoint enabled with a threshold of 1000 or + # SQLITE_DEFAULT_WAL_AUTOCHECKPOINT pages. + # + do_autocommit_threshold_test 1.$tn.2 1000 + db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) } + do_autocommit_threshold_test 1.$tn.3 1000 + + # EVIDENCE-OF: R-38128-34102 The sqlite3_wal_autocheckpoint(D,N) is a + # wrapper around sqlite3_wal_hook() that causes any database on database + # connection D to automatically checkpoint after committing a + # transaction if there are N or more frames in the write-ahead log file. + # + do_test 1.$tn.4 { + db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) } + autocheckpoint db 100 + } {100} + do_autocommit_threshold_test 1.$tn.5 100 + + do_test 1.$tn.6 { + db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) } + autocheckpoint db 500 + } {500} + do_autocommit_threshold_test 1.$tn.7 500 + + # EVIDENCE-OF: R-26993-43540 Passing zero or a negative value as the + # nFrame parameter disables automatic checkpoints entirely. + # + do_test 1.$tn.7 { + autocheckpoint db 0 ;# Set to zero + for {set i 0} {$i < 10000} {incr i} { + db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) } + } + expr {[file size test.db-wal] > (5 * 1024 * 1024)} + } 1 + do_test 1.$tn.8 { + sqlite3_wal_checkpoint_v2 db truncate + file size test.db-wal + } 0 + do_test 1.$tn.9 { + autocheckpoint db -4 ;# Set to a negative value + for {set i 0} {$i < 10000} {incr i} { + db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) } + } + expr {[file size test.db-wal] > (5 * 1024 * 1024)} + } 1 + + # EVIDENCE-OF: R-10203-42688 The callback registered by this function + # replaces any existing callback registered using sqlite3_wal_hook(). + # + set ::wal_hook_callback 0 + proc wal_hook_callback {args} { incr ::wal_hook_callback ; return 0 } + do_test 1.$tn.10.1 { + db wal_hook wal_hook_callback + db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) } + db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) } + set ::wal_hook_callback + } 2 + do_test 1.$tn.10.2 { + autocheckpoint db 100 + db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) } + db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) } + set ::wal_hook_callback + } 2 + + # EVIDENCE-OF: R-17497-43474 Likewise, registering a callback using + # sqlite3_wal_hook() disables the automatic checkpoint mechanism + # configured by this function. + do_test 1.$tn.11.1 { + sqlite3_wal_checkpoint_v2 db truncate + file size test.db-wal + } 0 + do_test 1.$tn.11.2 { + autocheckpoint db 100 + for {set i 0} {$i < 1000} {incr i} { + db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) } + } + expr {[file size test.db-wal] < (1 * 1024 * 1024)} + } 1 + do_test 1.$tn.11.3 { + db wal_hook wal_hook_callback + for {set i 0} {$i < 1000} {incr i} { + db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) } + } + expr {[file size test.db-wal] < (1 * 1024 * 1024)} + } 0 + + # EVIDENCE-OF: R-33080-59193 Checkpoints initiated by this mechanism + # are PASSIVE. + # + set ::busy_callback_count 0 + proc busy_callback {args} { + puts Hello + incr ::busy_callback_count + return 0 + } + do_test 1.$tn.12.1 { + sqlite3_wal_checkpoint_v2 db truncate + autocheckpoint db 100 + db busy busy_callback + db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) } + db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) } + } {} + do_test 1.$tn.12.2 { + sqlite3 db2 test.db + db2 eval { BEGIN; SELECT * FROM t1 LIMIT 10; } + read_nbackfill + } {0} + do_test 1.$tn.12.3 { + for {set i 0} {$i < 1000} {incr i} { + db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) } + } + read_nbackfill + } {2} + do_test 1.$tn.12.4 { + set ::busy_callback_count + } {0} + db2 close + + do_test 1.$tn.12.5 { + db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) } + read_nbackfill + } {1559} + + db close + close $shmfd +} + +finish_test diff --git a/test/e_walckpt.test b/test/e_walckpt.test new file mode 100644 index 0000000000..e022f840cf --- /dev/null +++ b/test/e_walckpt.test @@ -0,0 +1,754 @@ +# 2014 December 04 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +source $testdir/wal_common.tcl +set testprefix e_walckpt + +# The following two commands are used to determine if any of the files +# "test.db", "test.db2" and "test.db3" are modified by a test case. +# +# The [save_db_hashes] command saves a hash of the current contents of +# all three files in global variables. The [compare_db_hashes] compares +# the current contents with the saved hashes and returns a list of the +# files that have changed. +# +proc save_db_hashes {} { + global H + foreach f {test.db test.db2 test.db3} { + set H($f) 0 + catch { set H($f) [md5file $f] } + } +} +proc compare_db_hashes {} { + global H + set ret [list] + foreach f {test.db test.db2 test.db3} { + set expect 0 + catch { set expect [md5file $f] } + if {$H($f) != $expect} { lappend ret $f } + } + set ret +} + +#------------------------------------------------------------------------- +# All calls to the [sqlite3_wal_checkpoint_v2] command made within this +# file use this wrapper. It's sole purpose is to throw an error if the +# following requirement is violated: +# +# EVIDENCE-OF: R-60567-47780 Unless it returns SQLITE_MISUSE, the +# sqlite3_wal_checkpoint_v2() interface sets the error information that +# is queried by sqlite3_errcode() and sqlite3_errmsg(). +# +proc wal_checkpoint_v2 {db args} { + set rc [catch { + uplevel sqlite3_wal_checkpoint_v2 $db $args + } msg] + + set errcode "SQLITE_OK" + if {$rc} { + set errcode [lindex [split $msg " "] 0] + } elseif { [lindex $msg 0] } { + set errcode "SQLITE_BUSY" + } + + if {$errcode != "SQLITE_MISUSE" && [sqlite3_errcode $db] != $errcode} { + error "sqlite3_errcode mismatch! (1) $errcode!=[sqlite3_errcode $db]" + } + + if {$rc==0} { + return $msg + } else { + error $msg + } +} + + +# The following tests are run 3 times, each using a different method of +# invoking a checkpoint: +# +# 1) Using sqlite3_wal_checkpoint_v2() +# 2) Using "PRAGMA wal_checkpoint" +# 3) Using sqlite3_wal_checkpoint() in place of checkpoint_v2(PASSIVE) +# +# Cases (2) and (3) are to show that the following statements are +# correct, respectively: +# +# EVIDENCE-OF: R-36706-10507 The PRAGMA wal_checkpoint command can be +# used to invoke this interface from SQL. +# +# EVIDENCE-OF: R-41613-20553 The sqlite3_wal_checkpoint(D,X) is +# equivalent to +# sqlite3_wal_checkpoint_v2(D,X,SQLITE_CHECKPOINT_PASSIVE,0,0). +# +foreach {tn script} { + 1 { + proc checkpoint {db mode args} { + eval wal_checkpoint_v2 [list $db] [list $mode] $args + } + } + + 2 { + proc checkpoint {db mode args} { + set sql "PRAGMA wal_checkpoint = $mode" + if {[llength $args] && [lindex $args 0]!=""} { + set sql "PRAGMA [lindex $args 0].wal_checkpoint = $mode" + } + set rc [catch { $db eval $sql } msg] + if {$rc} { + regsub {database} $msg {database:} msg + error "[sqlite3_errcode $db] - $msg" + } + set msg + } + } + + 3 { + proc checkpoint {db mode args} { + if {$mode == "passive"} { + set rc [eval sqlite3_wal_checkpoint [list $db] $args] + if {$rc != "SQLITE_OK"} { + error "$rc - [sqlite3_errmsg $db]" + } + } else { + eval wal_checkpoint_v2 [list $db] [list $mode] $args + } + } + } + +} { + + eval $script + + reset_db + forcedelete test.db2 test.db3 test.db4 + execsql { + ATTACH 'test.db2' AS aux; + ATTACH 'test.db3' AS aux2; + ATTACH 'test.db4' AS aux3; + CREATE TABLE t1(x); + CREATE TABLE aux.t2(x); + CREATE TABLE aux2.t3(x); + CREATE TABLE aux3.t4(x); + PRAGMA main.journal_mode = WAL; + PRAGMA aux.journal_mode = WAL; + PRAGMA aux2.journal_mode = WAL; + /* Leave aux4 in rollback mode */ + } + + # EVIDENCE-OF: R-49787-09095 The sqlite3_wal_checkpoint_v2(D,X,M,L,C) + # interface runs a checkpoint operation on database X of database + # connection D in mode M. Status information is written back into + # integers pointed to by L and C. + # + # Tests 1, 2 and 3 below verify the "on database X" part of the + # above. Other parts of this requirement are tested below. + # + # EVIDENCE-OF: R-00653-06026 If parameter zDb is NULL or points to a + # zero length string, then the specified operation is attempted on all + # WAL databases attached to database connection db. + # + # Tests 4 and 5 below test this. + # + foreach {tn2 zDb dblist} { + 1 main test.db + 2 aux test.db2 + 3 aux2 test.db3 + 4 "" {test.db test.db2 test.db3} + 5 - {test.db test.db2 test.db3} + 6 temp {} + } { + do_test $tn.1.$tn2 { + execsql { + INSERT INTO t1 VALUES(1); + INSERT INTO t2 VALUES(2); + INSERT INTO t3 VALUES(3); + } + save_db_hashes + + if {$zDb == "-"} { + checkpoint db passive + } else { + checkpoint db passive $zDb + } + + compare_db_hashes + } $dblist + } + + # EVIDENCE-OF: R-38207-48996 If zDb is not NULL (or a zero length + # string) and is not the name of any attached database, SQLITE_ERROR is + # returned to the caller. + do_test $tn.2.1 { + list [catch { checkpoint db passive notadb } msg] $msg + } {1 {SQLITE_ERROR - unknown database: notadb}} + + # EVIDENCE-OF: R-14303-42483 If database zDb is the name of an attached + # database that is not in WAL mode, SQLITE_OK is returned and both + # *pnLog and *pnCkpt set to -1. + # + if {$tn==3} { + # With sqlite3_wal_checkpoint() the two output variables cannot be + # tested. So just test that no error is returned when attempting to + # checkpoint a db in rollback mode. + do_test $tn.2.2.a { checkpoint db passive aux3 } {} + } else { + do_test $tn.2.2.b { checkpoint db passive aux3 } {0 -1 -1} + } + + # EVIDENCE-OF: R-62028-47212 All calls obtain an exclusive "checkpoint" + # lock on the database file. + db close + testvfs tvfs + tvfs filter xShmLock + tvfs script filelock + proc filelock {method file handle details} { + # Test for an exclusive checkpoint lock. A checkpoint lock locks a + # single byte starting at offset 1. + if {$details == "1 1 lock exclusive"} { set ::seen_checkpoint_lock 1 } + } + sqlite3 db test.db -vfs tvfs + do_test $tn.3.1 { + execsql { INSERT INTO t1 VALUES('xyz') } + unset -nocomplain ::seen_checkpoint_lock + checkpoint db passive + set ::seen_checkpoint_lock + } {1} + db close + tvfs delete + reset_db + + + + + #----------------------------------------------------------------------- + # EVIDENCE-OF: R-10421-19736 If any other process is running a + # checkpoint operation at the same time, the lock cannot be obtained and + # SQLITE_BUSY is returned. + # + # EVIDENCE-OF: R-53820-33897 Even if there is a busy-handler configured, + # it will not be invoked in this case. + # + testvfs tvfs + tvfs filter xWrite + sqlite3 db test.db -vfs tvfs + sqlite3 db2 test.db -vfs tvfs + + do_test $tn.3.2.1 { + db2 eval { + PRAGMA auto_vacuum = 0; + PRAGMA journal_mode = WAL; + CREATE TABLE t1(x, y); + INSERT INTO t1 VALUES(1,2); + INSERT INTO t1 VALUES(3,4); + INSERT INTO t1 VALUES(5,6); + } + file size test.db-wal + } [wal_file_size 5 1024] + + + # Connection [db] runs a checkpoint. During this checkpoint, each + # time it calls xWrite() to write a page into the database file, we + # attempt to start a checkpoint using [db2]. According to the + # first requirement being tested, this should return SQLITE_BUSY. According + # to the second, the busy-handler belonging to [db2] should not be + # invoked. + # + set ::write_count 0 + set ::write_errors [list] + proc busy_callback {args} { + lappend ::write_errors "busy handler called!" + } + proc write_callback {args} { + set rc [catch {checkpoint db2 passive} msg] + if {0==[regexp "database is locked" $msg] && $msg!="1 -1 -1"} { + lappend ::write_errors "$rc $msg" + } + incr ::write_count + } + db2 busy busy_callback + tvfs script write_callback + + do_test $tn.3.2.2 { + db eval {SELECT * FROM sqlite_master} + checkpoint db full + set ::write_count + } {2} + + do_test $tn.3.2.3 { + set ::write_errors + } {} + + db close + db2 close + tvfs delete + + proc busy_handler {mode busy_handler_mode n} { + incr ::busy_handler_counter + switch -- $busy_handler_mode { + 1 { + # Do nothing. Do not block. + return 1 + } + + 2 { + # Close first the reader, then later the writer. Give up before + # closing the [db6] reader. + if {$n==5} { catch {db2 eval commit} } + if {$n==10} { catch {db3 eval commit} } + if {$n==15} { return 1 } + return 0 + } + + 3 { + # Close first the writer, then later the reader. And finally the + # [db6] reader. + if {$n==5} { catch {db2 eval commit} } + if {$n==10} { catch {db3 eval commit} } + if {$n==15} { catch {db6 eval commit} } + return 0 + } + } + } + + foreach {mode busy_handler_mode} { + passive 1 + full 1 full 2 full 3 + restart 1 restart 2 restart 3 + truncate 1 truncate 2 truncate 3 + } { + set tp "$tn.$mode.$busy_handler_mode" + + set ::sync_counter 0 + + # Set up a callback function for xSync and xWrite calls made during + # the checkpoint. + # + set ::checkpoint_ongoing 0 + proc tvfs_callback {method args} { + if {$::checkpoint_ongoing==0} return + + set tail [file tail [lindex $args 0]] + if {$method == "xSync" && $tail == "test.db"} { + incr ::sync_counter + } + if {$method == "xWrite" && $tail=="test.db"} { + if {$::write_ok < 0} { + set ::write_ok [expr ![catch {db5 eval { BEGIN IMMEDIATE }}]] + catch { db5 eval ROLLBACK } + } + if {$::read_ok < 0} { + set ::read_ok [expr ![catch {db5 eval { SELECT * FROM t1 }}]] + } + + # If one has not already been opened, open a read-transaction using + # connection [db6] + catch { db6 eval { BEGIN ; SELECT * FROM sqlite_master } } msg + } + if {$method == "xShmLock" } { + set details [lindex $args 2] + if {$details == "0 1 lock exclusive"} { set ::seen_writer_lock 1 } + } + } + + catch { db close } + forcedelete test.db + testvfs tvfs + sqlite3 db test.db -vfs tvfs + #tvfs filter xSync + tvfs script tvfs_callback + + do_execsql_test $tp.0 { + CREATE TABLE t1(a, b); + CREATE TABLE t2(a, b); + PRAGMA journal_mode = wal; + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t1 VALUES(5, 6); + } {wal} + + # Open a reader on the current database snapshot. + do_test $tp.1 { + sqlite3 db2 test.db -vfs tvfs + execsql { + BEGIN; + SELECT * FROM t1 UNION ALL SELECT * FROM t2; + } db2 + } {1 2 3 4 5 6} + + # Open a writer. Write a transaction. Then begin, but do not commit, + # a second transaction. + do_test $tp.2 { + sqlite3 db3 test.db -vfs tvfs + execsql { + INSERT INTO t2 VALUES(7, 8); + BEGIN; + INSERT INTO t2 VALUES(9, 10); + SELECT * FROM t1 UNION ALL SELECT * FROM t2; + } db3 + } {1 2 3 4 5 6 7 8 9 10} + + sqlite3 db5 test.db -vfs tvfs + sqlite3 db6 test.db -vfs tvfs + + # Register a busy-handler with connection [db]. + # + db busy [list busy_handler $mode $busy_handler_mode] + set ::sync_counter 0 + set ::busy_handler_counter 0 + set ::read_ok -1 + set ::write_ok -1 + set ::seen_writer_lock 0 + + set ::checkpoint_ongoing 1 + do_test $tp.3 { + checkpoint db $mode main + set {} {} + } {} + set ::checkpoint_ongoing 0 + set ::did_restart_blocking [expr {[catch {db6 eval commit}]}] + + if { $mode=="passive" } { + # EVIDENCE-OF: R-16333-64433 Checkpoint as many frames as possible + # without waiting for any database readers or writers to finish, then + # sync the database file if all frames in the log were checkpointed. + # + # "As many frames as possible" means all but the last two transactions + # (the two that write to table t2, of which the scond is unfinished). + # So copying the db file only we see the t1 change, but not the t2 + # modifications. + # + # The busy handler is not invoked (see below) and the db reader and + # writer are still active - so the checkpointer did not wait for either + # readers or writers. As a result the checkpoint was not finished and + # so the db file is not synced. + # + # EVIDENCE-OF: R-62920-47450 The busy-handler callback is never invoked + # in the SQLITE_CHECKPOINT_PASSIVE mode. + # + # It's not. Test case "$tp.6". + # + do_test $tp.4 { + forcecopy test.db abc.db + sqlite3 db4 abc.db + db4 eval { SELECT * FROM t1 UNION ALL SELECT * FROM t2 } + } {1 2 3 4 5 6} + do_test $tp.5 { set ::sync_counter } 0 + do_test $tp.6 { set ::busy_handler_counter } 0 + db4 close + + db2 eval COMMIT + db3 eval COMMIT + + # EVIDENCE-OF: R-65499-53765 On the other hand, passive mode might leave + # the checkpoint unfinished if there are concurrent readers or writers. + # + # The reader and writer have now dropped their locks. And so a + # checkpoint now is able to checkpoint more frames. Showing that the + # attempt above was left "unfinished". + # + # Also, because the checkpoint finishes this time, the db is synced. + # Which is part of R-16333-64433 above. + # + set ::checkpoint_ongoing 1 + do_test $tp.7 { + checkpoint db $mode main + forcecopy test.db abc.db + sqlite3 db4 abc.db + db4 eval { SELECT * FROM t1 UNION ALL SELECT * FROM t2 } + } {1 2 3 4 5 6 7 8 9 10} + set ::checkpoint_ongoing 0 + do_test $tp.7 { set ::sync_counter } 1 + do_test $tp.8 { set ::busy_handler_counter } 0 + db4 close + } + + if { $mode=="full" || $mode=="restart" || $mode=="truncate" } { + + # EVIDENCE-OF: R-59782-36818 The SQLITE_CHECKPOINT_FULL, RESTART and + # TRUNCATE modes also obtain the exclusive "writer" lock on the + # database file. + # + # Or at least attempts to obtain. + # + do_test $tp.9 { + set ::seen_writer_lock + } {1} + + if {$busy_handler_mode==2 || $busy_handler_mode==3} { + # EVIDENCE-OF: R-59171-47567 This mode blocks (it invokes the + # busy-handler callback) until there is no database writer and all + # readers are reading from the most recent database snapshot. + # + # The test below shows that both the reader and writer have + # finished: + # + # Also restated by the following two. That both busy_handler_mode + # values 2 and 3 work show that both of the following are true - as + # they release the reader and writer transactions in different + # orders. + # + # EVIDENCE-OF: R-60642-04082 If the writer lock cannot be obtained + # immediately, and a busy-handler is configured, it is invoked and the + # writer lock retried until either the busy-handler returns 0 or the + # lock is successfully obtained. + # + # EVIDENCE-OF: R-48107-00250 The busy-handler is also invoked while + # waiting for database readers as described above. + # + do_test $tp.7 { + list [catchsql COMMIT db2] [catchsql COMMIT db3] + } [list \ + {1 {cannot commit - no transaction is active}} \ + {1 {cannot commit - no transaction is active}} \ + ] + + # EVIDENCE-OF: R-29177-48281 It then checkpoints all frames in the log + # file and syncs the database file. + # + do_test $tp.8 { + forcecopy test.db abc.db + sqlite3 db4 abc.db + db4 eval { SELECT * FROM t1 UNION ALL SELECT * FROM t2 } + } {1 2 3 4 5 6 7 8 9 10} + do_test $tp.9 { set ::sync_counter } 1 + db4 close + + # EVIDENCE-OF: R-51867-44713 This mode blocks new database writers + # while it is pending, but new database readers are allowed to continue + # unimpeded. + # + # EVIDENCE-OF: R-47276-58266 Like SQLITE_CHECKPOINT_FULL, this mode + # blocks new database writer attempts while it is pending, but does not + # impede readers. + # + # The first of the above two refers to "full" mode. The second + # to "restart". + # + do_test $tp.10.1 { + list $::write_ok $::read_ok + } {0 1} + + # EVIDENCE-OF: R-12410-31217 This mode works the same way as + # SQLITE_CHECKPOINT_FULL with the addition that after checkpointing the + # log file it blocks (calls the busy-handler callback) until all + # readers are reading from the database file only. + # + # The stuff above passed, so the first part of this requirement + # is met. The second part is tested below. If the checkpoint mode + # was "restart" or "truncate", then the busy-handler will have + # been called to block on wal-file readers. + # + do_test $tp.11 { + set ::did_restart_blocking + } [expr {($mode=="restart"||$mode=="truncate")&&$busy_handler_mode==3}] + + # EVIDENCE-OF: R-44699-57140 This mode works the same way as + # SQLITE_CHECKPOINT_RESTART with the addition that it also truncates + # the log file to zero bytes just prior to a successful return. + if {$mode=="truncate" && $busy_handler_mode==3} { + do_test $tp.12 { + file size test.db-wal + } 0 + } + } elseif {$busy_handler_mode==1} { + + # EVIDENCE-OF: R-34519-06271 SQLITE_BUSY is returned in this case. + if {$tn!=2} { + # ($tn==2) is the loop that uses "PRAGMA wal_checkpoint" + do_test $tp.13 { sqlite3_errcode db } {SQLITE_BUSY} + } + + # EVIDENCE-OF: R-49155-63541 If the busy-handler returns 0 before the + # writer lock is obtained or while waiting for database readers, the + # checkpoint operation proceeds from that point in the same way as + # SQLITE_CHECKPOINT_PASSIVE - checkpointing as many frames as possible + # without blocking any further. + do_test $tp.14 { + forcecopy test.db abc.db + sqlite3 db4 abc.db + db4 eval { SELECT * FROM t1 UNION ALL SELECT * FROM t2 } + } {1 2 3 4 5 6} + do_test $tp.15 { set ::sync_counter } 0 + do_test $tp.16 { set ::busy_handler_counter } 1 + db4 close + } + } + + db2 close + db3 close + db5 close + db6 close + } + + db close + tvfs delete +} + +#----------------------------------------------------------------------- +# EVIDENCE-OF: R-03996-12088 The M parameter must be a valid checkpoint +# mode: +# +# Valid checkpoint modes are 0, 1, 2 and 3. +# +sqlite3 db test.db +foreach {tn mode res} { + 0 -1001 {1 {SQLITE_MISUSE - not an error}} + 1 -1 {1 {SQLITE_MISUSE - not an error}} + 2 0 {0 {0 -1 -1}} + 3 1 {0 {0 -1 -1}} + 4 2 {0 {0 -1 -1}} + 5 3 {0 {0 -1 -1}} + 6 4 {1 {SQLITE_MISUSE - not an error}} + 7 114 {1 {SQLITE_MISUSE - not an error}} + 8 1000000 {1 {SQLITE_MISUSE - not an error}} +} { + do_test 4.$tn { + list [catch "wal_checkpoint_v2 db $mode" msg] $msg + } $res +} +db close + +foreach tn {1 2 3} { + forcedelete test.db test.db2 test.db3 + testvfs tvfs + + sqlite3 db test.db -vfs tvfs + execsql { + ATTACH 'test.db2' AS aux2; + ATTACH 'test.db3' AS aux3; + PRAGMA main.journal_mode = WAL; + PRAGMA aux2.journal_mode = WAL; + PRAGMA aux3.journal_mode = WAL; + + CREATE TABLE main.t1(x,y); + CREATE TABLE aux2.t2(x,y); + CREATE TABLE aux3.t3(x,y); + + INSERT INTO t1 VALUES('a', 'b'); + INSERT INTO t2 VALUES('a', 'b'); + INSERT INTO t3 VALUES('a', 'b'); + } + sqlite3 db2 test.db2 -vfs tvfs + + switch -- $tn { + 1 { + # EVIDENCE-OF: R-41299-52117 If no error (SQLITE_BUSY or otherwise) is + # encountered while processing the attached databases, SQLITE_OK is + # returned. + do_test 5.$tn.1 { + lindex [wal_checkpoint_v2 db truncate] 0 + } {0} ;# 0 -> SQLITE_OK + do_test 5.$tn.2 { + list [expr [file size test.db-wal]==0] \ + [expr [file size test.db2-wal]==0] \ + [expr [file size test.db3-wal]==0] + } {1 1 1} + } + + 2 { + # EVIDENCE-OF: R-38578-34175 If an SQLITE_BUSY error is encountered when + # processing one or more of the attached WAL databases, the operation is + # still attempted on any remaining attached databases and SQLITE_BUSY is + # returned at the end. + db2 eval { BEGIN; INSERT INTO t2 VALUES('d', 'e'); } + do_test 5.$tn.1 { + lindex [wal_checkpoint_v2 db truncate] 0 + } {1} ;# 1 -> SQLITE_BUSY + do_test 5.$tn.2 { + list [expr [file size test.db-wal]==0] \ + [expr [file size test.db2-wal]==0] \ + [expr [file size test.db3-wal]==0] + } {1 0 1} + db2 eval ROLLBACK + } + + 3 { + # EVIDENCE-OF: R-38049-07913 If any other error occurs while processing + # an attached database, processing is abandoned and the error code is + # returned to the caller immediately. + tvfs filter xWrite + tvfs script inject_ioerr + proc inject_ioerr {method file args} { + if {[file tail $file]=="test.db2"} { + return "SQLITE_IOERR" + } + return 0 + } + do_test 5.$tn.1 { + list [catch { wal_checkpoint_v2 db truncate } msg] $msg + } {1 {SQLITE_IOERR - disk I/O error}} + do_test 5.$tn.2 { + list [expr [file size test.db-wal]==0] \ + [expr [file size test.db2-wal]==0] \ + [expr [file size test.db3-wal]==0] + } {1 0 0} + tvfs script "" + } + } + + db close + db2 close +} + +reset_db +sqlite3 db2 test.db + +do_test 6.1 { + execsql { + PRAGMA auto_vacuum = 0; + PRAGMA journal_mode = WAL; + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + } + file size test.db-wal +} [wal_file_size 3 1024] + +do_test 6.2 { + db2 eval { BEGIN; SELECT * FROM t1; } + db eval { INSERT INTO t1 VALUES(3, 4) } + file size test.db-wal +} [wal_file_size 4 1024] + +# At this point the log file contains 4 frames. 3 of which it should +# be possible to checkpoint. +# +# EVIDENCE-OF: R-16642-42503 If pnLog is not NULL, then *pnLog is set to +# the total number of frames in the log file or to -1 if the checkpoint +# could not run because of an error or because the database is not in +# WAL mode. +# +# EVIDENCE-OF: R-10514-25250 If pnCkpt is not NULL,then *pnCkpt is set +# to the total number of checkpointed frames in the log file (including +# any that were already checkpointed before the function was called) or +# to -1 if the checkpoint could not run due to an error or because the +# database is not in WAL mode. +# +do_test 6.4 { + lrange [wal_checkpoint_v2 db passive] 1 2 +} {4 3} + +# EVIDENCE-OF: R-37257-17813 Note that upon successful completion of an +# SQLITE_CHECKPOINT_TRUNCATE, the log file will have been truncated to +# zero bytes and so both *pnLog and *pnCkpt will be set to zero. +# +do_test 6.5 { + db2 eval COMMIT + wal_checkpoint_v2 db truncate +} {0 0 0} + + + +finish_test + diff --git a/test/e_walhook.test b/test/e_walhook.test new file mode 100644 index 0000000000..c8c8819493 --- /dev/null +++ b/test/e_walhook.test @@ -0,0 +1,200 @@ +# 2014 December 04 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/wal_common.tcl +set testprefix e_walhook + + +# EVIDENCE-OF: R-00752-43975 The sqlite3_wal_hook() function is used to +# register a callback that is invoked each time data is committed to a +# database in wal mode. +# +# 1.1: shows that the wal-hook is not invoked in rollback mode. +# 1.2: but is invoked in wal mode. +# +set ::wal_hook_count 0 +proc my_wal_hook {args} { + incr ::wal_hook_count + return 0 +} + +do_test 1.1.1 { + db wal_hook my_wal_hook + execsql { + CREATE TABLE t1(x); + INSERT INTO t1 VALUES(1); + } + set ::wal_hook_count +} 0 +do_test 1.1.2 { + execsql { PRAGMA journal_mode = wal } + set ::wal_hook_count +} 0 + +do_test 1.3 { + execsql { INSERT INTO t1 VALUES(2) } + set wal_hook_count +} 1 + +do_test 1.4 { + execsql { + BEGIN; + INSERT INTO t1 VALUES(3); + INSERT INTO t1 VALUES(4); + COMMIT; + } + set wal_hook_count +} 2 + +# EVIDENCE-OF: R-65366-15139 The callback is invoked by SQLite after the +# commit has taken place and the associated write-lock on the database +# released +# +set ::read_ok 0 +proc my_wal_hook {args} { + sqlite3 db2 test.db + if {[db2 eval { SELECT * FROM t1 }] == "1 2 3 4 5"} { + set ::read_ok 1 + } + db2 close +} +do_test 2.1 { + execsql { INSERT INTO t1 VALUES(5) } + set ::read_ok +} 1 + +# EVIDENCE-OF: R-44294-52863 The third parameter is the name of the +# database that was written to - either "main" or the name of an +# ATTACH-ed database. +# +# EVIDENCE-OF: R-18913-19355 The fourth parameter is the number of pages +# currently in the write-ahead log file, including those that were just +# committed. +# +set ::wal_hook_args [list] +proc my_wal_hook {dbname nEntry} { + set ::wal_hook_args [list $dbname $nEntry] +} +forcedelete test.db2 +do_test 3.0 { + execsql { + ATTACH 'test.db2' AS aux; + CREATE TABLE aux.t2(x); + PRAGMA aux.journal_mode = wal; + } +} {wal} + +# Database "aux" +do_test 3.1.1 { + set wal_hook_args [list] + execsql { INSERT INTO t2 VALUES('a') } +} {} +do_test 3.1.2 { + set wal_hook_args +} [list aux [wal_frame_count test.db2-wal 1024]] + +# Database "main" +do_test 3.2.1 { + set wal_hook_args [list] + execsql { INSERT INTO t1 VALUES(6) } +} {} +do_test 3.1.2 { + set wal_hook_args +} [list main [wal_frame_count test.db-wal 1024]] + +# EVIDENCE-OF: R-14034-00929 If an error code is returned, that error +# will propagate back up through the SQLite code base to cause the +# statement that provoked the callback to report an error, though the +# commit will have still occurred. +# +proc my_wal_hook {args} { return 1 ;# SQLITE_ERROR } +do_catchsql_test 4.1 { + INSERT INTO t1 VALUES(7) +} {1 {SQL logic error or missing database}} + +proc my_wal_hook {args} { return 5 ;# SQLITE_BUSY } +do_catchsql_test 4.2 { + INSERT INTO t1 VALUES(8) +} {1 {database is locked}} + +proc my_wal_hook {args} { return 14 ;# SQLITE_CANTOPEN } +do_catchsql_test 4.3 { + INSERT INTO t1 VALUES(9) +} {1 {unable to open database file}} + +do_execsql_test 4.4 { + SELECT * FROM t1 +} {1 2 3 4 5 6 7 8 9} + +# EVIDENCE-OF: R-10466-53920 Calling sqlite3_wal_hook() replaces any +# previously registered write-ahead log callback. +set ::old_wal_hook 0 +proc my_old_wal_hook {args} { + incr ::old_wal_hook + return 0 +} +db wal_hook my_old_wal_hook +do_test 5.1 { + execsql { INSERT INTO t1 VALUES(10) } + set ::old_wal_hook +} {1} + +# Replace old_wal_hook. Observe that it is not invoked after it has +# been replaced. +proc my_new_wal_hook {args} { return 0 } +db wal_hook my_new_wal_hook +do_test 5.2 { + execsql { INSERT INTO t1 VALUES(11) } + set ::old_wal_hook +} {1} + + + +# EVIDENCE-OF: R-42842-27162 Note that the sqlite3_wal_autocheckpoint() +# interface and the wal_autocheckpoint pragma both invoke +# sqlite3_wal_hook() and will those overwrite any prior +# sqlite3_wal_hook() settings. +# +set ::old_wal_hook 0 +proc my_old_wal_hook {args} { incr ::old_wal_hook ; return 0 } +db wal_hook my_old_wal_hook +do_test 6.1.1 { + execsql { INSERT INTO t1 VALUES(12) } + set ::old_wal_hook +} {1} +do_test 6.1.2 { + execsql { PRAGMA wal_autocheckpoint = 1000 } + execsql { INSERT INTO t1 VALUES(12) } + set ::old_wal_hook +} {1} + +# EVIDENCE-OF: R-52629-38967 The first parameter passed to the callback +# function when it is invoked is a copy of the third parameter passed to +# sqlite3_wal_hook() when registering the callback. +# +# This is tricky to test using the tcl interface. However, the +# mechanism used to invoke the tcl script registered as a wal-hook +# depends on the context pointer being correctly passed through. And +# since multiple different wal-hook scripts have been successfully +# invoked by this test script, consider this tested. +# +# EVIDENCE-OF: R-23378-42536 The second is a copy of the database +# handle. +# +# There is an assert() in the C wal-hook used by tclsqlite.c to +# prove this. And that hook has been invoked multiple times when +# running this script. So consider this requirement tested as well. +# + +finish_test diff --git a/test/eval.test b/test/eval.test index 912dc8215b..360d158f3c 100644 --- a/test/eval.test +++ b/test/eval.test @@ -58,6 +58,18 @@ do_test eval-2.2 { SELECT * FROM t2 } } {} +do_test eval-2.3 { + execsql { + INSERT INTO t2 SELECT x, x+1 FROM t1 WHERE x<5; + SELECT x, test_eval('DELETE FROM t2 WHERE x='||x), y FROM t2 + ORDER BY rowid DESC; + } +} {4 {} {} 3 {} {} 2 {} {} 1 {} {}} +do_test eval-2.4 { + execsql { + SELECT * FROM t2 + } +} {} # Modify a row while it is being read. # diff --git a/test/expr.test b/test/expr.test index cc4c9c67f1..8d913d2a1a 100644 --- a/test/expr.test +++ b/test/expr.test @@ -205,6 +205,10 @@ test_expr expr-1.125 {i1=6, i2=NULL} \ test_expr expr-1.126 {i1=8, i2=8} \ {CASE WHEN i1 IS NOT i2 THEN 'yes' ELSE 'no' END} no +do_catchsql_test expr-1.127 { + SELECT 1 IS #1; +} {1 {near "#1": syntax error}} + ifcapable floatingpoint {if {[working_64bit_int]} { test_expr expr-1.200\ {i1=9223372036854775806, i2=1} {i1+i2} 9223372036854775807 diff --git a/test/fkey2.test b/test/fkey2.test index 4c8daa0b02..53b90dc91c 100644 --- a/test/fkey2.test +++ b/test/fkey2.test @@ -1554,7 +1554,7 @@ ifcapable auth { } } {} - proc auth {args} {eval lappend ::authargs $args ; return SQLITE_OK} + proc auth {args} {eval lappend ::authargs [lrange $args 0 4]; return SQLITE_OK} db auth auth # An insert on the parent table must read the child key of any deferred diff --git a/test/fkey5.test b/test/fkey5.test index 5aa8b1d4b7..dc866cbdb8 100644 --- a/test/fkey5.test +++ b/test/fkey5.test @@ -252,13 +252,17 @@ do_test fkey5-6.5 { } {c12 1 p4 0 c12 3 p4 0 c12 6 p4 0} do_test fkey5-7.1 { + set res {} db eval { INSERT OR IGNORE INTO c13 SELECT * FROM c12; INSERT OR IGNORE INTO C14 SELECT * FROM c12; DELETE FROM c12; PRAGMA foreign_key_check; + } { + lappend res [list $table $rowid $fkid $parent] } -} {c14 1 p4 0 c14 3 p4 0 c14 6 p4 0 c13 1 p3 0 c13 2 p3 0 c13 3 p3 0 c13 4 p3 0 c13 5 p3 0 c13 6 p3 0} + lsort $res +} {{c13 1 0 p3} {c13 2 0 p3} {c13 3 0 p3} {c13 4 0 p3} {c13 5 0 p3} {c13 6 0 p3} {c14 1 0 p4} {c14 3 0 p4} {c14 6 0 p4}} do_test fkey5-7.2 { db eval { PRAGMA foreign_key_check(c14); diff --git a/test/fkey7.test b/test/fkey7.test index c2682edbe5..6c646a9a7f 100644 --- a/test/fkey7.test +++ b/test/fkey7.test @@ -50,5 +50,22 @@ do_tblsread_test 1.3 { UPDATE par SET a=? WHERE b=? } {c1 c2 par} do_tblsread_test 1.4 { UPDATE par SET c=? WHERE b=? } {c3 par} do_tblsread_test 1.5 { UPDATE par SET a=?,b=?,c=? WHERE b=? } {c1 c2 c3 par s1} +ifcapable incrblob { + do_execsql_test 2.0 { + CREATE TABLE pX(x PRIMARY KEY); + CREATE TABLE cX(a INTEGER PRIMARY KEY, b REFERENCES pX); + } + + do_catchsql_test 2.1 { + INSERT INTO cX VALUES(11, zeroblob(40)); + } {1 {FOREIGN KEY constraint failed}} + + do_test 2.2 { + set stmt [sqlite3_prepare_v2 db "INSERT INTO cX VALUES(11, ?)" -1] + sqlite3_bind_zeroblob $stmt 1 45 + sqlite3_step $stmt + sqlite3_finalize $stmt + } {SQLITE_CONSTRAINT} +} finish_test diff --git a/test/fkey8.test b/test/fkey8.test new file mode 100644 index 0000000000..60bb8e4640 --- /dev/null +++ b/test/fkey8.test @@ -0,0 +1,106 @@ +# 2001 September 15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# +# This file implements tests for foreign keys. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix fkey8 + +ifcapable {!foreignkey} { + finish_test + return +} +do_execsql_test 1.0 { PRAGMA foreign_keys = 1; } + + +foreach {tn use_stmt sql schema} { + 1 1 "DELETE FROM p1" { + CREATE TABLE p1(a PRIMARY KEY); + CREATE TABLE c1(b REFERENCES p1); + } + + 2.1 0 "DELETE FROM p1" { + CREATE TABLE p1(a PRIMARY KEY); + CREATE TABLE c1(b REFERENCES p1 ON DELETE CASCADE); + } + 2.2 0 "DELETE FROM p1" { + CREATE TABLE p1(a PRIMARY KEY); + CREATE TABLE c1(b REFERENCES p1 ON DELETE SET NULL); + } + 2.3 1 "DELETE FROM p1" { + CREATE TABLE p1(a PRIMARY KEY); + CREATE TABLE c1(b REFERENCES p1 ON DELETE SET DEFAULT); + } + + 3 1 "DELETE FROM p1" { + CREATE TABLE p1(a PRIMARY KEY); + CREATE TABLE c1(b REFERENCES p1 ON DELETE CASCADE); + CREATE TRIGGER ct1 AFTER DELETE ON c1 BEGIN + INSERT INTO p1 VALUES('x'); + END; + } + + 4 1 "DELETE FROM p1" { + CREATE TABLE p1(a PRIMARY KEY); + CREATE TABLE c1(b REFERENCES p1 ON DELETE CASCADE, c PRIMARY KEY); + CREATE TABLE cc1(d REFERENCES c1); + } + + 5.1 0 "DELETE FROM p1" { + CREATE TABLE p1(a PRIMARY KEY); + CREATE TABLE c1(b REFERENCES p1 ON DELETE CASCADE, c PRIMARY KEY); + CREATE TABLE cc1(d REFERENCES c1 ON DELETE CASCADE); + } + 5.2 0 "DELETE FROM p1" { + CREATE TABLE p1(a PRIMARY KEY); + CREATE TABLE c1(b REFERENCES p1 ON DELETE CASCADE, c PRIMARY KEY); + CREATE TABLE cc1(d REFERENCES c1 ON DELETE SET NULL); + } + 5.3 1 "DELETE FROM p1" { + CREATE TABLE p1(a PRIMARY KEY); + CREATE TABLE c1(b REFERENCES p1 ON DELETE CASCADE, c PRIMARY KEY); + CREATE TABLE cc1(d REFERENCES c1 ON DELETE SET DEFAULT); + } + + 6.1 1 "UPDATE p1 SET a = ?" { + CREATE TABLE p1(a PRIMARY KEY); + CREATE TABLE c1(b REFERENCES p1 ON UPDATE SET NULL, c); + } + 6.2 0 "UPDATE OR IGNORE p1 SET a = ?" { + CREATE TABLE p1(a PRIMARY KEY); + CREATE TABLE c1(b REFERENCES p1 ON UPDATE SET NULL, c); + } + 6.3 1 "UPDATE OR IGNORE p1 SET a = ?" { + CREATE TABLE p1(a PRIMARY KEY); + CREATE TABLE c1(b REFERENCES p1 ON UPDATE CASCADE, c); + } + 6.4 1 "UPDATE OR IGNORE p1 SET a = ?" { + CREATE TABLE p1(a PRIMARY KEY); + CREATE TABLE c1(b NOT NULL REFERENCES p1 ON UPDATE SET NULL, c); + } + +} { + drop_all_tables + do_test 1.$tn { + execsql $schema + set stmt [sqlite3_prepare_v2 db $sql -1 dummy] + set ret [uses_stmt_journal $stmt] + sqlite3_finalize $stmt + set ret + } $use_stmt +} + + +finish_test + diff --git a/test/fts3expr4.test b/test/fts3expr4.test index 94737971b4..4bc62030d5 100644 --- a/test/fts3expr4.test +++ b/test/fts3expr4.test @@ -24,12 +24,16 @@ ifcapable !fts3||!icu { set sqlite_fts3_enable_parentheses 1 -proc test_icu_fts3expr {expr} { - db one {SELECT fts3_exprtest('icu', $expr, 'a', 'b', 'c')} +proc test_fts3expr {tokenizer expr} { + db one {SELECT fts3_exprtest($tokenizer, $expr, 'a', 'b', 'c')} } proc do_icu_expr_test {tn expr res} { - uplevel [list do_test $tn [list test_icu_fts3expr $expr] $res] + uplevel [list do_test $tn [list test_fts3expr icu $expr] [list {*}$res]] +} + +proc do_simple_expr_test {tn expr res} { + uplevel [list do_test $tn [list test_fts3expr simple $expr] [list {*}$res]] } #------------------------------------------------------------------------- @@ -53,5 +57,26 @@ do_icu_expr_test 2.1 { f (e NEAR/2 a) } {AND {AND {AND {PHRASE 3 0 f} {PHRASE 3 0 (}} {NEAR/2 {PHRASE 3 0 e} {PHRASE 3 0 a}}} {PHRASE 3 0 )}} +#------------------------------------------------------------------------- +# +do_simple_expr_test 3.1 {*lOl* *h4h*} { + AND {PHRASE 3 0 lol+} {PHRASE 3 0 h4h+} +} + +do_icu_expr_test 3.2 {*lOl* *h4h*} { + AND {AND {AND {PHRASE 3 0 *} {PHRASE 3 0 lol+}} {PHRASE 3 0 *}} {PHRASE 3 0 h4h+} +} + +do_simple_expr_test 3.3 { * } { } +do_simple_expr_test 3.4 { *a } { PHRASE 3 0 a } +do_simple_expr_test 3.5 { a*b } { AND {PHRASE 3 0 a+} {PHRASE 3 0 b} } +do_simple_expr_test 3.6 { *a*b } { AND {PHRASE 3 0 a+} {PHRASE 3 0 b} } +do_simple_expr_test 3.7 { *"abc" } { PHRASE 3 0 abc } +do_simple_expr_test 3.8 { "abc"* } { PHRASE 3 0 abc } +do_simple_expr_test 3.8 { "ab*c" } { PHRASE 3 0 ab+ c } + +do_icu_expr_test 3.9 { "ab*c" } { PHRASE 3 0 ab+ * c } +do_icu_expr_test 3.10 { ab*c } { AND {PHRASE 3 0 ab+} {PHRASE 3 0 c}} + finish_test diff --git a/test/fts3matchinfo.test b/test/fts3matchinfo.test index fd475af2e4..36c9121118 100644 --- a/test/fts3matchinfo.test +++ b/test/fts3matchinfo.test @@ -433,4 +433,21 @@ do_execsql_test 9.1 { SELECT snippet(ft2, '[', ']', '', -1, 1) FROM ft2 WHERE ft2 MATCH 'c'; } {{[c]} {[c]}} +#--------------------------------------------------------------------------- +# Test for a memory leak +# +do_execsql_test 10.1 { + DROP TABLE t10; + CREATE VIRTUAL TABLE t10 USING fts4(idx, value); + INSERT INTO t10 values (1, 'one'),(2, 'two'),(3, 'three'); + SELECT docId, t10.* + FROM t10 + JOIN (SELECT 1 AS idx UNION SELECT 2 UNION SELECT 3) AS x + WHERE t10 MATCH x.idx + AND matchinfo(t10) not null + GROUP BY docId + ORDER BY 1; +} {1 1 one 2 2 two 3 3 three} + + finish_test diff --git a/test/fts4aa.test b/test/fts4aa.test index 88550c99f7..e6c7f9336e 100644 --- a/test/fts4aa.test +++ b/test/fts4aa.test @@ -170,7 +170,7 @@ foreach {q r} [array get fts4aa_res] { # Should get the same search results when an authorizer prevents # all PRAGMA statements. # -proc no_pragma_auth {code arg1 arg2 arg3 arg4} { +proc no_pragma_auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_PRAGMA"} {return SQLITE_DENY} return SQLITE_OK; } diff --git a/test/func3.test b/test/func3.test index b5a082a4ca..3b1613b56c 100644 --- a/test/func3.test +++ b/test/func3.test @@ -153,28 +153,38 @@ do_test func3-5.39 { db eval {EXPLAIN SELECT unlikely(min(1.0+'2.0',4*11))} } [db eval {EXPLAIN SELECT min(1.0+'2.0',4*11)}] -do_execsql_test func3-5.40 { + +# EVIDENCE-OF: R-23735-03107 The likely(X) function returns the argument +# X unchanged. +# +do_execsql_test func3-5.50 { SELECT likely(9223372036854775807); } {9223372036854775807} -do_execsql_test func3-5.41 { +do_execsql_test func3-5.51 { SELECT likely(-9223372036854775808); } {-9223372036854775808} -do_execsql_test func3-5.42 { +do_execsql_test func3-5.52 { SELECT likely(14.125); } {14.125} -do_execsql_test func3-5.43 { +do_execsql_test func3-5.53 { SELECT likely(NULL); } {{}} -do_execsql_test func3-5.44 { +do_execsql_test func3-5.54 { SELECT likely('test-string'); } {test-string} -do_execsql_test func3-5.45 { +do_execsql_test func3-5.55 { SELECT quote(likely(x'010203000405')); } {X'010203000405'} -do_test func3-5.49 { + +# EVIDENCE-OF: R-43464-09689 The likely(X) function is a no-op that the +# code generator optimizes away so that it consumes no CPU cycles at +# run-time (that is, during calls to sqlite3_step()). +# +do_test func3-5.59 { db eval {EXPLAIN SELECT likely(min(1.0+'2.0',4*11))} } [db eval {EXPLAIN SELECT min(1.0+'2.0',4*11)}] + finish_test diff --git a/test/hexlit.test b/test/hexlit.test new file mode 100644 index 0000000000..10909e6f4c --- /dev/null +++ b/test/hexlit.test @@ -0,0 +1,114 @@ +# 2014-07-23 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file implements tests for hexadecimal literals + + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +proc hexlit1 {tnum val ans} { + do_execsql_test hexlit-$tnum "SELECT $val" $ans +} + +hexlit1 100 0x0 0 +hexlit1 101 0x0000000000000000000000000000000000000000000001 1 +hexlit1 102 0x2 2 +hexlit1 103 0x4 4 +hexlit1 104 0x8 8 +hexlit1 105 0x00000000000000000000000000000000000000000000010 16 +hexlit1 103 0x20 32 +hexlit1 106 0x40 64 +hexlit1 107 0x80 128 +hexlit1 108 0x100 256 +hexlit1 109 0x200 512 +hexlit1 110 0X400 1024 +hexlit1 111 0x800 2048 +hexlit1 112 0x1000 4096 +hexlit1 113 0x2000 8192 +hexlit1 114 0x4000 16384 +hexlit1 115 0x8000 32768 +hexlit1 116 0x10000 65536 +hexlit1 117 0x20000 131072 +hexlit1 118 0x40000 262144 +hexlit1 119 0x80000 524288 +hexlit1 120 0x100000 1048576 +hexlit1 121 0x200000 2097152 +hexlit1 122 0x400000 4194304 +hexlit1 123 0x800000 8388608 +hexlit1 124 0x1000000 16777216 +hexlit1 125 0x2000000 33554432 +hexlit1 126 0x4000000 67108864 +hexlit1 127 0x8000000 134217728 +hexlit1 128 0x10000000 268435456 +hexlit1 129 0x20000000 536870912 +hexlit1 130 0x40000000 1073741824 +hexlit1 131 0x80000000 2147483648 +hexlit1 132 0x100000000 4294967296 +hexlit1 133 0x200000000 8589934592 +hexlit1 134 0x400000000 17179869184 +hexlit1 135 0x800000000 34359738368 +hexlit1 136 0x1000000000 68719476736 +hexlit1 137 0x2000000000 137438953472 +hexlit1 138 0x4000000000 274877906944 +hexlit1 139 0x8000000000 549755813888 +hexlit1 140 0x10000000000 1099511627776 +hexlit1 141 0x20000000000 2199023255552 +hexlit1 142 0x40000000000 4398046511104 +hexlit1 143 0x80000000000 8796093022208 +hexlit1 144 0x100000000000 17592186044416 +hexlit1 145 0x200000000000 35184372088832 +hexlit1 146 0x400000000000 70368744177664 +hexlit1 147 0x800000000000 140737488355328 +hexlit1 148 0x1000000000000 281474976710656 +hexlit1 149 0x2000000000000 562949953421312 +hexlit1 150 0x4000000000000 1125899906842624 +hexlit1 151 0x8000000000000 2251799813685248 +hexlit1 152 0x10000000000000 4503599627370496 +hexlit1 153 0x20000000000000 9007199254740992 +hexlit1 154 0x40000000000000 18014398509481984 +hexlit1 155 0x80000000000000 36028797018963968 +hexlit1 156 0x100000000000000 72057594037927936 +hexlit1 157 0x200000000000000 144115188075855872 +hexlit1 158 0x400000000000000 288230376151711744 +hexlit1 159 0x800000000000000 576460752303423488 +hexlit1 160 0X1000000000000000 1152921504606846976 +hexlit1 161 0x2000000000000000 2305843009213693952 +hexlit1 162 0X4000000000000000 4611686018427387904 +hexlit1 163 0x8000000000000000 -9223372036854775808 +hexlit1 164 0XFFFFFFFFFFFFFFFF -1 + +for {set n 1} {$n < 0x10} {incr n} { + hexlit1 200.$n.1 0X[format %03X $n] $n + hexlit1 200.$n.2 0x[format %03X $n] $n + hexlit1 200.$n.3 0X[format %03x $n] $n + hexlit1 200.$n.4 0x[format %03x $n] $n +} + +# String literals that look like hex do not get cast or coerced. +# +do_execsql_test hexlit-300 { + CREATE TABLE t1(x INT, y REAL); + INSERT INTO t1 VALUES('1234','4567'),('0x1234','0x4567'); + SELECT typeof(x), x, typeof(y), y, '#' FROM t1 ORDER BY rowid; +} {integer 1234 real 4567.0 # text 0x1234 text 0x4567 #} +do_execsql_test hexlit-301 { + SELECT CAST('0x1234' AS INTEGER); +} {0} + +# Oversized hex literals are rejected +# +do_catchsql_test hexlist-400 { + SELECT 0x10000000000000000; +} {1 {hex literal too big: 0x10000000000000000}} + + +finish_test diff --git a/test/in4.test b/test/in4.test index 0a4a75008b..a89961f82b 100644 --- a/test/in4.test +++ b/test/in4.test @@ -231,11 +231,11 @@ do_execsql_test in4-3.44 { SELECT * FROM t3 WHERE x IN (10); } {~/OpenEphemeral/} do_execsql_test in4-3.45 { - SELECT * FROM t3 WHERE x NOT IN (10,11); + SELECT * FROM t3 WHERE x NOT IN (10,11,99999); } {1 1 1} do_execsql_test in4-3.46 { EXPLAIN - SELECT * FROM t3 WHERE x NOT IN (10,11); + SELECT * FROM t3 WHERE x NOT IN (10,11,99999); } {/OpenEphemeral/} do_execsql_test in4-3.47 { SELECT * FROM t3 WHERE x NOT IN (10); diff --git a/test/in5.test b/test/in5.test index 8a43b8d44a..67d212589d 100644 --- a/test/in5.test +++ b/test/in5.test @@ -12,6 +12,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +set testprefix in5 do_test in5-1.1 { execsql { @@ -135,4 +136,51 @@ do_test in5-5.3 { }] } {0} +#------------------------------------------------------------------------- +# At one point SQLite was removing the DISTINCT keyword from expressions +# similar to: +# +# IN (SELECT DISTINCT FROM...) +# +# However, there are a few obscure cases where this is incorrect. For +# example, if the SELECT features a LIMIT clause, or if the collation +# sequence or affinity used by the DISTINCT does not match the one used +# by the IN(...) expression. +# +do_execsql_test 6.1.1 { + CREATE TABLE t1(a COLLATE nocase); + INSERT INTO t1 VALUES('one'); + INSERT INTO t1 VALUES('ONE'); +} +do_execsql_test 6.1.2 { + SELECT count(*) FROM t1 WHERE a COLLATE BINARY IN (SELECT DISTINCT a FROM t1) +} {1} + +do_execsql_test 6.2.1 { + CREATE TABLE t3(a, b); + INSERT INTO t3 VALUES(1, 1); + INSERT INTO t3 VALUES(1, 2); + INSERT INTO t3 VALUES(1, 3); + INSERT INTO t3 VALUES(2, 4); + INSERT INTO t3 VALUES(2, 5); + INSERT INTO t3 VALUES(2, 6); + INSERT INTO t3 VALUES(3, 7); + INSERT INTO t3 VALUES(3, 8); + INSERT INTO t3 VALUES(3, 9); +} +do_execsql_test 6.2.2 { + SELECT count(*) FROM t3 WHERE b IN (SELECT DISTINCT a FROM t3 LIMIT 5); +} {3} +do_execsql_test 6.2.3 { + SELECT count(*) FROM t3 WHERE b IN (SELECT a FROM t3 LIMIT 5); +} {2} + +do_execsql_test 6.3.1 { + CREATE TABLE x1(a); + CREATE TABLE x2(b); + INSERT INTO x1 VALUES(1), (1), (2); + INSERT INTO x2 VALUES(1), (2); + SELECT count(*) FROM x2 WHERE b IN (SELECT DISTINCT a FROM x1 LIMIT 2); +} {2} + finish_test diff --git a/test/incrblob_err.test b/test/incrblob_err.test index 35d37fee92..a08bea3e4e 100644 --- a/test/incrblob_err.test +++ b/test/incrblob_err.test @@ -14,6 +14,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +set ::testprefix incrblob_err ifcapable {!incrblob || !memdebug || !tclvar} { finish_test diff --git a/test/index5.test b/test/index5.test index 7895391309..e369d75717 100644 --- a/test/index5.test +++ b/test/index5.test @@ -16,6 +16,9 @@ source $testdir/tester.tcl set ::testprefix index5 do_test 1.1 { + if {[permutation]=="memsubsys1"} { + execsql { PRAGMA auto_vacuum = 0; } + } execsql { PRAGMA page_size = 1024; CREATE TABLE t1(x); @@ -38,7 +41,7 @@ tvfs filter xWrite tvfs script write_cb proc write_cb {xCall file handle iOfst args} { if {[file tail $file]=="test.db"} { - lappend ::write_list [expr $iOfst/1024] + lappend ::write_list [expr $iOfst/1024 + 1] } } diff --git a/test/index7.test b/test/index7.test index 1c81f6024b..0f341a3129 100644 --- a/test/index7.test +++ b/test/index7.test @@ -248,4 +248,34 @@ do_execsql_test index7-5.0 { SELECT stat+0 FROM sqlite_stat1 WHERE idx='t3b'; } {6 6} +# Verify that the problem identified by ticket [98d973b8f5] has been fixed. +# +do_execsql_test index7-6.1 { + CREATE TABLE t5(a, b); + CREATE TABLE t4(c, d); + INSERT INTO t5 VALUES(1, 'xyz'); + INSERT INTO t4 VALUES('abc', 'not xyz'); + SELECT * FROM (SELECT * FROM t5 WHERE a=1 AND b='xyz'), t4 WHERE c='abc'; +} { + 1 xyz abc {not xyz} +} +do_execsql_test index7-6.2 { + CREATE INDEX i4 ON t4(c) WHERE d='xyz'; + SELECT * FROM (SELECT * FROM t5 WHERE a=1 AND b='xyz'), t4 WHERE c='abc'; +} { + 1 xyz abc {not xyz} +} +do_execsql_test index7-6.3 { + CREATE VIEW v4 AS SELECT * FROM t4; + INSERT INTO t4 VALUES('def', 'xyz'); + SELECT * FROM v4 WHERE d='xyz' AND c='def' +} { + def xyz +} +do_eqp_test index7-6.4 { + SELECT * FROM v4 WHERE d='xyz' AND c='def' +} { + 0 0 0 {SEARCH TABLE t4 USING INDEX i4 (c=?)} +} + finish_test diff --git a/test/ioerr2.test b/test/ioerr2.test index 5150ace3ab..c08c3453f3 100644 --- a/test/ioerr2.test +++ b/test/ioerr2.test @@ -112,6 +112,12 @@ foreach bPersist [list 0 1] { } } +# When this test was written, an IO error within the UPDATE statement caused +# a rollback, which tripped all read-cursors, causing the outer SELECT to +# fail with "abort due to ROLLBACK". Now, the loop continues until the UPDATE +# is run successfully. At this point the next IO error occurs within the +# SELECT - throwing the "disk I/O error" that the test case now expects. +# do_test ioerr2-5 { execsql { CREATE TABLE t2 AS SELECT * FROM t1; @@ -130,7 +136,7 @@ do_test ioerr2-5 { } } msg] list $rc $msg -} {1 {abort due to ROLLBACK}} +} {1 {disk I/O error}} ;# used to be "{1 {abort due to ROLLBACK}}" if {$::tcl_platform(platform) == "unix"} { # Cause the call to xAccess used by [pragma temp_store_directory] to diff --git a/test/join.test b/test/join.test index 28d9ddf8b2..4c83fa6b3b 100644 --- a/test/join.test +++ b/test/join.test @@ -36,6 +36,17 @@ do_test join-1.2 { } } {1 2 3 2 3 4 3 4 5} +# A FROM clause of the form: "
    • ,
    ON " is not +# allowed by the SQLite syntax diagram, nor by any other SQL database +# engine that we are aware of. Nevertheless, historic versions of +# SQLite have allowed it. We need to continue to support it moving +# forward to prevent breakage of legacy applications. Though, we will +# not advertise it as being supported. +# +do_execsql_test join-1.2.1 { + SELECT t1.rowid, t2.rowid, '|' FROM t1, t2 ON t1.a=t2.b; +} {1 1 | 2 2 | 3 3 |} + do_test join-1.3 { execsql2 { SELECT * FROM t1 NATURAL JOIN t2; diff --git a/test/join5.test b/test/join5.test index 45d8a31596..b0b0df4f9d 100644 --- a/test/join5.test +++ b/test/join5.test @@ -106,5 +106,59 @@ do_test join5-2.12 { execsql {SELECT * FROM xy LEFT JOIN ab ON NULL WHERE NULL} } {} +# Ticket https://www.sqlite.org/src/tktview/6f2222d550f5b0ee7ed37601 +# Incorrect output on a LEFT JOIN. +# +do_execsql_test join5-3.1 { + DROP TABLE IF EXISTS t1; + DROP TABLE IF EXISTS t2; + DROP TABLE IF EXISTS t3; + CREATE TABLE x1(a); + INSERT INTO x1 VALUES(1); + CREATE TABLE x2(b NOT NULL); + CREATE TABLE x3(c, d); + INSERT INTO x3 VALUES('a', NULL); + INSERT INTO x3 VALUES('b', NULL); + INSERT INTO x3 VALUES('c', NULL); + SELECT * FROM x1 LEFT JOIN x2 LEFT JOIN x3 ON x3.d = x2.b; +} {1 {} {} {}} +do_execsql_test join5-3.2 { + DROP TABLE IF EXISTS t1; + DROP TABLE IF EXISTS t2; + DROP TABLE IF EXISTS t3; + DROP TABLE IF EXISTS t4; + DROP TABLE IF EXISTS t5; + CREATE TABLE t1(x text NOT NULL, y text); + CREATE TABLE t2(u text NOT NULL, x text NOT NULL); + CREATE TABLE t3(w text NOT NULL, v text); + CREATE TABLE t4(w text NOT NULL, z text NOT NULL); + CREATE TABLE t5(z text NOT NULL, m text); + INSERT INTO t1 VALUES('f6d7661f-4efe-4c90-87b5-858e61cd178b',NULL); + INSERT INTO t1 VALUES('f6ea82c3-2cad-45ce-ae8f-3ddca4fb2f48',NULL); + INSERT INTO t1 VALUES('f6f47499-ecb4-474b-9a02-35be73c235e5',NULL); + INSERT INTO t1 VALUES('56f47499-ecb4-474b-9a02-35be73c235e5',NULL); + INSERT INTO t3 VALUES('007f2033-cb20-494c-b135-a1e4eb66130c', + 'f6d7661f-4efe-4c90-87b5-858e61cd178b'); + SELECT * + FROM t3 + INNER JOIN t1 ON t1.x= t3.v AND t1.y IS NULL + LEFT JOIN t4 ON t4.w = t3.w + LEFT JOIN t5 ON t5.z = t4.z + LEFT JOIN t2 ON t2.u = t5.m + LEFT JOIN t1 xyz ON xyz.y = t2.x; +} {007f2033-cb20-494c-b135-a1e4eb66130c f6d7661f-4efe-4c90-87b5-858e61cd178b f6d7661f-4efe-4c90-87b5-858e61cd178b {} {} {} {} {} {} {} {} {}} +do_execsql_test join5-3.3 { + DROP TABLE IF EXISTS x1; + DROP TABLE IF EXISTS x2; + DROP TABLE IF EXISTS x3; + CREATE TABLE x1(a); + INSERT INTO x1 VALUES(1); + CREATE TABLE x2(b NOT NULL); + CREATE TABLE x3(c, d); + INSERT INTO x3 VALUES('a', NULL); + INSERT INTO x3 VALUES('b', NULL); + INSERT INTO x3 VALUES('c', NULL); + SELECT * FROM x1 LEFT JOIN x2 JOIN x3 WHERE x3.d = x2.b; +} {} finish_test diff --git a/test/lock5.test b/test/lock5.test index f0d495508a..99214afb19 100644 --- a/test/lock5.test +++ b/test/lock5.test @@ -154,6 +154,7 @@ do_test lock5-flock.8 { do_test lock5-none.1 { sqlite3 db test.db -vfs unix-none sqlite3 db2 test.db -vfs unix-none + execsql { PRAGMA mmap_size = 0 } db2 execsql { BEGIN; INSERT INTO t1 VALUES(3, 4); @@ -162,8 +163,8 @@ do_test lock5-none.1 { do_test lock5-none.2 { execsql { SELECT * FROM t1 } } {1 2 3 4} -do_test lock5-flock.3 { - execsql { SELECT * FROM t1 } db2 +do_test lock5-none.3 { + execsql { SELECT * FROM t1; } db2 } {1 2} do_test lock5-none.4 { execsql { @@ -183,7 +184,7 @@ ifcapable memorymanage { } {1 2 3 4} } -do_test lock5-flock.X { +do_test lock5-none.X { db close db2 close } {} diff --git a/test/malloc.test b/test/malloc.test index 5d03aa8fe8..86145672a2 100644 --- a/test/malloc.test +++ b/test/malloc.test @@ -20,6 +20,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +set ::testprefix malloc # Only run these tests if memory debugging is turned on. @@ -880,6 +881,48 @@ do_malloc_test 39 -tclprep { db close } +reset_db +add_test_utf16bin_collate db +do_execsql_test 40.1 { + CREATE TABLE t1(a); + INSERT INTO t1 VALUES('fghij'); + INSERT INTO t1 VALUES('pqrst'); + INSERT INTO t1 VALUES('abcde'); + INSERT INTO t1 VALUES('uvwxy'); + INSERT INTO t1 VALUES('klmno'); +} +do_execsql_test 40.2 { + SELECT * FROM t1 ORDER BY 1 COLLATE utf16bin; +} {abcde fghij klmno pqrst uvwxy} +do_faultsim_test 40.3 -faults oom-trans* -body { + execsql { + SELECT * FROM t1 ORDER BY 1 COLLATE utf16bin; + } +} -test { + faultsim_test_result {0 {abcde fghij klmno pqrst uvwxy}} + faultsim_integrity_check +} + +reset_db +add_test_utf16bin_collate db +set big [string repeat x 200] +do_execsql_test 41.1 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a COLLATE utf16bin); + INSERT INTO t1 VALUES('fghij' || $::big); + INSERT INTO t1 VALUES('pqrst' || $::big); + INSERT INTO t1 VALUES('abcde' || $::big); + INSERT INTO t1 VALUES('uvwxy' || $::big); + INSERT INTO t1 VALUES('klmno' || $::big); + CREATE INDEX i1 ON t1(a); +} +do_faultsim_test 41.2 -faults oom* -body { + execsql { SELECT * FROM t1 WHERE a = ('abcde' || $::big)} +} -test { + faultsim_test_result [list 0 "abcde$::big"] + faultsim_integrity_check +} + # Ensure that no file descriptors were leaked. do_test malloc-99.X { catch {db close} diff --git a/test/mallocA.test b/test/mallocA.test index 61e88a61e7..a78073d833 100644 --- a/test/mallocA.test +++ b/test/mallocA.test @@ -25,7 +25,6 @@ if {!$MEMDEBUG} { return } - # Construct a test database # forcedelete test.db.bu @@ -116,6 +115,26 @@ ifcapable stat3 { } } +do_execsql_test 7.0 { + PRAGMA cache_size = 5; +} +do_faultsim_test 7 -faults oom-trans* -prep { +} -body { + execsql { + WITH r(x,y) AS ( + SELECT 1, randomblob(100) + UNION ALL + SELECT x+1, randomblob(100) FROM r + LIMIT 1000 + ) + SELECT count(x), length(y) FROM r GROUP BY (x%5) + } +} -test { + set res [list 200 100 200 100 200 100 200 100 200 100] + faultsim_test_result [list 0 $res] +} + + # Ensure that no file descriptors were leaked. do_test malloc-99.X { catch {db close} diff --git a/test/mallocK.test b/test/mallocK.test index 971bd56137..dcf00da9aa 100644 --- a/test/mallocK.test +++ b/test/mallocK.test @@ -16,6 +16,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/malloc_common.tcl +set testprefix mallocK set sql {SELECT * FROM t1, t2 WHERE (a=1 OR a=2)} for {set x 1} {$x<5} {incr x} { @@ -68,5 +69,70 @@ ifcapable vtab { } } +#------------------------------------------------------------------------- +# Test that OOM errors are correctly handled by the code that uses stat4 +# data to estimate the number of rows visited by a skip-scan range query. +# +add_alignment_test_collations db +do_execsql_test 6.0 { + CREATE TABLE t3(a TEXT, b TEXT COLLATE utf16_aligned, c); + INSERT INTO t3 VALUES('one', '.....', 0); + INSERT INTO t3 VALUES('one', '....x', 1); + INSERT INTO t3 VALUES('one', '...x.', 2); + INSERT INTO t3 VALUES('one', '...xx', 3); + INSERT INTO t3 VALUES('one', '..x..', 4); + INSERT INTO t3 VALUES('one', '..x.x', 5); + INSERT INTO t3 VALUES('one', '..xx.', 6); + INSERT INTO t3 VALUES('one', '..xxx', 7); + INSERT INTO t3 VALUES('one', '.x...', 8); + INSERT INTO t3 VALUES('one', '.x..x', 9); + INSERT INTO t3 VALUES('one', '.x.x.', 10); + INSERT INTO t3 VALUES('one', '.x.xx', 11); + INSERT INTO t3 VALUES('one', '.xx..', 12); + INSERT INTO t3 VALUES('one', '.xx.x', 13); + INSERT INTO t3 VALUES('one', '.xxx.', 14); + INSERT INTO t3 VALUES('one', '.xxxx', 15); + + INSERT INTO t3 VALUES('two', 'x....', 16); + INSERT INTO t3 VALUES('two', 'x...x', 17); + INSERT INTO t3 VALUES('two', 'x..x.', 18); + INSERT INTO t3 VALUES('two', 'x..xx', 19); + INSERT INTO t3 VALUES('two', 'x.x..', 20); + INSERT INTO t3 VALUES('two', 'x.x.x', 21); + INSERT INTO t3 VALUES('two', 'x.xx.', 22); + INSERT INTO t3 VALUES('two', 'x.xxx', 23); + INSERT INTO t3 VALUES('two', 'xx...', 24); + INSERT INTO t3 VALUES('two', 'xx..x', 25); + INSERT INTO t3 VALUES('two', 'xx.x.', 26); + INSERT INTO t3 VALUES('two', 'xx.xx', 27); + INSERT INTO t3 VALUES('two', 'xxx..', 28); + INSERT INTO t3 VALUES('two', 'xxx.x', 29); + INSERT INTO t3 VALUES('two', 'xxxx.', 30); + INSERT INTO t3 VALUES('two', 'xxxxx', 31); + + INSERT INTO t3 SELECT * FROM t3; + + CREATE INDEX i3 ON t3(a, b); + ANALYZE; + + SELECT 'x' > '.'; +} {1} + +ifcapable stat4 { + do_eqp_test 6.1 { + SELECT DISTINCT c FROM t3 WHERE b BETWEEN '.xx..' AND '.xxxx'; + } { + 0 0 0 {SEARCH TABLE t3 USING INDEX i3 (ANY(a) AND b>? AND b=$max_pagecache*0.95 && $overflow<=$max_pagecache*1.05} +} 1 do_test memsubsys1-3.1.5 { set s_used [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_USED 0] 2] } 0 @@ -170,20 +176,20 @@ do_test memsubsys1-4.6 { set s_used [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_USED 0] 2] } 1 -# Test 5: Activate both PAGECACHE and SCRATCH. But make the page size +# Test 5: Activate both PAGECACHE and SCRATCH. But make the page size is # such that the SCRATCH allocations are too small. # db close sqlite3_shutdown sqlite3_config_pagecache [expr 4096+$xtra_size] 24 -sqlite3_config_scratch 6000 2 +sqlite3_config_scratch 4000 2 sqlite3_initialize reset_highwater_marks build_test_db memsubsys1-5 {PRAGMA page_size=4096} #show_memstats do_test memsubsys1-5.3 { set pg_used [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0] 2] -} 24 +} {/^2[34]$/} do_test memsubsys1-5.4 { set maxreq [lindex [sqlite3_status SQLITE_STATUS_MALLOC_SIZE 0] 2] expr {$maxreq>4096} @@ -209,7 +215,7 @@ build_test_db memsubsys1-6 {PRAGMA page_size=4096} #show_memstats do_test memsubsys1-6.3 { set pg_used [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0] 2] -} 24 +} {/^2[34]$/} #do_test memsubsys1-6.4 { # set maxreq [lindex [sqlite3_status SQLITE_STATUS_MALLOC_SIZE 0] 2] # expr {$maxreq>4096 && $maxreq<=(4096+$xtra_size)} diff --git a/test/minmax4.test b/test/minmax4.test index 0d8305b5ff..8063538bfd 100644 --- a/test/minmax4.test +++ b/test/minmax4.test @@ -56,14 +56,16 @@ do_test minmax4-1.5 { do_test minmax4-1.6 { db eval { SELECT p, min(q) FROM t1; + SELECT p FROM (SELECT p, min(q) FROM t1); } -} {1 2} +} {1 2 1} do_test minmax4-1.7 { db eval { INSERT INTO t1 VALUES(5,0); SELECT p, max(q) FROM t1; + SELECT p FROM (SELECT max(q), p FROM t1); } -} {3 4} +} {3 4 3} do_test minmax4-1.8 { db eval { SELECT p, min(q) FROM t1; @@ -73,8 +75,9 @@ do_test minmax4-1.9 { db eval { INSERT INTO t1 VALUES(6,1); SELECT p, max(q) FROM t1; + SELECT p FROM (SELECT max(q), p FROM t1); } -} {3 4} +} {3 4 3} do_test minmax4-1.10 { db eval { SELECT p, min(q) FROM t1; diff --git a/test/misc8.test b/test/misc8.test new file mode 100644 index 0000000000..3ff52e56f1 --- /dev/null +++ b/test/misc8.test @@ -0,0 +1,98 @@ +# 2014-11-10 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# The focus of this script is testing the "eval.c" loadable extension. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +load_static_extension db eval +do_execsql_test misc8-1.0 { + CREATE TABLE t1(a,b,c); + INSERT INTO t1 VALUES(1,2,3),(4,5,6); + SELECT quote(eval('SELECT * FROM t1 ORDER BY a','-abc-')); +} {'1-abc-2-abc-3-abc-4-abc-5-abc-6'} +do_execsql_test misc8-1.1 { + SELECT quote(eval('SELECT * FROM t1 ORDER BY a')); +} {{'1 2 3 4 5 6'}} +do_catchsql_test misc8-1.2 { + SELECT quote(eval('SELECT d FROM t1 ORDER BY a')); +} {1 {no such column: d}} +do_execsql_test misc8-1.3 { + INSERT INTO t1 VALUES(7,null,9); + SELECT eval('SELECT * FROM t1 ORDER BY a',','); +} {1,2,3,4,5,6,7,,9} +do_catchsql_test misc8-1.4 { + BEGIN; + INSERT INTO t1 VALUES(10,11,12); + SELECT a, coalesce(b, eval('ROLLBACK; SELECT ''bam'';')), c + FROM t1 ORDER BY a; +} {0 {1 2 3 4 5 6 7 bam 9}} +do_catchsql_test misc8-1.5 { + INSERT INTO t1 VALUES(10,11,12); + SELECT a, coalesce(b, eval('SELECT ''bam''')), c + FROM t1 + ORDER BY rowid; +} {0 {1 2 3 4 5 6 7 bam 9 10 11 12}} +do_catchsql_test misc8-1.6 { + SELECT a, coalesce(b, eval('DELETE FROM t1; SELECT ''bam''')), c + FROM t1 + ORDER BY rowid; +} {0 {1 2 3 4 5 6 7 bam {}}} +do_catchsql_test misc8-1.7 { + INSERT INTO t1 VALUES(1,2,3),(4,5,6),(7,null,9); + BEGIN; + CREATE TABLE t2(x); + SELECT a, coalesce(b, eval('ROLLBACK; SELECT ''bam''')), c + FROM t1 + ORDER BY rowid; +} {1 {abort due to ROLLBACK}} + + +reset_db + +proc dbeval {sql} { db eval $sql } +db func eval dbeval + +do_execsql_test misc8-2.1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b INTEGER) WITHOUT ROWID; + CREATE TABLE t2(c INTEGER PRIMARY KEY, d INTEGER, x BLOB); + INSERT INTO t1 VALUES(0,0); + INSERT INTO t1 VALUES(10,10); + INSERT INTO t2 VALUES(1,1,zeroblob(200)); + INSERT INTO t2 VALUES(2,2,zeroblob(200)); + INSERT INTO t2 VALUES(3,3,zeroblob(200)); + INSERT INTO t2 VALUES(4,4,zeroblob(200)); + INSERT INTO t2 VALUES(5,5,zeroblob(200)); + INSERT INTO t2 VALUES(6,6,zeroblob(200)); + INSERT INTO t2 VALUES(7,7,zeroblob(200)); + INSERT INTO t2 VALUES(8,8,zeroblob(200)); + INSERT INTO t2 VALUES(9,9,zeroblob(200)); + INSERT INTO t2 VALUES(10,10,zeroblob(200)); + SELECT a, c, eval( + printf('DELETE FROM t2 WHERE c=%d AND %d>5', a+c, a+c) + ) FROM t1, t2; +} { + 0 1 {} 10 1 {} + 0 2 {} 10 2 {} + 0 3 {} 10 3 {} + 0 4 {} 10 4 {} + 0 5 {} 10 5 {} + 0 6 {} 10 {} {} + 0 7 {} 10 {} {} + 0 8 {} 10 {} {} + 0 9 {} 10 {} {} + 0 10 {} 10 {} {} +} + + +finish_test diff --git a/test/mmap1.test b/test/mmap1.test index ece3e0201e..18aec9f8fa 100644 --- a/test/mmap1.test +++ b/test/mmap1.test @@ -33,7 +33,7 @@ proc register_rblob_code {dbname seed} { set ::rcnt $seed proc rblob {n} { set ::rcnt [expr (([set ::rcnt] << 3) + [set ::rcnt] + 456) & 0xFFFFFFFF] - set str [format %.8x [expr [set ::rcnt] ^ 0xbdf20da3]] + set str [format %.8x [expr [set ::rcnt] ^ 0xbdf20da3]] string range [string repeat [set str] [expr [set n]/4]] 1 [set n] } $dbname func rblob rblob @@ -42,7 +42,7 @@ proc register_rblob_code {dbname seed} { # For cases 1.1 and 1.4, the number of pages read using xRead() is 4 on # unix and 9 on windows. The difference is that windows only ever maps -# an integer number of OS pages (i.e. creates mappings that are a multiple +# an integer number of OS pages (i.e. creates mappings that are a multiple # of 4KB in size). Whereas on unix any sized mapping may be created. # foreach {t mmap_size nRead c2init} { @@ -106,50 +106,52 @@ foreach {t mmap_size nRead c2init} { set ::rcnt 0 proc rblob {n} { set ::rcnt [expr (($::rcnt << 3) + $::rcnt + 456) & 0xFFFFFFFF] - set str [format %.8x [expr $::rcnt ^ 0xbdf20da3]] + set str [format %.8x [expr $::rcnt ^ 0xbdf20da3]] string range [string repeat $str [expr $n/4]] 1 $n } reset_db db func rblob rblob -do_execsql_test 2.1 { - PRAGMA auto_vacuum = 1; - PRAGMA mmap_size = 67108864; - PRAGMA journal_mode = wal; - CREATE TABLE t1(a, b, UNIQUE(a, b)); - INSERT INTO t1 VALUES(rblob(500), rblob(500)); - INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 2 - INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 4 - INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 8 - INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 16 - INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 32 - PRAGMA wal_checkpoint; -} {67108864 wal 0 103 103} - -do_execsql_test 2.2 { - PRAGMA auto_vacuum; - SELECT count(*) FROM t1; -} {1 32} - -if {[permutation] != "inmemory_journal"} { - do_test 2.3 { - sqlite3 db2 test.db - db2 func rblob rblob - db2 eval { - DELETE FROM t1 WHERE (rowid%4); - PRAGMA wal_checkpoint; - } - db2 eval { - INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 16 - SELECT count(*) FROM t1; - } - } {16} - - do_execsql_test 2.4 { +ifcapable wal { + do_execsql_test 2.1 { + PRAGMA auto_vacuum = 1; + PRAGMA mmap_size = 67108864; + PRAGMA journal_mode = wal; + CREATE TABLE t1(a, b, UNIQUE(a, b)); + INSERT INTO t1 VALUES(rblob(500), rblob(500)); + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 2 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 4 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 8 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 16 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 32 PRAGMA wal_checkpoint; - } {0 24 24} - db2 close + } {67108864 wal 0 103 103} + + do_execsql_test 2.2 { + PRAGMA auto_vacuum; + SELECT count(*) FROM t1; + } {1 32} + + if {[permutation] != "inmemory_journal"} { + do_test 2.3 { + sqlite3 db2 test.db + db2 func rblob rblob + db2 eval { + DELETE FROM t1 WHERE (rowid%4); + PRAGMA wal_checkpoint; + } + db2 eval { + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 16 + SELECT count(*) FROM t1; + } + } {16} + + do_execsql_test 2.4 { + PRAGMA wal_checkpoint; + } {0 24 24} + db2 close + } } reset_db @@ -227,7 +229,7 @@ do_test 4.4 { do_execsql_test 4.5 { COMMIT } #------------------------------------------------------------------------- -# Ensure that existing cursors holding xFetch() references are not +# Ensure that existing cursors holding xFetch() references are not # confused if those pages are moved to make way for the root page of a # new table or index. # @@ -296,7 +298,7 @@ foreach {tn1 mmap1 mmap2} { sql1 "PRAGMA mmap_size = $mmap1" sql2 "PRAGMA mmap_size = $mmap2" - do_test $tn1.$tn { + do_test $tn1.$tn { for {set i 1} {$i <= 100} {incr i} { if {$i % 2} { set c1 sql1 @@ -311,7 +313,7 @@ foreach {tn1 mmap1 mmap2} { UPDATE t2 SET x = (SELECT md5sum(a) FROM t1); } - set res [$c2 { + set res [$c2 { SELECT count(*) FROM t1; SELECT x == (SELECT md5sum(a) FROM t1) FROM t2; PRAGMA integrity_check; diff --git a/test/multiplex.test b/test/multiplex.test index 32c87d9a52..5db56f264a 100644 --- a/test/multiplex.test +++ b/test/multiplex.test @@ -68,6 +68,12 @@ proc multiplex_delete {name} { } db close +sqlite3_shutdown +test_sqlite3_log xLog +proc xLog {error_code msg} { + lappend ::log $error_code $msg +} +unset -nocomplain log multiplex_delete test.db multiplex_delete test2.db @@ -188,12 +194,16 @@ do_test multiplex-2.3.1 { } {} +unset -nocomplain ::log do_test multiplex-2.4.1 { sqlite3_multiplex_shutdown } {SQLITE_MISUSE} do_test multiplex-2.4.2 { execsql { INSERT INTO t1 VALUES(3, randomblob(1100)) } } {} +do_test multiplex-2.4.3 { + set ::log +} {SQLITE_MISUSE {sqlite3_multiplex_shutdown() called while database connections are still open}} do_test multiplex-2.4.4 { file size [multiplex_name test.x 0] } {7168} do_test multiplex-2.4.5 { db close @@ -583,5 +593,9 @@ do_test multiplex-6.99 { } +catch { db close } catch { sqlite3_multiplex_shutdown } +sqlite3_shutdown +test_sqlite3_log +sqlite3_initialize finish_test diff --git a/test/multiplex4.test b/test/multiplex4.test new file mode 100644 index 0000000000..9c304c314d --- /dev/null +++ b/test/multiplex4.test @@ -0,0 +1,114 @@ +# 2014-09-25 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for the "truncate" option in the multiplexor. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix multiplex4 + +db close +sqlite3_shutdown +sqlite3_multiplex_initialize {} 0 + +# delete all filesl with the base name of $basename +# +proc multiplex_delete_db {basename} { + foreach file [glob -nocomplain $basename.*] { + forcedelete $file + } +} + +# Return a sorted list of all files with the base name of $basename. +# Except, delete all text from the end of $basename through the NNN +# suffix on the end of the filename. +# +proc multiplex_file_list {basename} { + set x {} + foreach file [glob -nocomplain $basename.*] { + regsub "^$basename\\..*(\\d\\d\\d)\$" $file $basename.\\1 file + lappend x $file + } + return [lsort $x] +} + +do_test multiplex4-1.0 { + multiplex_delete_db mx4test + sqlite3 db {file:mx4test.db?chunksize=10&truncate=1} -uri 1 -vfs multiplex + db eval { + CREATE TABLE t1(x); + INSERT INTO t1(x) VALUES(randomblob(250000)); + } + multiplex_file_list mx4test +} {mx4test.001 mx4test.db} + +do_test multiplex4-1.1 { + db eval { + DELETE FROM t1; + VACUUM; + } + multiplex_file_list mx4test +} {mx4test.db} + +do_test multiplex4-1.2 { + db eval {PRAGMA multiplex_truncate} +} {on} +do_test multiplex4-1.3 { + db eval {PRAGMA multiplex_truncate=off} +} {off} +do_test multiplex4-1.4 { + db eval {PRAGMA multiplex_truncate} +} {off} +do_test multiplex4-1.5 { + db eval {PRAGMA multiplex_truncate=on} +} {on} +do_test multiplex4-1.6 { + db eval {PRAGMA multiplex_truncate} +} {on} +do_test multiplex4-1.7 { + db eval {PRAGMA multiplex_truncate=0} +} {off} +do_test multiplex4-1.8 { + db eval {PRAGMA multiplex_truncate=1} +} {on} +do_test multiplex4-1.9 { + db eval {PRAGMA multiplex_truncate=0} +} {off} + +do_test multiplex4-1.10 { + db eval { + INSERT INTO t1(x) VALUES(randomblob(250000)); + } + multiplex_file_list mx4test +} {mx4test.001 mx4test.db} + +do_test multiplex4-1.11 { + db eval { + DELETE FROM t1; + VACUUM; + } + multiplex_file_list mx4test +} {mx4test.001 mx4test.db} + +do_test multiplex4-1.12 { + db eval { + PRAGMA multiplex_truncate=ON; + DROP TABLE t1; + VACUUM; + } + multiplex_file_list mx4test +} {mx4test.db} + +catch { db close } +forcedelete mx4test.db +sqlite3_multiplex_shutdown +finish_test diff --git a/test/notify2.test b/test/notify2.test index 9e40ed695b..12c6a537ef 100644 --- a/test/notify2.test +++ b/test/notify2.test @@ -101,6 +101,8 @@ set sql $zSql # This loop runs for ~20 seconds. # set iStart [clock_seconds] + set nOp 0 + set nAttempt 0 while { ([clock_seconds]-$iStart) < $nSecond } { # Each transaction does 3 operations. Each operation is either a read @@ -128,6 +130,7 @@ set sql $zSql # Execute the SQL transaction. # + incr nAttempt set rc [catch { execsql_blocking $::DB " BEGIN; $SQL(1); @@ -154,13 +157,14 @@ set sql $zSql # returned "1". Otherwise, the invariant was false, indicating that # some malfunction has occurred. foreach r $msg { if {$r != 1} { puts "Invariant check failed: $msg" } } + incr nOp } } # Close the database connection and return 0. # sqlite3_close $::DB - expr 0 + list $nOp $nAttempt } foreach {iTest xStep xPrepare} { @@ -204,7 +208,9 @@ foreach {iTest xStep xPrepare} { for {set ii 0} {$ii < $nThread} {incr ii} { do_test notify2-$iTest.2.$ii { if {![info exists finished($ii)]} { vwait finished($ii) } - set finished($ii) + incr anSuccess($xStep) [lindex $finished($ii) 0] + incr anAttempt($xStep) [lindex $finished($ii) 1] + expr 0 } {0} } @@ -225,17 +231,36 @@ foreach {iTest xStep xPrepare} { } # The following tests checks to make sure sqlite3_blocking_step() is -# faster than sqlite3_step(). blocking_step() is always faster on -# multi-core and is usually faster on single-core. But sometimes, by -# chance, step() will be faster on a single core, in which case the +# faster than sqlite3_step(). "Faster" in this case means uses fewer +# CPU cycles. This is not always the same as faster in wall-clock time +# for this type of test. The number of CPU cycles per transaction is +# roughly proportional to the number of attempts made (i.e. one plus the +# number of SQLITE_BUSY or SQLITE_LOCKED errors that require the transaction +# to be retried). So this test just measures that a greater percentage of +# transactions attempted using blocking_step() succeed. +# +# The blocking_step() function is almost always faster on multi-core and is +# usually faster on single-core. But sometimes, by chance, step() will be +# faster on a single core, in which case the # following test will fail. # puts "The following test seeks to demonstrate that the sqlite3_unlock_notify()" -puts "interface helps multi-core systems to run faster. This test sometimes" -puts "fails on single-core machines." +puts "interface helps multi-core systems to run more efficiently. This test" +puts "sometimes fails on single-core machines." puts [array get anWrite] do_test notify2-3 { - expr {$anWrite(sqlite3_blocking_step) > $anWrite(sqlite3_step)} + set blocking [expr { + double($anSuccess(sqlite3_blocking_step)) / + double($anAttempt(sqlite3_blocking_step)) + }] + set non [expr { + double($anSuccess(sqlite3_step)) / + double($anAttempt(sqlite3_step)) + }] + puts -nonewline [format " blocking: %.1f%% non-blocking %.1f%% ..." \ + [expr $blocking*100.0] [expr $non*100.0]] + + expr {$blocking > $non} } {1} sqlite3_enable_shared_cache $::enable_shared_cache diff --git a/test/orderby1.test b/test/orderby1.test index e06c9f19a0..6674e32220 100644 --- a/test/orderby1.test +++ b/test/orderby1.test @@ -481,5 +481,19 @@ do_execsql_test 6.0 { FROM abc; } {hardware hardware hardware} +# Here is a test for a query-planner problem reported on the SQLite +# mailing list on 2014-09-18 by "Merike". Beginning with version 3.8.0, +# a separate sort was being used rather than using the single-column +# index. This was due to an oversight in the indexMightHelpWithOrderby() +# routine in where.c. +# +do_execsql_test 7.0 { + CREATE TABLE t7(a,b); + CREATE INDEX t7a ON t7(a); + CREATE INDEX t7ab ON t7(a,b); + EXPLAIN QUERY PLAN + SELECT * FROM t7 WHERE a=?1 ORDER BY rowid; +} {~/ORDER BY/} + finish_test diff --git a/test/ovfl.test b/test/ovfl.test new file mode 100644 index 0000000000..075b1e43dd --- /dev/null +++ b/test/ovfl.test @@ -0,0 +1,49 @@ +# 2014 October 01 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the SQLITE_DIRECT_OVERFLOW_READ logic. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix ovfl + +# Populate table t2: +# +# CREATE TABLE t1(c1 TEXT, c2 TEXT); +# +# with 2000 rows. In each row, c2 spans multiple overflow pages. The text +# value of c1 ranges in size from 1 to 2000 bytes. The idea is to create +# at least one row where the first byte of c2 is also the first byte of +# an overflow page. This was at one point exposing an obscure bug in the +# SQLITE_DIRECT_OVERFLOW_READ logic. +# +do_test 1.1 { + set c2 [string repeat abcdefghij 200] + execsql { + PRAGMA cache_size = 10; + CREATE TABLE t1(c1 TEXT, c2 TEXT); + BEGIN; + } + for {set i 1} {$i <= 2000} {incr i} { + set c1 [string repeat . $i] + execsql { INSERT INTO t1 VALUES($c1, $c2) } + } + execsql COMMIT +} {} + +do_execsql_test 1.2 { + SELECT sum(length(c2)) FROM t1; +} [expr 2000 * 2000] + +finish_test + + diff --git a/test/percentile.test b/test/percentile.test index 9d471dfcde..b2bd061e86 100644 --- a/test/percentile.test +++ b/test/percentile.test @@ -200,7 +200,7 @@ ifcapable vtab { } { do_test percentile-2.1.$in { execsql { - SELECT percentile(x, $in) from t3; + SELECT round(percentile(x, $in),1) from t3; } } $out } diff --git a/test/permutations.test b/test/permutations.test index c5b83cb821..9f4f35aed8 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -112,7 +112,8 @@ set allquicktests [test_set $alltests -exclude { incrvacuum_ioerr.test autovacuum_crash.test btree8.test shared_err.test vtab_err.test walslow.test walcrash.test walcrash3.test walthread.test rtree3.test indexfault.test securedel2.test - fts4growth.test fts4growth2.test + sort3.test sort4.test fts4growth.test fts4growth2.test + bigsort.test }] if {[info exists ::env(QUICKTEST_INCLUDE)]} { set allquicktests [concat $allquicktests $::env(QUICKTEST_INCLUDE)] @@ -140,7 +141,7 @@ test_suite "veryquick" -prefix "" -description { ] test_suite "mmap" -prefix "mm-" -description { - Similar to veryquick. Except with memory mapping disabled. + Similar to veryquick. Except with memory mapping enabled. } -presql { pragma mmap_size = 268435456; } -files [ @@ -359,6 +360,12 @@ test_suite "coverage-analyze" -description { analyze.test analyzeB.test mallocA.test } +test_suite "coverage-sorter" -description { + Coverage tests for file vdbesort.c. +} -files { + sort.test sortfault.test +} + lappend ::testsuitelist xxx #------------------------------------------------------------------------- @@ -490,7 +497,7 @@ test_suite "multithread" -description { } -files { delete.test delete2.test insert.test rollback.test select1.test select2.test trans.test update.test vacuum.test types.test - types2.test types3.test + types2.test types3.test sort4.test } -shutdown { catch {db close} sqlite3_shutdown @@ -687,7 +694,7 @@ test_suite "inmemory_journal" -description { zerodamage.test # WAL mode is different. - wal* tkt-2d1a5c67d.test backcompat.test + wal* tkt-2d1a5c67d.test backcompat.test e_wal* }] ifcapable mem3 { diff --git a/test/pragma.test b/test/pragma.test index 8f54e695d7..09b9b6c14f 100644 --- a/test/pragma.test +++ b/test/pragma.test @@ -431,7 +431,56 @@ Page 6 is never used} {row 1 missing from index i2}} db eval {PRAGMA integrity_check} } {ok} } -#exit + +# Verify that PRAGMA integrity_check catches UNIQUE and NOT NULL +# constraint violations. +# +do_execsql_test pragma-3.20 { + CREATE TABLE t1(a,b); + CREATE INDEX t1a ON t1(a); + INSERT INTO t1 VALUES(1,1),(2,2),(3,3),(2,4),(NULL,5),(NULL,6); + PRAGMA writable_schema=ON; + UPDATE sqlite_master SET sql='CREATE UNIQUE INDEX t1a ON t1(a)' + WHERE name='t1a'; + UPDATE sqlite_master SET sql='CREATE TABLE t1(a NOT NULL,b)' + WHERE name='t1'; + PRAGMA writable_schema=OFF; + ALTER TABLE t1 RENAME TO t1x; + PRAGMA integrity_check; +} {{non-unique entry in index t1a} {NULL value in t1x.a} {non-unique entry in index t1a} {NULL value in t1x.a}} +do_execsql_test pragma-3.21 { + PRAGMA integrity_check(3); +} {{non-unique entry in index t1a} {NULL value in t1x.a} {non-unique entry in index t1a}} +do_execsql_test pragma-3.22 { + PRAGMA integrity_check(2); +} {{non-unique entry in index t1a} {NULL value in t1x.a}} +do_execsql_test pragma-3.23 { + PRAGMA integrity_check(1); +} {{non-unique entry in index t1a}} + +# PRAGMA integrity check (or more specifically the sqlite3BtreeCount() +# interface) used to leave index cursors in an inconsistent state +# which could result in an assertion fault in sqlite3BtreeKey() +# called from saveCursorPosition() if content is removed from the +# index while the integrity_check is still running. This test verifies +# that problem has been fixed. +# +do_test pragma-3.30 { + db close + delete_file test.db + sqlite3 db test.db + db eval { + CREATE TABLE t1(a,b,c); + WITH RECURSIVE + c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<100) + INSERT INTO t1(a,b,c) SELECT i, printf('xyz%08x',i), 2000-i FROM c; + CREATE INDEX t1a ON t1(a); + CREATE INDEX t1bc ON t1(b,c); + } + db eval {PRAGMA integrity_check} { + db eval {DELETE FROM t1} + } +} {} # Test modifying the cache_size of an attached database. ifcapable pager_pragmas&&attach { @@ -1165,13 +1214,13 @@ ifcapable schema_pragmas { execsql2 { pragma collation_list; } - } {seq 0 name NOCASE seq 1 name RTRIM seq 2 name BINARY} + } {seq 0 name RTRIM seq 1 name NOCASE seq 2 name BINARY} do_test pragma-11.2 { db collate New_Collation blah... execsql { pragma collation_list; } - } {0 New_Collation 1 NOCASE 2 RTRIM 3 BINARY} + } {0 New_Collation 1 RTRIM 2 NOCASE 3 BINARY} } ifcapable schema_pragmas&&tempdb { diff --git a/test/pragma3.test b/test/pragma3.test new file mode 100644 index 0000000000..b7ea4d3fc6 --- /dev/null +++ b/test/pragma3.test @@ -0,0 +1,253 @@ +# 2014-12-19 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# +# This file implements tests for PRAGMA data_version command. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +do_execsql_test pragma3-100 { + PRAGMA data_version; +} {1} +do_execsql_test pragma3-101 { + PRAGMA temp.data_version; +} {1} + +# Writing to the pragma is a no-op +do_execsql_test pragma3-102 { + PRAGMA main.data_version=1234; + PRAGMA main.data_version; +} {1 1} + +# EVIDENCE-OF: R-27726-60934 The "PRAGMA data_version" command provides +# an indication that the database file has been modified. +# +# EVIDENCE-OF: R-47505-58569 The "PRAGMA data_version" value is +# unchanged for commits made on the same database connection. +# +do_execsql_test pragma3-110 { + PRAGMA data_version; + BEGIN IMMEDIATE; + PRAGMA data_version; + CREATE TABLE t1(a); + INSERT INTO t1 VALUES(100),(200),(300); + PRAGMA data_version; + COMMIT; + SELECT * FROM t1; + PRAGMA data_version; +} {1 1 1 100 200 300 1} + +sqlite3 db2 test.db +do_test pragma3-120 { + db2 eval { + SELECT * FROM t1; + PRAGMA data_version; + } +} {100 200 300 1} + +do_execsql_test pragma3-130 { + PRAGMA data_version; + BEGIN IMMEDIATE; + PRAGMA data_version; + INSERT INTO t1 VALUES(400),(500); + PRAGMA data_version; + COMMIT; + SELECT * FROM t1; + PRAGMA data_version; + PRAGMA shrink_memory; +} {1 1 1 100 200 300 400 500 1} + +# EVIDENCE-OF: R-63005-41812 The integer values returned by two +# invocations of "PRAGMA data_version" from the same connection will be +# different if changes were committed to the database by any other +# connection in the interim. +# +# Value went from 1 in pragma3-120 to 2 here. +# +do_test pragma3-140 { + db2 eval { + SELECT * FROM t1; + PRAGMA data_version; + BEGIN IMMEDIATE; + PRAGMA data_version; + UPDATE t1 SET a=a+1; + COMMIT; + SELECT * FROM t1; + PRAGMA data_version; + } +} {100 200 300 400 500 2 2 101 201 301 401 501 2} +do_execsql_test pragma3-150 { + SELECT * FROM t1; + PRAGMA data_version; +} {101 201 301 401 501 2} + +# +do_test pragma3-160 { + db eval { + BEGIN; + PRAGMA data_version; + UPDATE t1 SET a=555 WHERE a=501; + PRAGMA data_version; + SELECT * FROM t1 ORDER BY a; + PRAGMA data_version; + } +} {2 2 101 201 301 401 555 2} +do_test pragma3-170 { + db2 eval { + PRAGMA data_version; + } +} {2} +do_test pragma3-180 { + db eval { + COMMIT; + PRAGMA data_version; + } +} {2} +do_test pragma3-190 { + db2 eval { + PRAGMA data_version; + } +} {3} + +# EVIDENCE-OF: R-19326-44825 The "PRAGMA data_version" value is a local +# property of each database connection and so values returned by two +# concurrent invocations of "PRAGMA data_version" on separate database +# connections are often different even though the underlying database is +# identical. +# +do_test pragma3-195 { + expr {[db eval {PRAGMA data_version}]!=[db2 eval {PRAGMA data_version}]} +} {1} + +# EVIDENCE-OF: R-54562-06892 The behavior of "PRAGMA data_version" is +# the same for all database connections, including database connections +# in separate processes and shared cache database connections. +# +# The next block checks the behavior for separate processes. +# +do_test pragma3-200 { + db eval {PRAGMA data_version; SELECT * FROM t1;} +} {2 101 201 301 401 555} +do_test pragma3-201 { + set fd [open pragma3.txt wb] + puts $fd { + sqlite3 db test.db; + db eval {DELETE FROM t1 WHERE a>300}; + db close; + exit; + } + close $fd + exec [info nameofexec] pragma3.txt + forcedelete pragma3.txt + db eval { + PRAGMA data_version; + SELECT * FROM t1; + } +} {3 101 201} +db2 close +db close + +# EVIDENCE-OF: R-54562-06892 The behavior of "PRAGMA data_version" is +# the same for all database connections, including database connections +# in separate processes and shared cache database connections. +# +# The next block checks that behavior is the same for shared-cache. +# +ifcapable shared_cache { + set ::enable_shared_cache [sqlite3_enable_shared_cache 1] + sqlite3 db test.db + sqlite3 db2 test.db + do_test pragma3-300 { + db eval { + PRAGMA data_version; + BEGIN; + CREATE TABLE t3(a,b,c); + CREATE TABLE t4(x,y,z); + INSERT INTO t4 VALUES(123,456,789); + PRAGMA data_version; + COMMIT; + PRAGMA data_version; + } + } {1 1 1} + do_test pragma3-310 { + db2 eval { + PRAGMA data_version; + BEGIN; + INSERT INTO t3(a,b,c) VALUES('abc','def','ghi'); + SELECT * FROM t3; + PRAGMA data_version; + } + } {2 abc def ghi 2} + # The transaction in db2 has not yet committed, so the data_version in + # db is unchanged. + do_test pragma3-320 { + db eval { + PRAGMA data_version; + SELECT * FROM t4; + } + } {1 123 456 789} + do_test pragma3-330 { + db2 eval { + COMMIT; + PRAGMA data_version; + SELECT * FROM t4; + } + } {2 123 456 789} + do_test pragma3-340 { + db eval { + PRAGMA data_version; + SELECT * FROM t3; + SELECT * FROM t4; + } + } {2 abc def ghi 123 456 789} + db2 close + db close + sqlite3_enable_shared_cache $::enable_shared_cache +} + +# Make sure this also works in WAL mode +# +# This will not work with the in-memory journal permutation, as opening +# [db2] switches the journal mode back to "memory" +# +ifcapable wal { +if {[permutation]!="inmemory_journal"} { + + sqlite3 db test.db + db eval {PRAGMA journal_mode=WAL} + sqlite3 db2 test.db + do_test pragma3-400 { + db eval { + PRAGMA data_version; + PRAGMA journal_mode; + SELECT * FROM t1; + } + } {2 wal 101 201} + do_test pragma3-410 { + db2 eval { + PRAGMA data_version; + PRAGMA journal_mode; + SELECT * FROM t1; + } + } {2 wal 101 201} + do_test pragma3-420 { + db eval {UPDATE t1 SET a=111*(a/100); PRAGMA data_version; SELECT * FROM t1} + } {2 111 222} + do_test pragma3-430 { + db2 eval {PRAGMA data_version; SELECT * FROM t1;} + } {3 111 222} + db2 close +} +} + +finish_test diff --git a/test/printf2.test b/test/printf2.test index 4cb1783bfb..21deeb779d 100644 --- a/test/printf2.test +++ b/test/printf2.test @@ -95,5 +95,25 @@ do_execsql_test printf2-2.3 { SELECT printf('%s=(%d/%g/%s)',a) FROM t1 ORDER BY a; } {-1=(0/0/) 1=(0/0/) 1.5=(0/0/) abc=(0/0/)} +# The precision of the %c conversion causes the character to repeat. +# +do_execsql_test printf2-3.1 { + SELECT printf('|%110.100c|','*'); +} {{| ****************************************************************************************************|}} +do_execsql_test printf2-3.2 { + SELECT printf('|%-110.100c|','*'); +} {{|**************************************************************************************************** |}} +do_execsql_test printf2-3.3 { + SELECT printf('|%9.8c|%-9.8c|','*','*'); +} {{| ********|******** |}} +do_execsql_test printf2-3.4 { + SELECT printf('|%8.8c|%-8.8c|','*','*'); +} {|********|********|} +do_execsql_test printf2-3.5 { + SELECT printf('|%7.8c|%-7.8c|','*','*'); +} {|********|********|} + + + finish_test diff --git a/test/releasetest.mk b/test/releasetest.mk deleted file mode 100644 index 5d217c645b..0000000000 --- a/test/releasetest.mk +++ /dev/null @@ -1,14 +0,0 @@ -######################################################## -TOP=/home/drh/sqlite/sqlite - -TCL_FLAGS=-I/home/drh/tcltk/86linux -LIBTCL=/home/drh/tcltk/86linux/libtcl8.6.a -lm -ldl -lpthread - -BCC = gcc -TCC = gcc -ansi -g $(CFLAGS) -NAWK = awk -AR = ar cr -RANLIB = ranlib -THREADLIB = -lpthread -ldl -lz -include $(TOP)/main.mk -######################################################## diff --git a/test/releasetest.tcl b/test/releasetest.tcl index b635061c55..96e7ffb4b2 100644 --- a/test/releasetest.tcl +++ b/test/releasetest.tcl @@ -1,59 +1,31 @@ - -set rcsid {$Id: $} - +#!/usr/bin/tclsh +# # Documentation for this script. This may be output to stderr # if the script is invoked incorrectly. See the [process_options] # proc below. # set ::USAGE_MESSAGE { This Tcl script is used to test the various configurations required -before releasing a new version. Supported command line options (all +before releasing a new version. Supported command line options (all optional) are: - -makefile PATH-TO-MAKEFILE (default "releasetest.mk") - -platform PLATFORM (see below) - -quick BOOLEAN (default "0") + --srcdir TOP-OF-SQLITE-TREE (see below) + --platform PLATFORM (see below) + --config CONFIGNAME (Run only CONFIGNAME) + --quick (Run "veryquick.test" only) + --buildonly (Just build testfixture - do not run) + --dryrun (Print what would have happened) + --info (Show diagnostic info) -The default value for -makefile is "./releasetest.mk". +The default value for --srcdir is the parent of the directory holding +this script. -The script determines the default value for -platform using the -$tcl_platform(os) and $tcl_platform(machine) variables. Supported +The script determines the default value for --platform using the +$tcl_platform(os) and $tcl_platform(machine) variables. Supported platforms are "Linux-x86", "Linux-x86_64" and "Darwin-i386". -If the -quick option is set to true, then the "veryquick.test" script -is run for all compilation configurations. Otherwise, sometimes "all.test" -is run, sometimes "veryquick.test". - -Almost any SQLite makefile (except those generated by configure - see below) -should work. The following properties are required: - - * The makefile should support the "fulltest" target. - * The makefile should support the variable "OPTS" as a way to pass - options from the make command line to lemon and the C compiler. - -More precisely, the following invocation must be supported: - - make -f $::MAKEFILE fulltest OPTS="-DSQLITE_SECURE_DELETE=1 -DSQLITE_DEBUG=1" - -Makefiles generated by the sqlite configure program cannot be used as -they do not respect the OPTS variable. - -Example Makefile contents: - - ######################################################## - TOP=/home/dan/work/sqlite/sqlite - - TCL_FLAGS=-I/home/dan/tcl/include - LIBTCL=-L/home/dan/tcl/lib -ltcl - - BCC = gcc - TCC = gcc -ansi -g $(CFLAGS) - NAWK = awk - AR = ar cr - RANLIB = ranlib - THREADLIB = -lpthread -ldl - include $(TOP)/main.mk - ######################################################## +Every test begins with a fresh run of the configure script at the top +of the SQLite source tree. } array set ::Configs { @@ -93,7 +65,6 @@ array set ::Configs { -DSQLITE_SECURE_DELETE=1 -DSQLITE_SOUNDEX=1 -DSQLITE_ENABLE_ATOMIC_WRITE=1 - -DSQLITE_ENABLE_IOTRACE=1 -DSQLITE_ENABLE_MEMORY_MANAGEMENT=1 -DSQLITE_ENABLE_OVERSIZE_CELL_CHECK=1 } @@ -160,8 +131,9 @@ array set ::Configs { -DSQLITE_DEFAULT_CACHE_SIZE=1000 -DSQLITE_MAX_LENGTH=2147483645 -DSQLITE_MAX_VARIABLE_NUMBER=500000 - -DSQLITE_DEBUG=1 + -DSQLITE_DEBUG=1 -DSQLITE_PREFER_PROXY_LOCKING=1 + -DSQLITE_ENABLE_API_ARMOR=1 } "Extra-Robustness" { -DSQLITE_ENABLE_OVERSIZE_CELL_CHECK=1 @@ -176,19 +148,27 @@ array set ::Configs { -DSQLITE_DISABLE_FTS4_DEFERRED -DSQLITE_ENABLE_RTREE } + + "No-lookaside" { + -DSQLITE_TEST_REALLOC_STRESS=1 + -DSQLITE_OMIT_LOOKASIDE=1 + -DHAVE_USLEEP=1 + } } array set ::Platforms { Linux-x86_64 { "Check-Symbols" checksymbols - "Debug-One" test + "Debug-One" "mptest test" "Secure-Delete" test "Unlock-Notify" "QUICKTEST_INCLUDE=notify2.test test" "Update-Delete-Limit" test "Extra-Robustness" test "Device-Two" test "Ftrapv" test - "Default" "threadtest test" + "No-lookaside" test + "Devkit" test + "Default" "threadtest fulltest" "Device-One" fulltest } Linux-i686 { @@ -199,9 +179,16 @@ array set ::Platforms { "Default" "threadtest fulltest" } Darwin-i386 { - "Locking-Style" test + "Locking-Style" "mptest test" "OS-X" "threadtest fulltest" } + Darwin-x86_64 { + "Locking-Style" "mptest test" + "OS-X" "threadtest fulltest" + } + "Windows NT-intel" { + "Default" "mptest fulltestonly" + } } @@ -218,14 +205,44 @@ foreach {key value} [array get ::Platforms] { } } +# Open the file $logfile and look for a report on the number of errors +# and the number of test cases run. Add these values to the global +# $::NERRCASE and $::NTESTCASE variables. +# +# If any errors occur, then write into $errmsgVar the text of an appropriate +# one-line error message to show on the output. +# +proc count_tests_and_errors {logfile rcVar errmsgVar} { + if {$::DRYRUN} return + upvar 1 $rcVar rc $errmsgVar errmsg + set fd [open $logfile rb] + set seen 0 + while {![eof $fd]} { + set line [gets $fd] + if {[regexp {(\d+) errors out of (\d+) tests} $line all nerr ntest]} { + incr ::NERRCASE $nerr + incr ::NTESTCASE $ntest + set seen 1 + if {$nerr>0} { + set rc 1 + set errmsg $line + } + } + } + close $fd + if {!$seen} { + set rc 1 + set errmsg "Test did not complete" + } +} + proc run_test_suite {name testtarget config} { - - # Tcl variable $opts is used to build up the value used to set the + # Tcl variable $opts is used to build up the value used to set the # OPTS Makefile variable. Variable $cflags holds the value for # CFLAGS. The makefile will pass OPTS to both gcc and lemon, but # CFLAGS is only passed to gcc. # - set cflags "" + set cflags "-g" set opts "" foreach arg $config { if {[string match -D* $arg]} { @@ -249,30 +266,72 @@ proc run_test_suite {name testtarget config} { append opts " -DSQLITE_OS_UNIX=1" } - # Run the test. - # - set makefile [file normalize $::MAKEFILE] - file mkdir $dir - puts -nonewline "Testing configuration \"$name\" (logfile=$dir/test.log)..." - flush stdout + dryrun file mkdir $dir + if {!$::DRYRUN} { + set title ${name}($testtarget) + set n [string length $title] + puts -nonewline "${title}[string repeat . [expr {63-$n}]]" + flush stdout + } - set makecmd [concat \ - [list exec make -C $dir -f $makefile clean] \ - $testtarget \ - [list CFLAGS=$cflags OPTS=$opts >& $dir/test.log] \ - ] - - set tm1 [clock seconds] - set rc [catch $makecmd] + set tm1 [clock seconds] + set origdir [pwd] + dryrun cd $dir + set errmsg {} + set rc [catch [configureCommand]] + if {!$rc} { + set rc [catch [makeCommand $testtarget $cflags $opts]] + count_tests_and_errors test.log rc errmsg + } set tm2 [clock seconds] + dryrun cd $origdir - set minutes [expr {($tm2-$tm1)/60}] - set seconds [expr {($tm2-$tm1)%60}] - puts -nonewline [format " (%d:%.2d) " $minutes $seconds] - if {$rc} { - puts "FAILED." + if {!$::DRYRUN} { + set hours [expr {($tm2-$tm2)/3600}] + set minutes [expr {(($tm2-$tm1)/60)%60}] + set seconds [expr {($tm2-$tm1)%60}] + set tm [format (%02d:%02d:%02d) $hours $minutes $seconds] + if {$rc} { + puts " FAIL $tm" + incr ::NERR + if {$errmsg!=""} {puts " $errmsg"} + } else { + puts " Ok $tm" + } + } +} + +# The following procedure returns the "configure" command to be exectued for +# the current platform, which may be Windows (via MinGW, etc). +# +proc configureCommand {} { + set result [list dryrun exec] + if {$::tcl_platform(platform)=="windows"} { + lappend result sh + } + lappend result $::SRCDIR/configure -enable-load-extension >& test.log +} + +# The following procedure returns the "make" command to be executed for the +# specified targets, compiler flags, and options. +# +proc makeCommand { targets cflags opts } { + set result [list dryrun exec make clean] + foreach target $targets { + lappend result $target + } + lappend result CFLAGS=$cflags OPTS=$opts >>& test.log +} + +# The following procedure either prints its arguments (if ::DRYRUN is true) +# or executes the command of its arguments in the calling context +# (if ::DRYRUN is false). +# +proc dryrun {args} { + if {$::DRYRUN} { + puts $args } else { - puts "Ok." + uplevel 1 $args } } @@ -283,15 +342,21 @@ proc run_test_suite {name testtarget config} { # option. # proc process_options {argv} { - set ::MAKEFILE releasetest.mk ;# Default value - set ::QUICK 0 ;# Default value + set ::SRCDIR [file normalize [file dirname [file dirname $::argv0]]] + set ::QUICK 0 + set ::BUILDONLY 0 + set ::DRYRUN 0 + set ::EXEC exec + set config {} set platform $::tcl_platform(os)-$::tcl_platform(machine) for {set i 0} {$i < [llength $argv]} {incr i} { - switch -- [lindex $argv $i] { - -makefile { + set x [lindex $argv $i] + if {[regexp {^--[a-z]} $x]} {set x [string range $x 1 end]} + switch -- $x { + -srcdir { incr i - set ::MAKEFILE [lindex $argv $i] + set ::SRCDIR [file normalize [lindex $argv $i]] } -platform { @@ -300,10 +365,41 @@ proc process_options {argv} { } -quick { - incr i - set ::QUICK [lindex $argv $i] + set ::QUICK 1 } - + + -config { + incr i + set config [lindex $argv $i] + } + + -buildonly { + set ::BUILDONLY 1 + } + + -dryrun { + set ::DRYRUN 1 + } + + -info { + puts "Command-line Options:" + puts " --srcdir $::SRCDIR" + puts " --platform [list $platform]" + puts " --config [list $config]" + if {$::QUICK} {puts " --quick"} + if {$::BUILDONLY} {puts " --buildonly"} + if {$::DRYRUN} {puts " --dryrun"} + puts "\nAvailable --platform options:" + foreach y [lsort [array names ::Platforms]] { + puts " [list $y]" + } + puts "\nAvailable --config options:" + foreach y [lsort [array names ::Configs]] { + puts " [list $y]" + } + exit + } + default { puts stderr "" puts stderr [string trim $::USAGE_MESSAGE] @@ -312,8 +408,6 @@ proc process_options {argv} { } } - set ::MAKEFILE [file normalize $::MAKEFILE] - if {0==[info exists ::Platforms($platform)]} { puts "Unknown platform: $platform" puts -nonewline "Set the -platform option to " @@ -326,9 +420,19 @@ proc process_options {argv} { exit } - set ::CONFIGLIST $::Platforms($platform) - puts "Running the following configurations for $platform:" + if {$config!=""} { + if {[llength $config]==1} {lappend config fulltest} + set ::CONFIGLIST $config + } else { + set ::CONFIGLIST $::Platforms($platform) + } + puts "Running the following test configurations for $platform:" puts " [string trim $::CONFIGLIST]" + puts -nonewline "Flags:" + if {$::DRYRUN} {puts -nonewline " --dryrun"} + if {$::BUILDONLY} {puts -nonewline " --buildonly"} + if {$::QUICK} {puts -nonewline " --quick"} + puts "" } # Main routine. @@ -337,30 +441,48 @@ proc main {argv} { # Process any command line options. process_options $argv + puts [string repeat * 79] + set ::NERR 0 + set ::NTEST 0 + set ::NTESTCASE 0 + set ::NERRCASE 0 + set STARTTIME [clock seconds] foreach {zConfig target} $::CONFIGLIST { if {$::QUICK} {set target test} + if {$::BUILDONLY} {set target testfixture} set config_options $::Configs($zConfig) + incr NTEST run_test_suite $zConfig $target $config_options # If the configuration included the SQLITE_DEBUG option, then remove # it and run veryquick.test. If it did not include the SQLITE_DEBUG option # add it and run veryquick.test. - if {$target!="checksymbols"} { + if {$target!="checksymbols" && !$::BUILDONLY} { set debug_idx [lsearch -glob $config_options -DSQLITE_DEBUG*] + set xtarget $target + regsub -all {fulltest[a-z]*} $xtarget test xtarget if {$debug_idx < 0} { - run_test_suite "${zConfig}_debug" test [ - concat $config_options -DSQLITE_DEBUG=1 - ] + incr NTEST + append config_options " -DSQLITE_DEBUG=1" + run_test_suite "${zConfig}_debug" $xtarget $config_options } else { - run_test_suite "${zConfig}_ndebug" test [ - lreplace $config_options $debug_idx $debug_idx - ] + incr NTEST + regsub { *-DSQLITE_MEMDEBUG[^ ]* *} $config_options { } config_options + regsub { *-DSQLITE_DEBUG[^ ]* *} $config_options { } config_options + run_test_suite "${zConfig}_ndebug" $xtarget $config_options } } - } + + set elapsetime [expr {[clock seconds]-$STARTTIME}] + set hr [expr {$elapsetime/3600}] + set min [expr {($elapsetime/60)%60}] + set sec [expr {$elapsetime%60}] + set etime [format (%02d:%02d:%02d) $hr $min $sec] + puts [string repeat * 79] + puts "$::NERRCASE failures of $::NTESTCASE tests run in $etime" } main $argv diff --git a/test/rollback.test b/test/rollback.test index c339c5d7d6..7abafece61 100644 --- a/test/rollback.test +++ b/test/rollback.test @@ -60,11 +60,11 @@ ifcapable conflict { # do_test rollback-1.5 { sqlite3_step $STMT - } {SQLITE_ERROR} + } {SQLITE_ROW} # Restart the SELECT statement # - do_test rollback-1.6 { sqlite3_reset $STMT } {SQLITE_ABORT} + do_test rollback-1.6 { sqlite3_reset $STMT } {SQLITE_OK} } else { do_test rollback-1.6 { sqlite3_reset $STMT } {SQLITE_OK} } diff --git a/test/rollback2.test b/test/rollback2.test new file mode 100644 index 0000000000..4d42dda5d6 --- /dev/null +++ b/test/rollback2.test @@ -0,0 +1,157 @@ +# 2014 November 12 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file containst tests to verify that ROLLBACK or ROLLBACK TO +# operations interact correctly with ongoing SELECT statements. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix rollback2 + +proc int2hex {i} { format %.2X $i } +db func int2hex int2hex +do_execsql_test 1.0 { + SELECT int2hex(0), int2hex(100), int2hex(255) +} {00 64 FF} +do_execsql_test 1.1 { + CREATE TABLE t1(i, h); + CREATE INDEX i1 ON t1(h); + WITH data(a, b) AS ( + SELECT 1, int2hex(1) + UNION ALL + SELECT a+1, int2hex(a+1) FROM data WHERE a<40 + ) + INSERT INTO t1 SELECT * FROM data; +} {} + + +# do_rollback_test ID SWITCHES +# +# where SWITCHES are: +# +# -setup SQL script to open transaction and begin writing. +# -select SELECT to execute after -setup script +# -result Expected result of -select statement +# -rollback Use this SQL command ("ROLLBACK" or "ROLLBACK TO ...") to +# rollback the transaction in the middle of the -select statment +# execution. +# +proc do_rollback_test {tn args} { + set A(-setup) "" + set A(-select) "" + set A(-result) "" + set A(-rollback) ROLLBACK + + array set O $args + foreach k [array names O] { + if {[info exists A($k)]==0} { error "unknown option: $k" } + set A($k) $O($k) + } + + for {set iRollback 0} 1 {incr iRollback} { + catch { db eval ROLLBACK } + set res [list] + db eval $A(-setup) + + set i 0 + db eval $A(-select) x { + if {$i==$iRollback} { db eval $A(-rollback) } + foreach k $x(*) { lappend res $x($k) } + incr i + } + + do_test $tn.$iRollback [list set {} $res] [list {*}$A(-result)] + if {$i < $iRollback} break + } +} + +do_rollback_test 2.1 -setup { + BEGIN; + DELETE FROM t1 WHERE (i%2)==1; +} -select { + SELECT i FROM t1 WHERE (i%2)==0 +} -result { + 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38 40 +} + +do_rollback_test 2.2 -setup { + BEGIN; + DELETE FROM t1 WHERE (i%4)==1; + SAVEPOINT one; + DELETE FROM t1 WHERE (i%2)==1; +} -rollback { + ROLLBACK TO one; +} -select { + SELECT i FROM t1 WHERE (i%2)==0 +} -result { + 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38 40 +} + +#-------------------------------------------------------------------- +# Try with some index scans +# +do_eqp_test 3.1 { + SELECT i FROM t1 WHERE (i%2)==0 ORDER BY h DESC; +} {0 0 0 {SCAN TABLE t1 USING INDEX i1}} +do_rollback_test 3.2 -setup { + BEGIN; + DELETE FROM t1 WHERE (i%2)==1; +} -select { + SELECT i FROM t1 WHERE (i%2)==0 ORDER BY h DESC; +} -result { + 40 38 36 34 32 30 28 26 24 22 20 18 16 14 12 10 8 6 4 2 +} +do_rollback_test 3.3 -setup { + BEGIN; + DELETE FROM t1 WHERE (i%4)==1; + SAVEPOINT one; + DELETE FROM t1 WHERE (i%2)==1; +} -rollback { + ROLLBACK TO one; +} -select { + SELECT i FROM t1 WHERE (i%2)==0 ORDER BY h DESC; +} -result { + 40 38 36 34 32 30 28 26 24 22 20 18 16 14 12 10 8 6 4 2 +} + +#-------------------------------------------------------------------- +# Now with some index scans that feature overflow keys. +# +set leader [string repeat "abcdefghij" 70] +do_execsql_test 4.1 { UPDATE t1 SET h = $leader || h; } + +do_eqp_test 4.2 { + SELECT i FROM t1 WHERE (i%2)==0 ORDER BY h ASC; +} {0 0 0 {SCAN TABLE t1 USING INDEX i1}} +do_rollback_test 4.3 -setup { + BEGIN; + DELETE FROM t1 WHERE (i%2)==1; +} -select { + SELECT i FROM t1 WHERE (i%2)==0 ORDER BY h ASC; +} -result { + 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38 40 +} +do_rollback_test 4.4 -setup { + BEGIN; + DELETE FROM t1 WHERE (i%4)==1; + SAVEPOINT one; + DELETE FROM t1 WHERE (i%2)==1; +} -rollback { + ROLLBACK TO one; +} -select { + SELECT i FROM t1 WHERE (i%2)==0 ORDER BY h ASC; +} -result { + 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38 40 +} + +finish_test + diff --git a/test/rollbackfault.test b/test/rollbackfault.test new file mode 100644 index 0000000000..f248d0758d --- /dev/null +++ b/test/rollbackfault.test @@ -0,0 +1,84 @@ +# 2014-11-12 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test that errors encountered during a ROLLBACK operation correctly +# affect ongoing SQL statements. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/malloc_common.tcl +set testprefix rollbackfault + + +proc int2hex {i} { format %.2X $i } +db func int2hex int2hex +do_execsql_test 1.0 { + SELECT int2hex(0), int2hex(100), int2hex(255) +} {00 64 FF} +do_execsql_test 1.1 { + CREATE TABLE t1(i, h); + CREATE INDEX i1 ON t1(h); + WITH data(a, b) AS ( + SELECT 1, int2hex(1) + UNION ALL + SELECT a+1, int2hex(a+1) FROM data WHERE a<40 + ) + INSERT INTO t1 SELECT * FROM data; +} {} + +foreach f {oom ioerr} { + do_faultsim_test 1.2 -faults $f* -prep { + set sql1 { SELECT i FROM t1 WHERE (i%2)==0 } + set sql2 { SELECT i FROM t1 WHERE (i%2)==0 ORDER BY h } + set ::s1 [sqlite3_prepare db $sql1 -1 dummy] + set ::s2 [sqlite3_prepare db $sql2 -1 dummy] + + for {set i 0} {$i < 10} {incr i} { sqlite3_step $::s1 } + for {set i 0} {$i < 3} {incr i} { sqlite3_step $::s2 } + + execsql { + BEGIN; DELETE FROM t1 WHERE (i%2) + } + } -body { + execsql { ROLLBACK } + } -test { + + set res1 [list] + set res2 [list] + while {"SQLITE_ROW" == [sqlite3_step $::s1]} { + lappend res1 [sqlite3_column_text $::s1 0] + } + while {"SQLITE_ROW" == [sqlite3_step $::s2]} { + lappend res2 [sqlite3_column_text $::s2 0] + } + set rc1 [sqlite3_finalize $::s1] + set rc2 [sqlite3_finalize $::s2] + + catchsql { ROLLBACK } + + if {$rc1=="SQLITE_OK" && $rc2=="SQLITE_OK" + && $res1=="22 24 26 28 30 32 34 36 38 40" + && $res2=="8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38 40" + } { + # This is Ok. + } elseif {$rc1!="SQLITE_OK" && $rc2!="SQLITE_OK" && $res1=="" &&$res2==""} { + # Also Ok. + } else { + error "statements don't look right" + } + } +} + + +finish_test + + diff --git a/test/rowid.test b/test/rowid.test index 6d068d79bb..b00b5287fd 100644 --- a/test/rowid.test +++ b/test/rowid.test @@ -679,9 +679,9 @@ do_test rowid-12.2 { save_prng_state execsql { INSERT INTO t7 VALUES(NULL,'b'); - SELECT x, y FROM t7; + SELECT x, y FROM t7 ORDER BY x; } -} {1 b 9223372036854775807 a} +} {/\d+ b 9223372036854775807 a/} execsql {INSERT INTO t7 VALUES(2,'y');} for {set i 1} {$i<100} {incr i} { do_test rowid-12.3.$i { @@ -701,5 +701,19 @@ do_test rowid-12.4 { } } {1 {database or disk is full}} +# INSERTs that happen inside of nested function calls are recorded +# by last_insert_rowid. +# +proc rowid_addrow_func {n} { + db eval {INSERT INTO t13(rowid,x) VALUES($n,$n*$n)} + return [db last_insert_rowid] +} +db function addrow rowid_addrow_func +do_execsql_test rowid-13.1 { + CREATE TABLE t13(x); + INSERT INTO t13(rowid,x) VALUES(1234,5); + SELECT rowid, x, addrow(rowid+1000), '|' FROM t13 LIMIT 3; + SELECT last_insert_rowid(); +} {1234 5 2234 | 2234 4990756 3234 | 3234 10458756 4234 | 4234} finish_test diff --git a/test/savepoint.test b/test/savepoint.test index 9f4571abef..8055e61d9e 100644 --- a/test/savepoint.test +++ b/test/savepoint.test @@ -315,7 +315,7 @@ ifcapable incrblob { do_test savepoint-5.3.2.3 { set rc [catch {seek $fd 0; read $fd} res] set rc - } {1} + } {0} do_test savepoint-5.3.3 { catchsql {RELEASE def} } {0 {}} @@ -561,7 +561,7 @@ do_test savepoint-8-2 { # ifcapable auth { proc auth {args} { - eval lappend ::authdata $args + eval lappend ::authdata [lrange $args 0 4] return SQLITE_OK } db auth auth @@ -583,7 +583,7 @@ ifcapable auth { } {SQLITE_SAVEPOINT RELEASE sp1 {} {}} proc auth {args} { - eval lappend ::authdata $args + eval lappend ::authdata [lrange $args 0 4] return SQLITE_DENY } db auth auth diff --git a/test/savepoint7.test b/test/savepoint7.test index bc99187d27..d8a02f1f80 100644 --- a/test/savepoint7.test +++ b/test/savepoint7.test @@ -30,6 +30,7 @@ do_test savepoint7-1.1 { db eval {SELECT * FROM t1} { db eval { SAVEPOINT x2; + CREATE TABLE IF NOT EXISTS t3(xyz); INSERT INTO t2 VALUES($a,$b,$c); RELEASE x2; } @@ -46,7 +47,7 @@ do_test savepoint7-1.2 { RELEASE x2; } } - db eval {SELECT * FROM t2} + db eval {SELECT * FROM t2;} } {1 2 3 4 5 6 7 8 9} do_test savepoint7-1.3 { @@ -65,7 +66,7 @@ do_test savepoint7-1.3 { # queries in outer contexts. # do_test savepoint7-2.1 { - db eval {DELETE FROM t2; SAVEPOINT x1;} + db eval {DELETE FROM t2; SAVEPOINT x1; CREATE TABLE t4(abc);} set rc [catch { db eval {SELECT * FROM t1} { db eval { @@ -77,7 +78,7 @@ do_test savepoint7-2.1 { } msg] db eval {RELEASE x1} list $rc $msg [db eval {SELECT * FROM t2}] -} {1 {callback requested query abort} {}} +} {1 {abort due to ROLLBACK} {}} do_test savepoint7-2.2 { db eval {DELETE FROM t2;} @@ -85,12 +86,13 @@ do_test savepoint7-2.2 { db eval {SELECT * FROM t1} { db eval { SAVEPOINT x2; + CREATE TABLE t5(pqr); INSERT INTO t2 VALUES($a,$b,$c); ROLLBACK TO x2; } } } msg] list $rc $msg [db eval {SELECT * FROM t2}] -} {1 {callback requested query abort} {}} +} {1 {abort due to ROLLBACK} {}} finish_test diff --git a/test/scanstatus.test b/test/scanstatus.test new file mode 100644 index 0000000000..ed24d97437 --- /dev/null +++ b/test/scanstatus.test @@ -0,0 +1,398 @@ +# 2014 November 1 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix scanstatus + +ifcapable !scanstatus { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b); + CREATE TABLE t2(x, y); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t2 VALUES('a', 'b'); + INSERT INTO t2 VALUES('c', 'd'); + INSERT INTO t2 VALUES('e', 'f'); +} + +proc do_scanstatus_test {tn res} { + set stmt [db_last_stmt_ptr db] + set idx 0 + set ret [list] + while {1} { + set r [sqlite3_stmt_scanstatus $stmt $idx] + if {[llength $r]==0} break + lappend ret {*}$r + incr idx + } + + uplevel [list do_test $tn [list set {} $ret] [list {*}$res]] +} + +do_execsql_test 1.1 { SELECT count(*) FROM t1, t2; } 6 +do_scanstatus_test 1.2 { + nLoop 1 nVisit 2 nEst 1048576.0 zName t1 zExplain {SCAN TABLE t1} + nLoop 2 nVisit 6 nEst 1048576.0 zName t2 zExplain {SCAN TABLE t2} +} + +do_execsql_test 1.3 { + ANALYZE; + SELECT count(*) FROM t1, t2; +} 6 +do_scanstatus_test 1.4 { + nLoop 1 nVisit 2 nEst 2.0 zName t1 zExplain {SCAN TABLE t1} + nLoop 2 nVisit 6 nEst 3.0 zName t2 zExplain {SCAN TABLE t2} +} + +do_execsql_test 1.5 { ANALYZE } +do_execsql_test 1.6 { + SELECT count(*) FROM t1, t2 WHERE t2.rowid>1; +} 4 +do_scanstatus_test 1.7 { + nLoop 1 nVisit 2 nEst 2.0 zName t2 zExplain + {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid>?)} + nLoop 2 nVisit 4 nEst 2.0 zName t1 zExplain {SCAN TABLE t1} +} + +do_execsql_test 1.8 { + SELECT count(*) FROM t1, t2 WHERE t2.rowid>1; +} 4 + +do_scanstatus_test 1.9 { + nLoop 2 nVisit 4 nEst 2.0 zName t2 zExplain + {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid>?)} + nLoop 4 nVisit 8 nEst 2.0 zName t1 zExplain {SCAN TABLE t1} +} + +do_test 1.9 { + sqlite3_stmt_scanstatus_reset [db_last_stmt_ptr db] +} {} + +do_scanstatus_test 1.10 { + nLoop 0 nVisit 0 nEst 2.0 zName t2 zExplain + {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid>?)} + nLoop 0 nVisit 0 nEst 2.0 zName t1 zExplain {SCAN TABLE t1} +} + +#------------------------------------------------------------------------- +# Try a few different types of scans. +# +reset_db +do_execsql_test 2.1 { + CREATE TABLE x1(i INTEGER PRIMARY KEY, j); + INSERT INTO x1 VALUES(1, 'one'); + INSERT INTO x1 VALUES(2, 'two'); + INSERT INTO x1 VALUES(3, 'three'); + INSERT INTO x1 VALUES(4, 'four'); + CREATE INDEX x1j ON x1(j); + + SELECT * FROM x1 WHERE i=2; +} {2 two} + +do_scanstatus_test 2.2 { + nLoop 1 nVisit 1 nEst 1.0 zName x1 + zExplain {SEARCH TABLE x1 USING INTEGER PRIMARY KEY (rowid=?)} +} + +do_execsql_test 2.3.1 { + SELECT * FROM x1 WHERE j='two' +} {2 two} +do_scanstatus_test 2.3.2 { + nLoop 1 nVisit 1 nEst 10.0 zName x1j + zExplain {SEARCH TABLE x1 USING COVERING INDEX x1j (j=?)} +} + +do_execsql_test 2.4.1 { + SELECT * FROM x1 WHERE j<'two' +} {4 four 1 one 3 three} +do_scanstatus_test 2.4.2 { + nLoop 1 nVisit 3 nEst 262144.0 zName x1j + zExplain {SEARCH TABLE x1 USING COVERING INDEX x1j (j='two' +} {2 two} +do_scanstatus_test 2.5.2 { + nLoop 1 nVisit 1 nEst 262144.0 zName x1j + zExplain {SEARCH TABLE x1 USING COVERING INDEX x1j (j>?)} +} + +do_execsql_test 2.6.1 { + SELECT * FROM x1 WHERE j BETWEEN 'three' AND 'two' +} {3 three 2 two} +do_scanstatus_test 2.6.2 { + nLoop 1 nVisit 2 nEst 16384.0 zName x1j + zExplain {SEARCH TABLE x1 USING COVERING INDEX x1j (j>? AND j? AND j? AND a? AND b? AND b? AND a=1 + # expr $::aborted>=1 + expr $::aborted==0 } {1} # This test is designed to catch a specific bug that was present during diff --git a/test/shell1.test b/test/shell1.test index e6fb0c28d2..27bbe87259 100644 --- a/test/shell1.test +++ b/test/shell1.test @@ -45,20 +45,21 @@ do_test shell1-1.1.1 { list $rc \ [regexp {Error: unknown option: -bad} $res] } {1 1} +do_test shell1-1.1.1b { + set res [catchcmd "test.db -bad" ""] + set rc [lindex $res 0] + list $rc \ + [regexp {Error: unknown option: -bad} $res] +} {1 1} # error on extra options do_test shell1-1.1.2 { - set res [catchcmd "-bad test.db \"select 3\" \"select 4\"" ""] - set rc [lindex $res 0] - list $rc \ - [regexp {Error: too many options: "select 4"} $res] -} {1 1} + catchcmd "test.db \"select 3\" \"select 4\"" "" +} {0 {3 +4}} # error on extra options do_test shell1-1.1.3 { - set res [catchcmd "-bad FOO test.db BAD" ".quit"] - set rc [lindex $res 0] - list $rc \ - [regexp {Error: too many options: "BAD"} $res] -} {1 1} + catchcmd "test.db FOO test.db BAD" ".quit" +} {1 {Error: near "FOO": syntax error}} # -help do_test shell1-1.2.1 { @@ -75,11 +76,11 @@ do_test shell1-1.3.1 { catchcmd "-init FOO test.db" "" } {0 {}} do_test shell1-1.3.2 { - set res [catchcmd "-init FOO test.db .quit BAD" ""] - set rc [lindex $res 0] - list $rc \ - [regexp {Error: too many options: "BAD"} $res] -} {1 1} + catchcmd "-init FOO test.db .quit BAD" "" +} {0 {}} +do_test shell1-1.3.3 { + catchcmd "-init FOO test.db BAD .quit" "" +} {1 {Error: near "BAD": syntax error}} # -echo print commands before execution do_test shell1-1.4.1 { @@ -588,14 +589,17 @@ db eval {DROP VIEW v1; DROP VIEW v2; DROP TABLE t1;} # .separator STRING Change separator used by output mode and .import do_test shell1-3.22.1 { catchcmd "test.db" ".separator" -} {1 {Usage: .separator STRING}} +} {1 {Usage: .separator SEPARATOR ?NEWLINE?}} do_test shell1-3.22.2 { catchcmd "test.db" ".separator FOO" } {0 {}} do_test shell1-3.22.3 { + catchcmd "test.db" ".separator ABC XYZ" +} {0 {}} +do_test shell1-3.22.4 { # too many arguments - catchcmd "test.db" ".separator FOO BAD" -} {1 {Usage: .separator STRING}} + catchcmd "test.db" ".separator FOO BAD BAD2" +} {1 {Usage: .separator SEPARATOR ?NEWLINE?}} # .show Show the current values for various settings do_test shell1-3.23.1 { diff --git a/test/shell2.test b/test/shell2.test index def574c76c..616610bd4b 100644 --- a/test/shell2.test +++ b/test/shell2.test @@ -52,9 +52,9 @@ do_test shell2-1.1.1 { # Ticket [f5cb008a65]. do_test shell2-1.2.1 { set rc [catch { eval exec $CLI \":memory:\" \"select 3\" \"select 4\" } msg] - list $rc \ - [regexp {Error: too many options: "select 4"} $msg] -} {1 1} + list $rc $msg +} {0 {3 +4}} # Test a problem reported on the mailing list. The shell was at one point # returning the generic SQLITE_ERROR message ("SQL error or missing database") diff --git a/test/shell5.test b/test/shell5.test index 6e9dd20639..8d740cb980 100644 --- a/test/shell5.test +++ b/test/shell5.test @@ -55,14 +55,17 @@ do_test shell5-1.1.3 { # .separator STRING Change separator used by output mode and .import do_test shell5-1.2.1 { catchcmd "test.db" ".separator" -} {1 {Usage: .separator STRING}} +} {1 {Usage: .separator SEPARATOR ?NEWLINE?}} do_test shell5-1.2.2 { - catchcmd "test.db" ".separator FOO" + catchcmd "test.db" ".separator ONE" } {0 {}} do_test shell5-1.2.3 { + catchcmd "test.db" ".separator ONE TWO" +} {0 {}} +do_test shell5-1.2.4 { # too many arguments - catchcmd "test.db" ".separator FOO BAD" -} {1 {Usage: .separator STRING}} + catchcmd "test.db" ".separator ONE TWO THREE" +} {1 {Usage: .separator SEPARATOR ?NEWLINE?}} # separator should default to "|" do_test shell5-1.3.1 { diff --git a/test/skipscan1.test b/test/skipscan1.test index 8150b012f6..4f996df972 100644 --- a/test/skipscan1.test +++ b/test/skipscan1.test @@ -245,6 +245,51 @@ do_execsql_test skipscan1-5.3 { SELECT xh, loc FROM t5 WHERE loc >= 'M' AND loc < 'N'; } {/.*COVERING INDEX t5i1 .*/} +# The column used by the skip-scan needs to be sufficiently selective. +# See the private email from Adi Zaimi to drh@sqlite.org on 2014-09-22. +# +db close +forcedelete test.db +sqlite3 db test.db +do_execsql_test skipscan1-6.1 { + CREATE TABLE t1(a,b,c,d,e,f,g,h varchar(300)); + CREATE INDEX t1ab ON t1(a,b); + ANALYZE sqlite_master; + -- Only two distinct values for the skip-scan column. Skip-scan is not used. + INSERT INTO sqlite_stat1 VALUES('t1','t1ab','500000 250000 125000'); + ANALYZE sqlite_master; + EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE b=1; +} {~/ANY/} +do_execsql_test skipscan1-6.2 { + -- Four distinct values for the skip-scan column. Skip-scan is used. + UPDATE sqlite_stat1 SET stat='500000 250000 62500'; + ANALYZE sqlite_master; + EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE b=1; +} {/ANY.a. AND b=/} +do_execsql_test skipscan1-6.3 { + -- Two distinct values for the skip-scan column again. Skip-scan is not used. + UPDATE sqlite_stat1 SET stat='500000 125000 62500'; + ANALYZE sqlite_master; + EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE b=1; +} {~/ANY/} +# If the sqlite_stat1 entry includes the "noskipscan" token, then never use +# skipscan with that index. +# +do_execsql_test skipscan1-7.1 { + UPDATE sqlite_stat1 SET stat='500000 125000 1 sz=100'; + ANALYZE sqlite_master; + EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE b=1; +} {/ANY/} +do_execsql_test skipscan1-7.2 { + UPDATE sqlite_stat1 SET stat='500000 125000 1 noskipscan sz=100'; + ANALYZE sqlite_master; + EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE b=1; +} {~/ANY/} +do_execsql_test skipscan1-7.3 { + UPDATE sqlite_stat1 SET stat='500000 125000 1 sz=100 noskipscan'; + ANALYZE sqlite_master; + EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE b=1; +} {~/ANY/} finish_test diff --git a/test/skipscan3.test b/test/skipscan3.test new file mode 100644 index 0000000000..260d11cac9 --- /dev/null +++ b/test/skipscan3.test @@ -0,0 +1,73 @@ +# 2014-08-20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file implements tests of the "skip-scan" query strategy. +# In particular, this file looks at skipping intermediate terms +# in an index. For example, if (a,b,c) are indexed, and we have +# "WHERE a=?1 AND c=?2" - verify that skip-scan can still be used. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +do_execsql_test skipscan3-1.1 { + CREATE TABLE t1(a,b,c,d,PRIMARY KEY(a,b,c)); + WITH RECURSIVE + c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<1000) + INSERT INTO t1(a,b,c,d) + SELECT 1, 1, x, printf('x%04d',x) FROM c; + ANALYZE; +} {} + +# This version has long used skip-scan because of the "+a" +# +do_execsql_test skipscan3-1.2eqp { + EXPLAIN QUERY PLAN SELECT d FROM t1 WHERE +a=1 AND c=32; +} {/*ANY(a) AND ANY(b)*/} +do_execsql_test skipscan3-1.2 { + SELECT d FROM t1 WHERE +a=1 AND c=32; +} {x0032} + +# This version (with "a" instead of "+a") should use skip-scan but +# did not prior to changes implemented on 2014-08-20 +# +do_execsql_test skipscan3-1.3eqp { + EXPLAIN QUERY PLAN SELECT d FROM t1 WHERE a=1 AND c=32; +} {/*ANY(a) AND ANY(b)*/} +do_execsql_test skipscan3-1.3 { + SELECT d FROM t1 WHERE a=1 AND c=32; +} {x0032} + +# Repeat the test on a WITHOUT ROWID table +# +do_execsql_test skipscan3-2.1 { + CREATE TABLE t2(a,b,c,d,PRIMARY KEY(a,b,c)) WITHOUT ROWID; + WITH RECURSIVE + c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<1000) + INSERT INTO t2(a,b,c,d) + SELECT 1, 1, x, printf('x%04d',x) FROM c; + ANALYZE; +} {} +do_execsql_test skipscan3-2.2eqp { + EXPLAIN QUERY PLAN SELECT d FROM t2 WHERE +a=1 AND c=32; +} {/*ANY(a) AND ANY(b)*/} +do_execsql_test skipscan3-2.2 { + SELECT d FROM t2 WHERE +a=1 AND c=32; +} {x0032} +do_execsql_test skipscan3-2.3eqp { + EXPLAIN QUERY PLAN SELECT d FROM t2 WHERE a=1 AND c=32; +} {/*ANY(a) AND ANY(b)*/} +do_execsql_test skipscan3-2.3 { + SELECT d FROM t2 WHERE a=1 AND c=32; +} {x0032} + + +finish_test diff --git a/test/skipscan5.test b/test/skipscan5.test new file mode 100644 index 0000000000..7c3b166a8c --- /dev/null +++ b/test/skipscan5.test @@ -0,0 +1,182 @@ +# 2013-11-13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file implements tests of the "skip-scan" query strategy. In +# particular it tests that stat4 data can be used by a range query +# that uses the skip-scan approach. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix skipscan5 + +ifcapable !stat4 { + finish_test + return +} + +do_execsql_test 1.1 { + CREATE TABLE t1(a INT, b INT, c INT); + CREATE INDEX i1 ON t1(a, b); +} {} + +expr srand(4) +do_test 1.2 { + for {set i 0} {$i < 100} {incr i} { + set a [expr int(rand()*4.0) + 1] + set b [expr int(rand()*20.0) + 1] + execsql { INSERT INTO t1 VALUES($a, $b, NULL) } + } + execsql ANALYZE +} {} + +foreach {tn q res} { + 1 "b = 5" {/*ANY(a) AND b=?*/} + 2 "b > 12 AND b < 16" {/*ANY(a) AND b>? AND b 2 AND b < 16" {/*SCAN TABLE t1*/} + 4 "b > 18 AND b < 25" {/*ANY(a) AND b>? AND b 15" {/*ANY(a) AND b>?*/} + 6 "b > 5" {/*SCAN TABLE t1*/} + 7 "b < 15" {/*SCAN TABLE t1*/} + 8 "b < 5" {/*ANY(a) AND b b" {/*ANY(a) AND b '12' AND b < '16'" {/*ANY(a) AND b>? AND b '2' AND b < '16'" {/*SCAN TABLE t1*/} + 13 "b > '18' AND b < '25'" {/*ANY(a) AND b>? AND b '15'" {/*ANY(a) AND b>?*/} + 15 "b > '5'" {/*SCAN TABLE t1*/} + 16 "b < '15'" {/*SCAN TABLE t1*/} + 17 "b < '5'" {/*ANY(a) AND b b" {/*ANY(a) AND b? AND c 'q' } {/*ANY(a) AND ANY(b) AND c>?*/} + 4 { c > 'e' } {/*SCAN TABLE t2*/} + 5 { c < 'q' } {/*SCAN TABLE t2*/} + 6 { c < 'c' } {/*ANY(a) AND ANY(b) AND c? AND b X'5555'" {/*ANY(a) AND b>?*/} + 5 "b > 'zzz'" {/*ANY(a) AND b>?*/} + 6 "b < 'zzz'" {/*SCAN TABLE t3*/} +} { + set sql "EXPLAIN QUERY PLAN SELECT * FROM t3 WHERE $q" + do_execsql_test 3.3.$tn $sql $res +} + +finish_test diff --git a/test/skipscan6.test b/test/skipscan6.test new file mode 100644 index 0000000000..026c4d7b00 --- /dev/null +++ b/test/skipscan6.test @@ -0,0 +1,200 @@ +# 2014-10-21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file implements tests of the "skip-scan" query strategy. In +# particular, this file verifies that use of all columns of an index +# is always preferred over the use of a skip-scan on some columns of +# the same index. Because of difficulties in scoring a skip-scan, +# the skip-scan can sometimes come out with a lower raw score when +# using STAT4. But the query planner should detect this and use the +# full index rather than the skip-scan. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix skipscan6 + +ifcapable !stat4 { + finish_test + return +} + +do_execsql_test 1.1 { + CREATE TABLE t1( + aa int, + bb int, + cc int, + dd int, + ee int + ); + CREATE INDEX ix on t1(aa, bb, cc, dd DESC); + ANALYZE sqlite_master; + INSERT INTO sqlite_stat1 VALUES('t1','ix','2695116 1347558 264 18 2'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 196859 196859 32 1','0 15043 15043 92468 92499','0 19 286 81846 92499',X'0609010804031552977BD725BD28'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 14687 161 1 1','0 289067 299306 299457 299457','0 199 6772 273984 299457',X'060902020403013406314D67456415B819'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 19313 19308 22 1','0 325815 325815 343725 343746','0 261 9545 315009 343746',X'060902080403018A49B0A3AD1ED931'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 25047 9051 15 1','0 350443 350443 356590 356604','0 266 9795 325519 356604',X'06090208040301914C2DD2E91F93CF'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 42327 9906 7 1','0 376381 376381 380291 380297','0 268 10100 344232 380297',X'06090208040301934BF672511F7ED3'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 24513 2237 1 1','0 455150 467779 470015 470015','0 286 10880 425401 470015',X'06090202040301A703464A28F2611EF1EE'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 18730 18724 15 1','0 479663 479663 498271 498285','0 287 10998 450793 498285',X'06090208040301A8494AF3A41EC50C'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 119603 47125 1 1','0 572425 572425 598915 598915','0 404 14230 546497 598915',X'06090208040302474FD1929A03194F'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 1454 1454 1 1','0 898346 898346 898373 898373','0 952 31165 827562 898373',X'06090208040304FD53F6A2A2097F64'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 57138 7069 1 1','0 1122389 1122389 1129457 1129457','0 1967 46801 1045943 1129457',X'06090208040309884BC4C52F1F6EB7'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 285 11 1 1','0 1197683 1197824 1197831 1197831','0 2033 50990 1112280 1197831',X'06090202040309D80346503FE2A9038E4F'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 25365 9773 1 1','0 1301013 1301013 1310785 1310785','0 2561 58806 1217877 1310785',X'0609020804030C5F4C8F88AB0AF2A2'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 45180 7222 1 1','0 1326378 1326378 1333599 1333599','0 2562 59921 1240187 1333599',X'0609020804030C604CAB75490B0351'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 8537 41 1 1','0 1496959 1497288 1497289 1497289','0 3050 68246 1394126 1497289',X'0609020204030EA0057F527459B0257C4B'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 26139 26131 17 1','0 1507977 1507977 1520578 1520594','0 3074 69188 1416111 1520594',X'0609020804030EB95169453423D4EA'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 102894 29678 1 1','0 1537421 1550467 1564894 1564894','0 3109 69669 1459820 1564894',X'0609020204030EE3183652A6ED3006EBCB'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 319 3 1 1','0 1796728 1796746 1796747 1796747','0 3650 86468 1682243 1796747',X'0609020204031163033550D0C41018C28D'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 127 127 1 1','0 2096194 2096194 2096205 2096205','0 5145 106437 1951535 2096205',X'060902080403180F53BB1AF727EE50'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 66574 5252 1 1','0 2230524 2265961 2271212 2271212','0 5899 114976 2085829 2271212',X'0609020204031B8A05195009976D223B90'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 19440 19440 1 1','0 2391680 2391680 2395663 2395663','0 6718 123714 2184781 2395663',X'0609020804031F7452E00A7B07431A'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 18321 2177 1 1','0 2522928 2523231 2525407 2525407','0 7838 139084 2299958 2525407',X'06090201040324A7475231103B1AA7B8'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 22384 1361 1 1','0 2541249 2544834 2546194 2546194','0 7839 139428 2308416 2546194',X'06090202040324A8011652323D4B1AA9EB'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 18699 855 1 1','0 2563633 2578178 2579032 2579032','0 7840 139947 2321671 2579032',X'06090202040324A9077452323D7D1052C5'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','17965 1579 1579 1 1','2677151 2690666 2690666 2692244 2692244','1 9870 153959 2418294 2692244',X'060102080403021B8A4FE1AB84032B35'); + ANALYZE sqlite_master; +} {} +do_execsql_test 1.2 { + EXPLAIN QUERY PLAN + SELECT COUNT(*) + FROM t1 + WHERE bb=21 + AND aa=1 + AND dd BETWEEN 1413833728 and 1413837331; +} {/INDEX ix .aa=. AND bb=../} + +do_execsql_test 2.1 { + DROP INDEX ix; + CREATE INDEX good on t1(bb, aa, dd DESC); + CREATE INDEX bad on t1(aa, bb, cc, dd DESC); + DELETE FROM sqlite_stat1; + DELETE FROM sqlite_stat4; + INSERT INTO sqlite_stat1 VALUES('t1','good','2695116 299 264 2'); + INSERT INTO sqlite_stat1 VALUES('t1','bad','2695116 1347558 264 18 2'); + INSERT INTO sqlite_stat4 VALUES('t1','good','197030 196859 32 1','15086 15086 92511 92536','19 25 81644 92536',X'05010904031552977BD725BD22'); + INSERT INTO sqlite_stat4 VALUES('t1','good','14972 14687 1 1','289878 289878 299457 299457','199 244 267460 299457',X'050209040301344F7E569402C419'); + INSERT INTO sqlite_stat4 VALUES('t1','good','19600 19313 22 1','327127 327127 346222 346243','261 319 306884 346243',X'0502090403018A49503BC01EC577'); + INSERT INTO sqlite_stat4 VALUES('t1','good','25666 25047 15 1','352087 352087 372692 372706','266 327 325601 372706',X'050209040301914C2DD2E91F93CF'); + INSERT INTO sqlite_stat4 VALUES('t1','good','42392 42327 26 1','378657 378657 382547 382572','268 331 333529 382572',X'05020904030193533B2FE326ED48'); + INSERT INTO sqlite_stat4 VALUES('t1','good','24619 24513 11 1','457872 457872 461748 461758','286 358 399322 461758',X'050209040301A752B1557825EA7C'); + INSERT INTO sqlite_stat4 VALUES('t1','good','18969 18730 15 1','482491 482491 501105 501119','287 360 433605 501119',X'050209040301A8494AF3A41EC50C'); + INSERT INTO sqlite_stat4 VALUES('t1','good','119710 119603 1 1','576500 576500 598915 598915','404 505 519877 598915',X'05020904030247539A7A7912F617'); + INSERT INTO sqlite_stat4 VALUES('t1','good','11955 11946 1 1','889796 889796 898373 898373','938 1123 794694 898373',X'050209040304EF4DF9C4150BBB28'); + INSERT INTO sqlite_stat4 VALUES('t1','good','57197 57138 24 1','1129865 1129865 1151492 1151515','1967 2273 1027048 1151515',X'05020904030988533510BC26E20A'); + INSERT INTO sqlite_stat4 VALUES('t1','good','3609 3543 1 1','1196265 1196265 1197831 1197831','2002 2313 1070108 1197831',X'050209040309B050E95CD718D94D'); + INSERT INTO sqlite_stat4 VALUES('t1','good','25391 25365 13 1','1309378 1309378 1315567 1315579','2561 2936 1178358 1315579',X'05020904030C5F53DF9E13283570'); + INSERT INTO sqlite_stat4 VALUES('t1','good','45232 45180 17 1','1334769 1334769 1337946 1337962','2562 2938 1198998 1337962',X'05020904030C60541CACEE28BCAC'); + INSERT INTO sqlite_stat4 VALUES('t1','good','5496 5493 1 1','1495882 1495882 1497289 1497289','3043 3479 1348695 1497289',X'05020904030E99515C62AD0F0B34'); + INSERT INTO sqlite_stat4 VALUES('t1','good','26348 26139 17 1','1517381 1517381 1529990 1530006','3074 3519 1378320 1530006',X'05020904030EB95169453423D4EA'); + INSERT INTO sqlite_stat4 VALUES('t1','good','102927 102894 10 1','1547088 1547088 1649950 1649959','3109 3559 1494260 1649959',X'05020904030EE34D309F671FFA47'); + INSERT INTO sqlite_stat4 VALUES('t1','good','3602 3576 1 1','1793873 1793873 1796747 1796747','3601 4128 1630783 1796747',X'050209040311294FE88B432219B9'); + INSERT INTO sqlite_stat4 VALUES('t1','good','154 154 1 1','2096059 2096059 2096205 2096205','5037 5779 1893039 2096205',X'050209040317994EFF05A016DCED'); + INSERT INTO sqlite_stat4 VALUES('t1','good','68153 66574 60 1','2244039 2244039 2268892 2268951','5899 6749 2027553 2268951',X'05020904031B8A532DBC5A26D2BA'); + INSERT INTO sqlite_stat4 VALUES('t1','good','321 321 1 1','2395618 2395618 2395663 2395663','6609 7528 2118435 2395663',X'05020904031EFA54078EEE1E2D65'); + INSERT INTO sqlite_stat4 VALUES('t1','good','19449 19440 22 1','2407769 2407769 2426049 2426070','6718 7651 2146904 2426070',X'05020904031F7450E6118C2336BD'); + INSERT INTO sqlite_stat4 VALUES('t1','good','18383 18321 56 1','2539949 2539949 2551080 2551135','7838 8897 2245459 2551135',X'050209040324A752EA2E1E2642B2'); + INSERT INTO sqlite_stat4 VALUES('t1','good','22479 22384 60 1','2558332 2558332 2565233 2565292','7839 8899 2251202 2565292',X'050209040324A853926538279A5F'); + INSERT INTO sqlite_stat4 VALUES('t1','good','18771 18699 63 1','2580811 2580811 2596914 2596976','7840 8901 2263572 2596976',X'050209040324A9526C1DE9256E72'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 196859 196859 32 1','0 15043 15043 92468 92499','0 19 286 81846 92499',X'0609010804031552977BD725BD28'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 14687 161 1 1','0 289067 299306 299457 299457','0 199 6772 273984 299457',X'060902020403013406314D67456415B819'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 19313 19308 22 1','0 325815 325815 343725 343746','0 261 9545 315009 343746',X'060902080403018A49B0A3AD1ED931'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 25047 9051 15 1','0 350443 350443 356590 356604','0 266 9795 325519 356604',X'06090208040301914C2DD2E91F93CF'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 42327 9906 7 1','0 376381 376381 380291 380297','0 268 10100 344232 380297',X'06090208040301934BF672511F7ED3'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 24513 2237 1 1','0 455150 467779 470015 470015','0 286 10880 425401 470015',X'06090202040301A703464A28F2611EF1EE'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 18730 18724 15 1','0 479663 479663 498271 498285','0 287 10998 450793 498285',X'06090208040301A8494AF3A41EC50C'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 119603 47125 1 1','0 572425 572425 598915 598915','0 404 14230 546497 598915',X'06090208040302474FD1929A03194F'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 1454 1454 1 1','0 898346 898346 898373 898373','0 952 31165 827562 898373',X'06090208040304FD53F6A2A2097F64'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 57138 7069 1 1','0 1122389 1122389 1129457 1129457','0 1967 46801 1045943 1129457',X'06090208040309884BC4C52F1F6EB7'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 285 11 1 1','0 1197683 1197824 1197831 1197831','0 2033 50990 1112280 1197831',X'06090202040309D80346503FE2A9038E4F'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 25365 9773 1 1','0 1301013 1301013 1310785 1310785','0 2561 58806 1217877 1310785',X'0609020804030C5F4C8F88AB0AF2A2'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 45180 7222 1 1','0 1326378 1326378 1333599 1333599','0 2562 59921 1240187 1333599',X'0609020804030C604CAB75490B0351'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 8537 41 1 1','0 1496959 1497288 1497289 1497289','0 3050 68246 1394126 1497289',X'0609020204030EA0057F527459B0257C4B'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 26139 26131 17 1','0 1507977 1507977 1520578 1520594','0 3074 69188 1416111 1520594',X'0609020804030EB95169453423D4EA'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 102894 29678 1 1','0 1537421 1550467 1564894 1564894','0 3109 69669 1459820 1564894',X'0609020204030EE3183652A6ED3006EBCB'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 319 3 1 1','0 1796728 1796746 1796747 1796747','0 3650 86468 1682243 1796747',X'0609020204031163033550D0C41018C28D'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 127 127 1 1','0 2096194 2096194 2096205 2096205','0 5145 106437 1951535 2096205',X'060902080403180F53BB1AF727EE50'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 66574 5252 1 1','0 2230524 2265961 2271212 2271212','0 5899 114976 2085829 2271212',X'0609020204031B8A05195009976D223B90'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 19440 19440 1 1','0 2391680 2391680 2395663 2395663','0 6718 123714 2184781 2395663',X'0609020804031F7452E00A7B07431A'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 18321 2177 1 1','0 2522928 2523231 2525407 2525407','0 7838 139084 2299958 2525407',X'06090201040324A7475231103B1AA7B8'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 22384 1361 1 1','0 2541249 2544834 2546194 2546194','0 7839 139428 2308416 2546194',X'06090202040324A8011652323D4B1AA9EB'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 18699 855 1 1','0 2563633 2578178 2579032 2579032','0 7840 139947 2321671 2579032',X'06090202040324A9077452323D7D1052C5'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','17965 1579 1579 1 1','2677151 2690666 2690666 2692244 2692244','1 9870 153959 2418294 2692244',X'060102080403021B8A4FE1AB84032B35'); + ANALYZE sqlite_master; +} {} +do_execsql_test 2.2 { + EXPLAIN QUERY PLAN + SELECT COUNT(*) + FROM t1 + WHERE bb=21 + AND aa=1 + AND dd BETWEEN 1413833728 and 1413837331; +} {/INDEX good .bb=. AND aa=. AND dd>. AND dd<../} + + +# Create a table containing 100 rows. Column "a" contains a copy of the +# rowid value - sequentially increasing integers from 1 to 100. Column +# "b" contains the value of (a % 5). Columns "c" and "d" both contain +# constant values (i.e. the same for every row). +# +# Then create a second table t2. t2 is the same as t3 except for the +# order in which the indexes are created. +# +do_execsql_test 3.0 { + CREATE TABLE t3(a, b, c, d); + CREATE INDEX t3_ba ON t3(b, a, c); + CREATE INDEX t3_a ON t3(a); + + WITH d(a, b) AS ( + SELECT 1, 1 + UNION ALL + SELECT a+1, (a+1) % 5 FROM d WHERE a<100 + ) + INSERT INTO t3 SELECT a, b, 'c', 'd' FROM d; + + CREATE TABLE t2(a, b, c, d); + CREATE INDEX t2_a ON t2(a); + CREATE INDEX t2_ba ON t2(b, a, c); + INSERT INTO t2 SELECT * FROM t3; + + ANALYZE; + SELECT * FROM sqlite_stat1; +} { + t2 t2_ba {100 20 1 1} + t2 t2_a {100 1} + t3 t3_a {100 1} + t3 t3_ba {100 20 1 1} +} + +# Use index "t3_a", as (a=?) is expected to match only a single row. +# +do_eqp_test 3.1 { + SELECT * FROM t3 WHERE a = ? AND c = ? +} { + 0 0 0 {SEARCH TABLE t3 USING INDEX t3_a (a=?)} +} + +# The same query on table t2. This should use index "t2_a", for the +# same reason. At one point though, it was mistakenly using a skip-scan. +# +do_eqp_test 3.2 { + SELECT * FROM t2 WHERE a = ? AND c = ? +} { + 0 0 0 {SEARCH TABLE t2 USING INDEX t2_a (a=?)} +} + +finish_test + + + + +finish_test diff --git a/test/sort.test b/test/sort.test index 08d496b259..be2a6f531c 100644 --- a/test/sort.test +++ b/test/sort.test @@ -8,13 +8,14 @@ # May you share freely, never taking more than you give. # #*********************************************************************** -# This file implements regression tests for SQLite library. The -# focus of this file is testing the CREATE TABLE statement. # -# $Id: sort.test,v 1.25 2005/11/14 22:29:06 drh Exp $ +# This file implements regression tests for SQLite library. The +# focus of this file is testing the sorter (code in vdbesort.c). +# set testdir [file dirname $argv0] source $testdir/tester.tcl +set testprefix sort # Create a bunch of data to sort against # @@ -464,4 +465,175 @@ do_test sort-12.1 { } } {1 2 xxx 1 3 yyy 1 1 zzz} +#------------------------------------------------------------------------- +# Check that the sorter in vdbesort.c sorts in a stable fashion. +# +do_execsql_test sort-13.0 { + CREATE TABLE t10(a, b); +} +do_test sort-13.1 { + db transaction { + for {set i 0} {$i < 100000} {incr i} { + execsql { INSERT INTO t10 VALUES( $i/10, $i%10 ) } + } + } +} {} +do_execsql_test sort-13.2 { + SELECT a, b FROM t10 ORDER BY a; +} [db eval {SELECT a, b FROM t10 ORDER BY a, b}] +do_execsql_test sort-13.3 { + PRAGMA cache_size = 5; + SELECT a, b FROM t10 ORDER BY a; +} [db eval {SELECT a, b FROM t10 ORDER BY a, b}] + +#------------------------------------------------------------------------- +# Sort some large ( > 4KiB) records. +# +proc cksum {x} { + set i1 1 + set i2 2 + binary scan $x c* L + foreach {a b} $L { + set i1 [expr (($i2<<3) + $a) & 0x7FFFFFFF] + set i2 [expr (($i1<<3) + $b) & 0x7FFFFFFF] + } + list $i1 $i2 +} +db func cksum cksum + +do_execsql_test sort-14.0 { + PRAGMA cache_size = 5; + CREATE TABLE t11(a, b); + INSERT INTO t11 VALUES(randomblob(5000), NULL); + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --2 + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --3 + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --4 + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --5 + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --6 + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --7 + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --8 + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --9 + UPDATE t11 SET b = cksum(a); +} + +foreach {tn mmap_limit} { + 1 0 + 2 1000000 +} { + do_test sort-14.$tn { + sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $mmap_limit + set prev "" + db eval { SELECT * FROM t11 ORDER BY b } { + if {$b != [cksum $a]} {error "checksum failed"} + if {[string compare $b $prev] < 0} {error "sort failed"} + set prev $b + } + set {} {} + } {} +} + +#------------------------------------------------------------------------- +# +foreach {tn mmap_limit nWorker tmpstore coremutex fakeheap softheaplimit} { + 1 0 3 file true false 0 + 2 0 3 file true true 0 + 3 0 0 file true false 0 + 4 1000000 3 file true false 0 + 5 0 0 memory false true 0 + 6 0 0 file false true 1000000 + 7 0 0 file false true 10000 +} { + db close + sqlite3_shutdown + if {$coremutex} { + sqlite3_config multithread + } else { + sqlite3_config singlethread + } + sqlite3_initialize + sorter_test_fakeheap $fakeheap + sqlite3_soft_heap_limit $softheaplimit + + reset_db + sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $mmap_limit + execsql "PRAGMA temp_store = $tmpstore; PRAGMA threads = $nWorker" + + + set ten [string repeat X 10300] + set one [string repeat y 200] + + if {$softheaplimit} { + execsql { PRAGMA cache_size = 20 }; + } else { + execsql { PRAGMA cache_size = 5 }; + } + + do_execsql_test 15.$tn.1 { + WITH rr AS ( + SELECT 4, $ten UNION ALL + SELECT 2, $one UNION ALL + SELECT 1, $ten UNION ALL + SELECT 3, $one + ) + SELECT * FROM rr ORDER BY 1; + } [list 1 $ten 2 $one 3 $one 4 $ten] + + do_execsql_test 15.$tn.2 { + CREATE TABLE t1(a); + INSERT INTO t1 VALUES(4); + INSERT INTO t1 VALUES(5); + INSERT INTO t1 VALUES(3); + INSERT INTO t1 VALUES(2); + INSERT INTO t1 VALUES(6); + INSERT INTO t1 VALUES(1); + CREATE INDEX i1 ON t1(a); + SELECT * FROM t1 ORDER BY a; + } {1 2 3 4 5 6} + + do_execsql_test 15.$tn.3 { + WITH rr AS ( + SELECT 4, $ten UNION ALL + SELECT 2, $one + ) + SELECT * FROM rr ORDER BY 1; + } [list 2 $one 4 $ten] + + sorter_test_fakeheap 0 +} + +db close +sqlite3_shutdown +set t(0) singlethread +set t(1) multithread +set t(2) serialized +sqlite3_config $t($sqlite_options(threadsafe)) +sqlite3_initialize +sqlite3_soft_heap_limit 0 + +reset_db +do_catchsql_test 16.1 { + CREATE TABLE t1(a, b, c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(1, NULL, 3); + INSERT INTO t1 VALUES(NULL, 2, 3); + INSERT INTO t1 VALUES(1, 2, NULL); + INSERT INTO t1 VALUES(4, 5, 6); + CREATE UNIQUE INDEX i1 ON t1(b, a, c); +} {0 {}} +reset_db +do_catchsql_test 16.2 { + CREATE TABLE t1(a, b, c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(1, NULL, 3); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(1, 2, NULL); + INSERT INTO t1 VALUES(4, 5, 6); + CREATE UNIQUE INDEX i1 ON t1(b, a, c); +} {1 {UNIQUE constraint failed: t1.b, t1.a, t1.c}} + +reset_db +do_execsql_test 17.1 { + SELECT * FROM sqlite_master ORDER BY sql; +} {} + finish_test diff --git a/test/sort2.test b/test/sort2.test new file mode 100644 index 0000000000..a4c55c9f24 --- /dev/null +++ b/test/sort2.test @@ -0,0 +1,79 @@ +# 2014 March 25. +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# +# Specifically, the tests in this file attempt to verify that +# multi-threaded sorting works. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix sort2 + +foreach {tn script} { + 1 { } + 2 { + catch { db close } + reset_db + catch { db eval {PRAGMA threads=7} } + } +} { + + eval $script + + do_execsql_test $tn.1 { + PRAGMA cache_size = 5; + WITH r(x,y) AS ( + SELECT 1, randomblob(100) + UNION ALL + SELECT x+1, randomblob(100) FROM r + LIMIT 100000 + ) + SELECT count(x), length(y) FROM r GROUP BY (x%5) + } { + 20000 100 20000 100 20000 100 20000 100 20000 100 + } + + do_execsql_test $tn.2.1 { + CREATE TABLE t1(a, b); + WITH r(x,y) AS ( + SELECT 1, randomblob(100) + UNION ALL + SELECT x+1, randomblob(100) FROM r + LIMIT 10000 + ) INSERT INTO t1 SELECT * FROM r; + } + + do_execsql_test $tn.2.2 { + CREATE UNIQUE INDEX i1 ON t1(b, a); + } + + do_execsql_test $tn.2.3 { + CREATE UNIQUE INDEX i2 ON t1(a); + } + + do_execsql_test $tn.2.4 { PRAGMA integrity_check } {ok} + + do_execsql_test $tn.3 { + PRAGMA cache_size = 5; + WITH r(x,y) AS ( + SELECT 1, randomblob(100) + UNION ALL + SELECT x+1, randomblob(100) FROM r + LIMIT 1000000 + ) + SELECT count(x), length(y) FROM r GROUP BY (x%5) + } { + 200000 100 200000 100 200000 100 200000 100 200000 100 + } +} + +finish_test diff --git a/test/sort3.test b/test/sort3.test new file mode 100644 index 0000000000..80d8bbca3f --- /dev/null +++ b/test/sort3.test @@ -0,0 +1,67 @@ +# 2014 March 25. +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# +# The tests in this file verify that sorting works when the library is +# configured to use mmap(), but the temporary files generated by the +# sorter are too large to be completely mapped. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix sort3 + +# Sort roughly 20MB of data. Once with a mmap limit of 5MB and once without. +# +foreach {itest limit} { + 1 5000000 + 2 0x7FFFFFFF +} { + sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $limit + do_execsql_test 1.$itest { + WITH r(x,y) AS ( + SELECT 1, randomblob(1000) + UNION ALL + SELECT x+1, randomblob(1000) FROM r + LIMIT 20000 + ) + SELECT count(*), sum(length(y)) FROM r GROUP BY (x%5); + } { + 4000 4000000 + 4000 4000000 + 4000 4000000 + 4000 4000000 + 4000 4000000 + } +} + +# Sort more than 2GB of data. At one point this was causing a problem. +# This test might take one minute or more to run. +# +do_execsql_test 2 { + PRAGMA cache_size = 20000; + WITH r(x,y) AS ( + SELECT 1, randomblob(1000) + UNION ALL + SELECT x+1, randomblob(1000) FROM r + LIMIT 2200000 + ) + SELECT count(*), sum(length(y)) FROM r GROUP BY (x%5); +} { + 440000 440000000 + 440000 440000000 + 440000 440000000 + 440000 440000000 + 440000 440000000 +} + +finish_test + diff --git a/test/sort4.test b/test/sort4.test new file mode 100644 index 0000000000..01fcbfee95 --- /dev/null +++ b/test/sort4.test @@ -0,0 +1,189 @@ +# 2014 May 6. +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# +# The tests in this file are brute force tests of the multi-threaded +# sorter. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix sort4 + +# Configure the sorter to use 3 background threads. +db eval {PRAGMA threads=3} + +# Minimum number of seconds to run for. If the value is 0, each test +# is run exactly once. Otherwise, tests are repeated until the timeout +# expires. +set SORT4TIMEOUT 0 +if {[permutation] == "multithread"} { set SORT4TIMEOUT 300 } + +#-------------------------------------------------------------------- +# Set up a table "t1" containing $nRow rows. Each row contains also +# contains blob fields that collectively contain at least $nPayload +# bytes of content. The table schema is as follows: +# +# CREATE TABLE t1(a INTEGER, , b INTEGER); +# +# For each row, the values of columns "a" and "b" are set to the same +# pseudo-randomly selected integer. The "extra-columns", of which there +# are at most eight, are named c0, c1, c2 etc. Column c0 contains a 4 +# byte string. Column c1 an 8 byte string. Field c2 16 bytes, and so on. +# +# This table is intended to be used for testing queries of the form: +# +# SELECT a, , b FROM t1 ORDER BY a; +# +# The test code checks that rows are returned in order, and that the +# values of "a" and "b" are the same for each row (the idea being that +# if field "b" at the end of the sorter record has not been corrupted, +# the rest of the record is probably Ok as well). +# +proc populate_table {nRow nPayload} { + set nCol 0 + + set n 0 + for {set nCol 0} {$n < $nPayload} {incr nCol} { + incr n [expr (4 << $nCol)] + } + + set cols [lrange [list xxx c0 c1 c2 c3 c4 c5 c6 c7] 1 $nCol] + set data [lrange [list xxx \ + randomblob(4) randomblob(8) randomblob(16) randomblob(32) \ + randomblob(64) randomblob(128) randomblob(256) randomblob(512) \ + ] 1 $nCol] + + execsql { DROP TABLE IF EXISTS t1 } + + db transaction { + execsql "CREATE TABLE t1(a, [join $cols ,], b);" + set insert "INSERT INTO t1 VALUES(:k, [join $data ,], :k)" + for {set i 0} {$i < $nRow} {incr i} { + set k [expr int(rand()*1000000000)] + execsql $insert + } + } +} + +# Helper for [do_sorter_test] +# +proc sorter_test {nRow nRead nPayload} { + set res [list] + + set nLoad [expr ($nRow > $nRead) ? $nRead : $nRow] + + set nPayload [expr (($nPayload+3)/4) * 4] + set cols [list] + foreach {mask col} { + 0x04 c0 0x08 c1 0x10 c2 0x20 c3 + 0x40 c4 0x80 c5 0x100 c6 0x200 c7 + } { + if {$nPayload & $mask} { lappend cols $col } + } + + # Create two SELECT statements. Statement $sql1 uses the sorter to sort + # $nRow records of a bit over $nPayload bytes each read from the "t1" + # table created by [populate_table] proc above. Rows are sorted in order + # of the integer field in each "t1" record. + # + # The second SQL statement sorts the same set of rows as the first, but + # uses a LIMIT clause, causing SQLite to use a temp table instead of the + # sorter for sorting. + # + set sql1 "SELECT a, [join $cols ,], b FROM t1 WHERE rowid<=$nRow ORDER BY a" + set sql2 "SELECT a FROM t1 WHERE rowid<=$nRow ORDER BY a LIMIT $nRead" + + # Pass the two SQL statements to a helper command written in C. This + # command steps statement $sql1 $nRead times and compares the integer + # values in the rows returned with the results of executing $sql2. If + # the comparison fails (indicating some bug in the sorter), a Tcl + # exception is thrown. + # + sorter_test_sort4_helper db $sql1 $nRead $sql2 + set {} {} +} + +# Usage: +# +# do_sorter_test ... +# +# where are any of the following switches: +# +# -rows N (number of rows to have sorter sort) +# -read N (number of rows to read out of sorter) +# -payload N (bytes of payload to read with each row) +# -cachesize N (Value for "PRAGMA cache_size = ?") +# -repeats N (number of times to repeat test) +# -fakeheap BOOL (true to use separate allocations for in-memory records) +# +proc do_sorter_test {tn args} { + set a(-rows) 1000 + set a(-repeats) 1 + set a(-read) 100 + set a(-payload) 100 + set a(-cachesize) 100 + set a(-fakeheap) 0 + + foreach {s val} $args { + if {[info exists a($s)]==0} { + unset a(-cachesize) + set optlist "[join [array names a] ,] or -cachesize" + error "Unknown option $s, expected $optlist" + } + set a($s) $val + } + if {[permutation] == "memsys3" || [permutation] == "memsys5"} { + set a(-fakeheap) 0 + } + if {$a(-fakeheap)} { sorter_test_fakeheap 1 } + + + db eval "PRAGMA cache_size = $a(-cachesize)" + do_test $tn [subst -nocommands { + for {set i 0} {[set i] < $a(-repeats)} {incr i} { + sorter_test $a(-rows) $a(-read) $a(-payload) + } + }] {} + + if {$a(-fakeheap)} { sorter_test_fakeheap 0 } +} + +proc clock_seconds {} { + db one {SELECT strftime('%s')} +} + +#------------------------------------------------------------------------- +# Begin tests here. + +# Create a test database. +do_test 1 { + execsql "PRAGMA page_size = 4096" + populate_table 100000 500 +} {} + +set iTimeLimit [expr [clock_seconds] + $SORT4TIMEOUT] + +for {set t 2} {1} {incr tn} { + do_sorter_test $t.2 -repeats 10 -rows 1000 -read 100 + do_sorter_test $t.3 -repeats 10 -rows 100000 -read 1000 + do_sorter_test $t.4 -repeats 10 -rows 100000 -read 1000 -payload 500 + do_sorter_test $t.5 -repeats 10 -rows 100000 -read 100000 -payload 8 + do_sorter_test $t.6 -repeats 10 -rows 100000 -read 10 -payload 8 + do_sorter_test $t.7 -repeats 10 -rows 10000 -read 10000 -payload 8 -fakeheap 1 + do_sorter_test $t.8 -repeats 10 -rows 100000 -read 10000 -cachesize 250 + + set iNow [clock_seconds] + if {$iNow>=$iTimeLimit} break + do_test "$testprefix-([expr $iTimeLimit-$iNow] seconds remain)" {} {} +} + +finish_test diff --git a/test/sort5.test b/test/sort5.test new file mode 100644 index 0000000000..a02b8f920f --- /dev/null +++ b/test/sort5.test @@ -0,0 +1,45 @@ +# 2014 September 15. +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix sort5 + + +#------------------------------------------------------------------------- +# Verify that sorting works with a version 1 sqlite3_io_methods structure. +# +testvfs tvfs -iversion 1 -default true +reset_db +do_execsql_test 1.0 { + PRAGMA mmap_size = 10000000; + PRAGMA cache_size = 10; + CREATE TABLE t1(a, b); +} {0} + +do_test 1.1 { + execsql BEGIN + for {set i 0} {$i < 2000} {incr i} { + execsql { INSERT INTO t1 VALUES($i, randomblob(2000)) } + } + execsql COMMIT +} {} + +do_execsql_test 1.2 { + CREATE INDEX i1 ON t1(b); +} + +db close +tvfs delete +finish_test + diff --git a/test/sortfault.test b/test/sortfault.test new file mode 100644 index 0000000000..a1983ac1c0 --- /dev/null +++ b/test/sortfault.test @@ -0,0 +1,165 @@ +# 2014 March 25. +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# +# Specifically, it tests the effects of fault injection on the sorter +# module (code in vdbesort.c). +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix sortfault + +do_execsql_test 1.0 { + PRAGMA cache_size = 5; +} + +foreach {tn mmap_limit nWorker tmpstore threadsmode fakeheap lookaside} { + 1 0 0 file multithread false false + 2 100000 0 file multithread false false + 3 100000 1 file multithread false false + 4 2000000 0 file singlethread false true +} { + if {$sqlite_options(threadsafe)} { set threadsmode singlethread } + + db eval "PRAGMA threads=$nWorker" + sqlite3_config $threadsmode + if { $lookaside } { + sqlite3_config_lookaside 100 500 + } else { + sqlite3_config_lookaside 0 0 + } + sqlite3_initialize + sorter_test_fakeheap $fakeheap + + set str [string repeat a 1000] + puts $threadsmode + + do_faultsim_test 1.$tn -prep { + sqlite3 db test.db + sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $::mmap_limit + execsql { PRAGMA cache_size = 5 } + } -body { + execsql { + WITH r(x,y) AS ( + SELECT 1, $::str + UNION ALL + SELECT x+1, $::str FROM r + LIMIT 200 + ) + SELECT count(x), length(y) FROM r GROUP BY (x%5) + } + } -test { + faultsim_test_result {0 {40 1000 40 1000 40 1000 40 1000 40 1000}} + } + + do_faultsim_test 2.$tn -faults oom* -prep { + sqlite3 db test.db + sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $::mmap_limit + add_test_utf16bin_collate db + execsql { PRAGMA cache_size = 5 } + } -body { + execsql { + WITH r(x,y) AS ( + SELECT 100, $::str + UNION ALL + SELECT x-1, $::str FROM r + LIMIT 100 + ) + SELECT count(x), length(y) FROM r GROUP BY y COLLATE utf16bin, (x%5) + } + } -test { + faultsim_test_result {0 {20 1000 20 1000 20 1000 20 1000 20 1000}} + } + + if {$mmap_limit > 1000000} { + set str2 [string repeat $str 10] + + sqlite3_memdebug_vfs_oom_test 0 + sqlite3 db test.db + sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $::mmap_limit + execsql { PRAGMA cache_size = 5 } + + do_faultsim_test 3.$tn -faults oom-trans* -body { + execsql { + WITH r(x,y) AS ( + SELECT 300, $::str2 + UNION ALL + SELECT x-1, $::str2 FROM r + LIMIT 300 + ) + SELECT count(x), length(y) FROM r GROUP BY y, (x%5) + } + } -test { + faultsim_test_result {0 {60 10000 60 10000 60 10000 60 10000 60 10000}} + } + + sqlite3_memdebug_vfs_oom_test 1 + } +} + +catch { db close } +sqlite3_shutdown +set t(0) singlethread +set t(1) multithread +set t(2) serialized +sqlite3_config $t($sqlite_options(threadsafe)) +sqlite3_config_lookaside 100 500 +sqlite3_initialize + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 4.0 { + CREATE TABLE t1(a, b, c); + INSERT INTO t1 VALUES(1, 2, 3); +} +do_test 4.1 { + for {set i 0} {$i < 256} {incr i} { + execsql { + INSERT INTO t1 SELECT + ((a<<3) + b) & 2147483647, + ((b<<3) + c) & 2147483647, + ((c<<3) + a) & 2147483647 + FROM t1 ORDER BY rowid DESC LIMIT 1; + } + } +} {} + +faultsim_save_and_close + +do_faultsim_test 4.2 -faults oom* -prep { + faultsim_restore_and_reopen +} -body { + execsql { CREATE UNIQUE INDEX i1 ON t1(a,b,c) } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +# +reset_db +set a [string repeat a 500] +set b [string repeat b 500] +set c [string repeat c 500] +do_execsql_test 5.0 { + CREATE TABLE t1(a, b, c); + INSERT INTO t1 VALUES($a, $b, $c); + INSERT INTO t1 VALUES($c, $b, $a); +} + +do_faultsim_test 5.1 -faults oom* -body { + execsql { SELECT * FROM t1 ORDER BY a } +} -test { + faultsim_test_result [list 0 [list $::a $::b $::c $::c $::b $::a]] +} + +finish_test diff --git a/test/speedtest1.c b/test/speedtest1.c index 383f5809a9..8e5b74c56e 100644 --- a/test/speedtest1.c +++ b/test/speedtest1.c @@ -27,6 +27,7 @@ static const char zHelp[] = " --stats Show statistics at the end\n" " --testset T Run test-set T\n" " --trace Turn on SQL tracing\n" + " --threads N Use up to N threads for sorting\n" " --utf16be Set text encoding to UTF-16BE\n" " --utf16le Set text encoding to UTF-16LE\n" " --verify Run additional verification steps.\n" @@ -933,6 +934,7 @@ void testset_cte(void){ } +#ifdef SQLITE_ENABLE_RTREE /* Generate two numbers between 1 and mx. The first number is less than ** the second. Usually the numbers are near each other but can sometimes ** be far apart. @@ -953,7 +955,9 @@ static void twoCoords( *pX0 = x0; *pX1 = x1; } +#endif +#ifdef SQLITE_ENABLE_RTREE /* The following routine is an R-Tree geometry callback. It returns ** true if the object overlaps a slice on the Y coordinate between the ** two values given as arguments. In other words @@ -973,7 +977,9 @@ static int xsliceGeometryCallback( *pRes = aCoord[3]>=p->aParam[0] && aCoord[2]<=p->aParam[1]; return SQLITE_OK; } +#endif /* SQLITE_ENABLE_RTREE */ +#ifdef SQLITE_ENABLE_RTREE /* ** A testset for the R-Tree virtual table */ @@ -1109,6 +1115,7 @@ void testset_rtree(int p1, int p2){ } speedtest1_end_test(); } +#endif /* SQLITE_ENABLE_RTREE */ /* ** A testset used for debugging speedtest1 itself. @@ -1141,6 +1148,7 @@ int main(int argc, char **argv){ int nPCache = 0, szPCache = 0;/* --pcache configuration */ int nScratch = 0, szScratch=0;/* --scratch configuration */ int showStats = 0; /* True for --stats */ + int nThread = 0; /* --threads value */ const char *zTSet = "main"; /* Which --testset torun */ int doTrace = 0; /* True for --trace */ const char *zEncoding = 0; /* --utf16be or --utf16le */ @@ -1225,6 +1233,9 @@ int main(int argc, char **argv){ zTSet = argv[++i]; }else if( strcmp(z,"trace")==0 ){ doTrace = 1; + }else if( strcmp(z,"threads")==0 ){ + if( i>=argc-1 ) fatal_error("missing argument on %s\n", argv[i]); + nThread = integerValue(argv[++i]); }else if( strcmp(z,"utf16le")==0 ){ zEncoding = "utf16le"; }else if( strcmp(z,"utf16be")==0 ){ @@ -1290,6 +1301,7 @@ int main(int argc, char **argv){ /* Set database connection options */ sqlite3_create_function(g.db, "random", 0, SQLITE_UTF8, 0, randomFunc, 0, 0); if( doTrace ) sqlite3_trace(g.db, traceCallback, 0); + speedtest1_exec("PRAGMA threads=%d", nThread); if( zKey ){ speedtest1_exec("PRAGMA key('%s')", zKey); } @@ -1323,7 +1335,12 @@ int main(int argc, char **argv){ }else if( strcmp(zTSet,"cte")==0 ){ testset_cte(); }else if( strcmp(zTSet,"rtree")==0 ){ +#ifdef SQLITE_ENABLE_RTREE testset_rtree(6, 147); +#else + fatal_error("compile with -DSQLITE_ENABLE_RTREE to enable " + "the R-Tree tests\n"); +#endif }else{ fatal_error("unknown testset: \"%s\"\nChoices: main debug1 cte rtree\n", zTSet); diff --git a/test/spellfix.test b/test/spellfix.test index 21383c2d33..954bdb21f1 100644 --- a/test/spellfix.test +++ b/test/spellfix.test @@ -124,6 +124,22 @@ do_execsql_test 1.23 { SELECT next_char('ab','vocab2','w',null,'binary'); } {c} +do_execsql_test 1.30 { + SELECT rowid FROM t1 WHERE word='rabbit'; +} {2} +do_execsql_test 1.31 { + UPDATE t1 SET rowid=2000 WHERE word='rabbit'; + SELECT rowid FROM t1 WHERE word='rabbit'; +} {2000} +do_execsql_test 1.32 { + INSERT INTO t1(rowid, word) VALUES(3000,'melody'); + SELECT rowid, word, matchlen FROM t1 WHERE word MATCH 'melotti' + ORDER BY score LIMIT 3; +} {3000 melody 6} +do_test 1.33 { + catchsql {INSERT INTO t1(rowid, word) VALUES(3000,'garden');} +} {1 {constraint failed}} + do_execsql_test 2.1 { CREATE VIRTUAL TABLE t2 USING spellfix1; INSERT INTO t2 (word, soundslike) VALUES('school', 'skuul'); diff --git a/test/sqllimits1.test b/test/sqllimits1.test index 2cbad3ffb8..57fc931f7c 100644 --- a/test/sqllimits1.test +++ b/test/sqllimits1.test @@ -51,6 +51,13 @@ do_test sqllimits1-1.9 { do_test sqllimits1-1.10 { sqlite3_limit db SQLITE_LIMIT_VARIABLE_NUMBER -1 } $SQLITE_MAX_VARIABLE_NUMBER +do_test sqllimits1-1.11 { + sqlite3_limit db SQLITE_LIMIT_TRIGGER_DEPTH -1 +} $SQLITE_MAX_TRIGGER_DEPTH +do_test sqllimits1-1.12 { + sqlite3_limit db SQLITE_LIMIT_WORKER_THREADS 99999 + sqlite3_limit db SQLITE_LIMIT_WORKER_THREADS -1 +} $SQLITE_MAX_WORKER_THREADS # Limit parameters out of range. # diff --git a/test/subquery2.test b/test/subquery2.test index 4406efccf2..de637d5d25 100644 --- a/test/subquery2.test +++ b/test/subquery2.test @@ -103,5 +103,50 @@ do_execsql_test 2.2 { LIMIT (SELECT a FROM t5) } {2 3 3 6 4 10} +############################################################################ +# Ticket http://www.sqlite.org/src/info/d11a6e908f (2014-09-20) +# Query planner fault on three-way nested join with compound inner SELECT +# +do_execsql_test 3.0 { + DROP TABLE IF EXISTS t1; + DROP TABLE IF EXISTS t2; + CREATE TABLE t1 (id INTEGER PRIMARY KEY, data TEXT); + INSERT INTO t1(id,data) VALUES(9,'nine-a'); + INSERT INTO t1(id,data) VALUES(10,'ten-a'); + INSERT INTO t1(id,data) VALUES(11,'eleven-a'); + CREATE TABLE t2 (id INTEGER PRIMARY KEY, data TEXT); + INSERT INTO t2(id,data) VALUES(9,'nine-b'); + INSERT INTO t2(id,data) VALUES(10,'ten-b'); + INSERT INTO t2(id,data) VALUES(11,'eleven-b'); + + SELECT id FROM ( + SELECT id,data FROM ( + SELECT * FROM t1 UNION ALL SELECT * FROM t2 + ) + WHERE id=10 ORDER BY data + ); +} {10 10} +do_execsql_test 3.1 { + SELECT data FROM ( + SELECT 'dummy', data FROM ( + SELECT data FROM t1 UNION ALL SELECT data FROM t1 + ) ORDER BY data + ); +} {eleven-a eleven-a nine-a nine-a ten-a ten-a} +do_execsql_test 3.2 { + DROP TABLE IF EXISTS t3; + DROP TABLE IF EXISTS t4; + CREATE TABLE t3(id INTEGER, data TEXT); + CREATE TABLE t4(id INTEGER, data TEXT); + INSERT INTO t3 VALUES(4, 'a'),(2,'c'); + INSERT INTO t4 VALUES(3, 'b'),(1,'d'); + + SELECT data, id FROM ( + SELECT id, data FROM ( + SELECT * FROM t3 UNION ALL SELECT * FROM t4 + ) ORDER BY data + ); +} {a 4 b 3 c 2 d 1} + finish_test diff --git a/test/table.test b/test/table.test index ed9efc02cd..69f105aa6c 100644 --- a/test/table.test +++ b/test/table.test @@ -11,7 +11,6 @@ # This file implements regression tests for SQLite library. The # focus of this file is testing the CREATE TABLE statement. # -# $Id: table.test,v 1.53 2009/06/05 17:09:12 drh Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -726,4 +725,68 @@ do_test table-15.2 { execsql {COMMIT} } {} +# Ticket 3a88d85f36704eebe134f7f48aebf00cd6438c1a (2014-08-05) +# The following SQL script segfaults while running the INSERT statement: +# +# CREATE TABLE t1(x DEFAULT(max(1))); +# INSERT INTO t1(rowid) VALUES(1); +# +# The problem appears to be the use of an aggregate function as part of +# the default value for a column. This problem has been in the code since +# at least 2006-01-01 and probably before that. This problem was detected +# and reported on the sqlite-users@sqlite.org mailing list by Zsbán Ambrus. +# +do_execsql_test table-16.1 { + CREATE TABLE t16(x DEFAULT(max(1))); + INSERT INTO t16(x) VALUES(123); + SELECT rowid, x FROM t16; +} {1 123} +do_catchsql_test table-16.2 { + INSERT INTO t16(rowid) VALUES(4); +} {1 {unknown function: max()}} +do_execsql_test table-16.3 { + DROP TABLE t16; + CREATE TABLE t16(x DEFAULT(abs(1))); + INSERT INTO t16(rowid) VALUES(4); + SELECT rowid, x FROM t16; +} {4 1} +do_catchsql_test table-16.4 { + DROP TABLE t16; + CREATE TABLE t16(x DEFAULT(avg(1))); + INSERT INTO t16(rowid) VALUES(123); + SELECT rowid, x FROM t16; +} {1 {unknown function: avg()}} +do_catchsql_test table-16.5 { + DROP TABLE t16; + CREATE TABLE t16(x DEFAULT(count())); + INSERT INTO t16(rowid) VALUES(123); + SELECT rowid, x FROM t16; +} {1 {unknown function: count()}} +do_catchsql_test table-16.6 { + DROP TABLE t16; + CREATE TABLE t16(x DEFAULT(group_concat('x',','))); + INSERT INTO t16(rowid) VALUES(123); + SELECT rowid, x FROM t16; +} {1 {unknown function: group_concat()}} +do_catchsql_test table-16.7 { + INSERT INTO t16 DEFAULT VALUES; +} {1 {unknown function: group_concat()}} + +# Ticket [https://www.sqlite.org/src/info/094d39a4c95ee4abbc417f04214617675ba15c63] +# describes a assertion fault that occurs on a CREATE TABLE .. AS SELECT statement. +# the following test verifies that the problem has been fixed. +# +do_execsql_test table-17.1 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a TEXT); + INSERT INTO t1(a) VALUES(1),(2); + DROP TABLE IF EXISTS t2; + CREATE TABLE t2(x TEXT, y TEXT); + INSERT INTO t2(x,y) VALUES(3,4); + DROP TABLE IF EXISTS t3; + CREATE TABLE t3 AS + SELECT a AS p, coalesce(y,a) AS q FROM t1 LEFT JOIN t2 ON a=x; + SELECT p, q, '|' FROM t3 ORDER BY p; +} {1 1 | 2 2 |} + finish_test diff --git a/test/tester.tcl b/test/tester.tcl index 2b5d871d5f..48931cb915 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -863,6 +863,7 @@ proc speed_trial_summary {name} { # proc finish_test {} { catch {db close} + catch {db1 close} catch {db2 close} catch {db3 close} if {0==[info exists ::SLAVE]} { finalize_testing } @@ -1076,6 +1077,7 @@ proc explain_i {sql {db db}} { foreach opcode { Seek SeekGe SeekGt SeekLe SeekLt NotFound Last Rewind NoConflict Next Prev VNext VPrev VFilter + SorterSort SorterNext } { set color($opcode) $B } @@ -1098,6 +1100,7 @@ proc explain_i {sql {db db}} { if {$opcode=="Next" || $opcode=="Prev" || $opcode=="VNext" || $opcode=="VPrev" + || $opcode=="SorterNext" } { for {set i $p2} {$i<$addr} {incr i} { incr x($i) 2 diff --git a/test/threadtest3.c b/test/threadtest3.c index cb7e2fa41b..25caeb89f9 100644 --- a/test/threadtest3.c +++ b/test/threadtest3.c @@ -1,41 +1,41 @@ - /* +** 2010-07-22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** ** The code in this file runs a few multi-threaded test cases using the ** SQLite library. It can be compiled to an executable on unix using the ** following command: ** ** gcc -O2 threadtest3.c sqlite3.c -ldl -lpthread -lm ** -** Then run the compiled program. The exit status is non-zero if any tests -** failed (hopefully there is also some output to stdout to clarify what went -** wrong). +** Even though threadtest3.c is the only C source code file mentioned on +** the compiler command-line, #include macros are used to pull in additional +** C code files named "tt3_*.c". ** -** There are three parts to the code in this file, in the following order: +** After compiling, run this program with an optional argument telling +** which test to run. All tests are run if no argument is given. The +** argument can be a glob pattern to match multiple tests. Examples: ** -** 1. Code for the SQL aggregate function md5sum() copied from -** tclsqlite.c in the SQLite distribution. The names of all the -** types and functions in this section begin with "MD5" or "md5". +** ./a.out -- Run all tests +** ./a.out walthread3 -- Run the "walthread3" test +** ./a.out 'wal*' -- Run all of the wal* tests +** ./a.out --help -- List all available tests ** -** 2. A set of utility functions that may be used to implement -** multi-threaded test cases. These are all called by test code -** via macros that help with error reporting. The macros are defined -** immediately below this comment. -** -** 3. The test code itself. And a main() routine to drive the test -** code. +** The exit status is non-zero if any test fails. */ -/************************************************************************* -** Start of test code/infrastructure interface macros. -** -** The following macros constitute the interface between the test -** programs and the test infrastructure. Test infrastructure code -** does not itself use any of these macros. Test code should not -** call any of the macroname_x() functions directly. -** -** See the header comments above the corresponding macroname_x() -** function for a description of each interface. +/* +** The "Set Error Line" macro. */ +#define SEL(e) ((e)->iLine = ((e)->rc ? (e)->iLine : __LINE__)) /* Database functions */ #define opendb(w,x,y,z) (SEL(w), opendb_x(w,x,y,z)) @@ -47,10 +47,13 @@ #define execsql_i64(x,y,...) (SEL(x), execsql_i64_x(x,y,__VA_ARGS__)) #define execsql_text(x,y,z,...) (SEL(x), execsql_text_x(x,y,z,__VA_ARGS__)) #define execsql(x,y,...) (SEL(x), (void)execsql_i64_x(x,y,__VA_ARGS__)) +#define sql_script_printf(x,y,z,...) ( \ + SEL(x), sql_script_printf_x(x,y,z,__VA_ARGS__) \ +) /* Thread functions */ -#define launch_thread(w,x,y,z) (SEL(w), launch_thread_x(w,x,y,z)) -#define join_all_threads(y,z) (SEL(y), join_all_threads_x(y,z)) +#define launch_thread(w,x,y,z) (SEL(w), launch_thread_x(w,x,y,z)) +#define join_all_threads(y,z) (SEL(y), join_all_threads_x(y,z)) /* Timer functions */ #define setstoptime(y,z) (SEL(y), setstoptime_x(y,z)) @@ -64,6 +67,9 @@ #define filesize(y,z) (SEL(y), filesize_x(y,z)) #define filecopy(x,y,z) (SEL(x), filecopy_x(x,y,z)) +#define PTR2INT(x) ((int)((intptr_t)x)) +#define INT2PTR(x) ((void*)((intptr_t)x)) + /* ** End of test code/infrastructure interface macros. *************************************************************************/ @@ -115,7 +121,10 @@ struct MD5Context { int isInit; uint32 buf[4]; uint32 bits[2]; - unsigned char in[64]; + union { + unsigned char in[64]; + uint32 in32[16]; + } u; }; typedef struct MD5Context MD5Context; @@ -264,7 +273,7 @@ void MD5Update(MD5Context *ctx, const unsigned char *buf, unsigned int len){ /* Handle any leading odd-sized chunks */ if ( t ) { - unsigned char *p = (unsigned char *)ctx->in + t; + unsigned char *p = (unsigned char *)ctx->u.in + t; t = 64-t; if (len < t) { @@ -272,8 +281,8 @@ void MD5Update(MD5Context *ctx, const unsigned char *buf, unsigned int len){ return; } memcpy(p, buf, t); - byteReverse(ctx->in, 16); - MD5Transform(ctx->buf, (uint32 *)ctx->in); + byteReverse(ctx->u.in, 16); + MD5Transform(ctx->buf, (uint32 *)ctx->u.in); buf += t; len -= t; } @@ -281,16 +290,16 @@ void MD5Update(MD5Context *ctx, const unsigned char *buf, unsigned int len){ /* Process data in 64-byte chunks */ while (len >= 64) { - memcpy(ctx->in, buf, 64); - byteReverse(ctx->in, 16); - MD5Transform(ctx->buf, (uint32 *)ctx->in); + memcpy(ctx->u.in, buf, 64); + byteReverse(ctx->u.in, 16); + MD5Transform(ctx->buf, (uint32 *)ctx->u.in); buf += 64; len -= 64; } /* Handle any remaining bytes of data. */ - memcpy(ctx->in, buf, len); + memcpy(ctx->u.in, buf, len); } /* @@ -306,7 +315,7 @@ static void MD5Final(unsigned char digest[16], MD5Context *ctx){ /* Set the first char of padding to 0x80. This is safe since there is always at least one byte free */ - p = ctx->in + count; + p = ctx->u.in + count; *p++ = 0x80; /* Bytes of padding needed to make 64 bytes */ @@ -316,25 +325,25 @@ static void MD5Final(unsigned char digest[16], MD5Context *ctx){ if (count < 8) { /* Two lots of padding: Pad the first block to 64 bytes */ memset(p, 0, count); - byteReverse(ctx->in, 16); - MD5Transform(ctx->buf, (uint32 *)ctx->in); + byteReverse(ctx->u.in, 16); + MD5Transform(ctx->buf, (uint32 *)ctx->u.in); /* Now fill the next block with 56 bytes */ - memset(ctx->in, 0, 56); + memset(ctx->u.in, 0, 56); } else { /* Pad block to 56 bytes */ memset(p, 0, count-8); } - byteReverse(ctx->in, 14); + byteReverse(ctx->u.in, 14); /* Append length in bits and transform */ - ((uint32 *)ctx->in)[ 14 ] = ctx->bits[0]; - ((uint32 *)ctx->in)[ 15 ] = ctx->bits[1]; + ctx->u.in32[14] = ctx->bits[0]; + ctx->u.in32[15] = ctx->bits[1]; - MD5Transform(ctx->buf, (uint32 *)ctx->in); + MD5Transform(ctx->buf, (uint32 *)ctx->u.in); byteReverse((unsigned char *)ctx->buf, 4); memcpy(digest, ctx->buf, 16); - memset(ctx, 0, sizeof(ctx)); /* In case it is sensitive */ + memset(ctx, 0, sizeof(*ctx)); /* In case it is sensitive */ } /* @@ -382,9 +391,9 @@ static void md5finalize(sqlite3_context *context){ sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT); } -/************************************************************************* +/* ** End of copied md5sum() code. -*/ +**************************************************************************/ typedef sqlite3_int64 i64; @@ -398,9 +407,6 @@ typedef struct Thread Thread; /* Total number of errors in this process so far. */ static int nGlobalErr = 0; -/* Set to true to run in "process" instead of "thread" mode. */ -static int bProcessMode = 0; - struct Error { int rc; int iLine; @@ -421,10 +427,10 @@ struct Statement { struct Thread { int iTid; /* Thread number within test */ - int iArg; /* Integer argument passed by caller */ + void* pArg; /* Pointer argument passed by caller */ pthread_t tid; /* Thread id */ - char *(*xProc)(int, int); /* Thread main proc */ + char *(*xProc)(int, void*); /* Thread main proc */ Thread *pNext; /* Next in this list of threads */ }; @@ -441,8 +447,13 @@ static void free_err(Error *p){ static void print_err(Error *p){ if( p->rc!=SQLITE_OK ){ - printf("Error: (%d) \"%s\" at line %d\n", p->rc, p->zErr, p->iLine); - nGlobalErr++; + int isWarn = 0; + if( p->rc==SQLITE_SCHEMA ) isWarn = 1; + if( sqlite3_strglob("* - no such table: *",p->zErr)==0 ) isWarn = 1; + printf("%s: (%d) \"%s\" at line %d\n", isWarn ? "Warning" : "Error", + p->rc, p->zErr, p->iLine); + if( !isWarn ) nGlobalErr++; + fflush(stdout); } } @@ -506,8 +517,9 @@ static void opendb_x( ){ if( pErr->rc==SQLITE_OK ){ int rc; + int flags = SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE | SQLITE_OPEN_URI; if( bDelete ) unlink(zFile); - rc = sqlite3_open(zFile, &pDb->db); + rc = sqlite3_open_v2(zFile, &pDb->db, flags, 0); if( rc ){ sqlite_error(pErr, pDb, "open"); sqlite3_close(pDb->db); @@ -556,6 +568,22 @@ static void sql_script_x( } } +static void sql_script_printf_x( + Error *pErr, /* IN/OUT: Error code */ + Sqlite *pDb, /* Database handle */ + const char *zFormat, /* SQL printf format string */ + ... /* Printf args */ +){ + va_list ap; /* ... printf arguments */ + va_start(ap, zFormat); + if( pErr->rc==SQLITE_OK ){ + char *zSql = sqlite3_vmprintf(zFormat, ap); + pErr->rc = sqlite3_exec(pDb->db, zSql, 0, 0, &pErr->zErr); + sqlite3_free(zSql); + } + va_end(ap); +} + static Statement *getSqlStatement( Error *pErr, /* IN/OUT: Error code */ Sqlite *pDb, /* Database handle */ @@ -624,11 +652,9 @@ static i64 execsql_i64_x( if( pErr->rc==SQLITE_OK ){ sqlite3_stmt *pStmt; /* SQL statement to execute */ va_list ap; /* ... arguments */ - int i; /* Used to iterate through parameters */ va_start(ap, pDb); pStmt = getAndBindSqlStatement(pErr, pDb, ap); if( pStmt ){ - int rc; int first = 1; while( SQLITE_ROW==sqlite3_step(pStmt) ){ if( first && sqlite3_column_count(pStmt)>0 ){ @@ -663,11 +689,9 @@ static char * execsql_text_x( if( pErr->rc==SQLITE_OK ){ sqlite3_stmt *pStmt; /* SQL statement to execute */ va_list ap; /* ... arguments */ - int i; /* Used to iterate through parameters */ va_start(ap, iSlot); pStmt = getAndBindSqlStatement(pErr, pDb, ap); if( pStmt ){ - int rc; int first = 1; while( SQLITE_ROW==sqlite3_step(pStmt) ){ if( first && sqlite3_column_count(pStmt)>0 ){ @@ -693,14 +717,13 @@ static void integrity_check_x( ){ if( pErr->rc==SQLITE_OK ){ Statement *pStatement; /* Statement to execute */ - int rc; /* Return code */ char *zErr = 0; /* Integrity check error */ pStatement = getSqlStatement(pErr, pDb, "PRAGMA integrity_check"); if( pStatement ){ sqlite3_stmt *pStmt = pStatement->pStmt; while( SQLITE_ROW==sqlite3_step(pStmt) ){ - const char *z = sqlite3_column_text(pStmt, 0); + const char *z = (const char*)sqlite3_column_text(pStmt, 0); if( strcmp(z, "ok") ){ if( zErr==0 ){ zErr = sqlite3_mprintf("%s", z); @@ -721,14 +744,14 @@ static void integrity_check_x( static void *launch_thread_main(void *pArg){ Thread *p = (Thread *)pArg; - return (void *)p->xProc(p->iTid, p->iArg); + return (void *)p->xProc(p->iTid, p->pArg); } static void launch_thread_x( Error *pErr, /* IN/OUT: Error code */ Threadset *pThreads, /* Thread set */ - char *(*xProc)(int, int), /* Proc to run */ - int iArg /* Argument passed to thread proc */ + char *(*xProc)(int, void*), /* Proc to run */ + void *pArg /* Argument passed to thread proc */ ){ if( pErr->rc==SQLITE_OK ){ int iTid = ++pThreads->iMaxTid; @@ -738,7 +761,7 @@ static void launch_thread_x( p = (Thread *)sqlite3_malloc(sizeof(Thread)); memset(p, 0, sizeof(Thread)); p->iTid = iTid; - p->iArg = iArg; + p->pArg = pArg; p->xProc = xProc; rc = pthread_create(&p->tid, NULL, launch_thread_main, (void *)p); @@ -767,6 +790,7 @@ static void join_all_threads_x( if( pErr->rc==SQLITE_OK ) system_error(pErr, rc); }else{ printf("Thread %d says: %s\n", p->iTid, (ret==0 ? "..." : (char *)ret)); + fflush(stdout); } sqlite3_free(p); } @@ -880,11 +904,6 @@ static int timetostop_x( return ret; } -/* -** The "Set Error Line" macro. -*/ -#define SEL(e) ((e)->iLine = ((e)->rc ? (e)->iLine : __LINE__)) - /************************************************************************* ************************************************************************** @@ -895,7 +914,7 @@ static int timetostop_x( #define WALTHREAD1_NTHREAD 10 #define WALTHREAD3_NTHREAD 6 -static char *walthread1_thread(int iTid, int iArg){ +static char *walthread1_thread(int iTid, void *pArg){ Error err = {0}; /* Error code and message */ Sqlite db = {0}; /* SQLite database connection */ int nIter = 0; /* Iterations so far */ @@ -934,7 +953,7 @@ static char *walthread1_thread(int iTid, int iArg){ return sqlite3_mprintf("%d iterations", nIter); } -static char *walthread1_ckpt_thread(int iTid, int iArg){ +static char *walthread1_ckpt_thread(int iTid, void *pArg){ Error err = {0}; /* Error code and message */ Sqlite db = {0}; /* SQLite database connection */ int nCkpt = 0; /* Checkpoints so far */ @@ -966,6 +985,7 @@ static void walthread1(int nMs){ "INSERT INTO t1 VALUES(randomblob(100));" "INSERT INTO t1 SELECT md5sum(x) FROM t1;" ); + closedb(&err, &db); setstoptime(&err, nMs); for(i=0; i2 ) goto usage; - if( argc==2 ){ - zTest = argv[1]; - nTest = strlen(zTest); - if( zTest[nTest-1]=='*' ){ - nTest--; - bPrefix = 1; - } - } + static char *substArgv[] = { 0, "*", 0 }; + int i, iArg; + int nTestfound = 0; sqlite3_config(SQLITE_CONFIG_MULTITHREAD); - - for(i=0; i=sizeof(aTest)/sizeof(aTest[0]) ) goto usage; + } + for(iArg=1; iArg0 ? 255 : 0); usage: - printf("Usage: %s [testname|testprefix*]\n", argv[0]); + printf("Usage: %s [testname|testprefix*]...\n", argv[0]); printf("Available tests are:\n"); for(i=0; i +#include +#include +#include +#include +#include +#include + +/* +** An instance of the following structure is passed into each worker +** thread. +*/ +typedef struct WorkerInfo WorkerInfo; +struct WorkerInfo { + int tid; /* Thread ID */ + int nWorker; /* Total number of workers */ + unsigned wkrFlags; /* Flags */ + sqlite3 *mainDb; /* Database connection of the main thread */ + sqlite3 *db; /* Database connection of this thread */ + int nErr; /* Number of errors seen by this thread */ + int nTest; /* Number of tests run by this thread */ + char *zMsg; /* Message returned by this thread */ + pthread_t id; /* Thread id */ + pthread_mutex_t *pWrMutex; /* Hold this mutex while writing */ +}; + +/* +** Allowed values for WorkerInfo.wkrFlags +*/ +#define TT4_SERIALIZED 0x0000001 /* The --serialized option is used */ +#define TT4_WAL 0x0000002 /* WAL mode in use */ +#define TT4_TRACE 0x0000004 /* Trace activity */ + + +/* +** Report an OOM error and die if the argument is NULL +*/ +static void check_oom(void *x){ + if( x==0 ){ + fprintf(stderr, "out of memory\n"); + exit(1); + } +} + +/* +** Allocate memory. If the allocation fails, print an error message and +** kill the process. +*/ +static void *safe_malloc(int sz){ + void *x = sqlite3_malloc(sz>0?sz:1); + check_oom(x); + return x; +} + +/* +** Print a trace message for a worker +*/ +static void worker_trace(WorkerInfo *p, const char *zFormat, ...){ + va_list ap; + char *zMsg; + if( (p->wkrFlags & TT4_TRACE)==0 ) return; + va_start(ap, zFormat); + zMsg = sqlite3_vmprintf(zFormat, ap); + check_oom(zMsg); + va_end(ap); + fprintf(stderr, "TRACE(%02d): %s\n", p->tid, zMsg); + sqlite3_free(zMsg); +} + +/* +** Prepare a single SQL query +*/ +static sqlite3_stmt *prep_sql(sqlite3 *db, const char *zFormat, ...){ + va_list ap; + char *zSql; + int rc; + sqlite3_stmt *pStmt = 0; + + va_start(ap, zFormat); + zSql = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + check_oom(zSql); + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + fprintf(stderr, "SQL error (%d,%d): %s\nWhile preparing: [%s]\n", + rc, sqlite3_extended_errcode(db), sqlite3_errmsg(db), zSql); + exit(1); + } + sqlite3_free(zSql); + return pStmt; +} + +/* +** Run a SQL statements. Panic if unable. +*/ +static void run_sql(WorkerInfo *p, const char *zFormat, ...){ + va_list ap; + char *zSql; + int rc; + sqlite3_stmt *pStmt = 0; + int nRetry = 0; + + va_start(ap, zFormat); + zSql = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + check_oom(zSql); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + fprintf(stderr, "SQL error (%d,%d): %s\nWhile preparing: [%s]\n", + rc, sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), zSql); + exit(1); + } + worker_trace(p, "running [%s]", zSql); + while( (rc = sqlite3_step(pStmt))!=SQLITE_DONE ){ + if( (rc&0xff)==SQLITE_BUSY || (rc&0xff)==SQLITE_LOCKED ){ + sqlite3_reset(pStmt); + nRetry++; + if( nRetry<10 ){ + worker_trace(p, "retry %d for [%s]", nRetry, zSql); + sched_yield(); + continue; + }else{ + fprintf(stderr, "Deadlock in thread %d while running [%s]\n", + p->tid, zSql); + exit(1); + } + } + if( rc!=SQLITE_ROW ){ + fprintf(stderr, "SQL error (%d,%d): %s\nWhile running [%s]\n", + rc, sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), zSql); + exit(1); + } + } + sqlite3_free(zSql); + sqlite3_finalize(pStmt); +} + + +/* +** Open the database connection for WorkerInfo. The order in which +** the files are opened is a function of the tid value. +*/ +static void worker_open_connection(WorkerInfo *p, int iCnt){ + char *zFile; + int x; + int rc; + static const unsigned char aOrder[6][3] = { + { 1, 2, 3}, + { 1, 3, 2}, + { 2, 1, 3}, + { 2, 3, 1}, + { 3, 1, 2}, + { 3, 2, 1} + }; + x = (p->tid + iCnt) % 6; + zFile = sqlite3_mprintf("tt4-test%d.db", aOrder[x][0]); + check_oom(zFile); + worker_trace(p, "open %s", zFile); + rc = sqlite3_open_v2(zFile, &p->db, + SQLITE_OPEN_READWRITE|SQLITE_OPEN_SHAREDCACHE, 0); + if( rc!=SQLITE_OK ){ + fprintf(stderr, "sqlite_open_v2(%s) failed on thread %d\n", + zFile, p->tid); + exit(1); + } + sqlite3_free(zFile); + run_sql(p, "PRAGMA read_uncommitted=ON;"); + sqlite3_busy_timeout(p->db, 10000); + run_sql(p, "PRAGMA synchronous=OFF;"); + run_sql(p, "ATTACH 'tt4-test%d.db' AS aux1", aOrder[x][1]); + run_sql(p, "ATTACH 'tt4-test%d.db' AS aux2", aOrder[x][2]); +} + +/* +** Close the worker database connection +*/ +static void worker_close_connection(WorkerInfo *p){ + if( p->db ){ + worker_trace(p, "close"); + sqlite3_close(p->db); + p->db = 0; + } +} + +/* +** Delete all content in the three databases associated with a +** single thread. Make this happen all in a single transaction if +** inTrans is true, or separately for each database if inTrans is +** false. +*/ +static void worker_delete_all_content(WorkerInfo *p, int inTrans){ + if( inTrans ){ + pthread_mutex_lock(p->pWrMutex); + run_sql(p, "BEGIN"); + run_sql(p, "DELETE FROM t1 WHERE tid=%d", p->tid); + run_sql(p, "DELETE FROM t2 WHERE tid=%d", p->tid); + run_sql(p, "DELETE FROM t3 WHERE tid=%d", p->tid); + run_sql(p, "COMMIT"); + pthread_mutex_unlock(p->pWrMutex); + p->nTest++; + }else{ + pthread_mutex_lock(p->pWrMutex); + run_sql(p, "DELETE FROM t1 WHERE tid=%d", p->tid); + pthread_mutex_unlock(p->pWrMutex); + p->nTest++; + pthread_mutex_lock(p->pWrMutex); + run_sql(p, "DELETE FROM t2 WHERE tid=%d", p->tid); + pthread_mutex_unlock(p->pWrMutex); + p->nTest++; + pthread_mutex_lock(p->pWrMutex); + run_sql(p, "DELETE FROM t3 WHERE tid=%d", p->tid); + pthread_mutex_unlock(p->pWrMutex); + p->nTest++; + } +} + +/* +** Create rows mn through mx in table iTab for the given worker +*/ +static void worker_add_content(WorkerInfo *p, int mn, int mx, int iTab){ + char *zTabDef; + switch( iTab ){ + case 1: zTabDef = "t1(tid,sp,a,b,c)"; break; + case 2: zTabDef = "t2(tid,sp,d,e,f)"; break; + case 3: zTabDef = "t3(tid,sp,x,y,z)"; break; + } + pthread_mutex_lock(p->pWrMutex); + run_sql(p, + "WITH RECURSIVE\n" + " c(i) AS (VALUES(%d) UNION ALL SELECT i+1 FROM c WHERE i<%d)\n" + "INSERT INTO %s SELECT %d, zeroblob(3000), i, printf('%%d',i), i FROM c;", + mn, mx, zTabDef, p->tid + ); + pthread_mutex_unlock(p->pWrMutex); + p->nTest++; +} + +/* +** Set an error message on a worker +*/ +static void worker_error(WorkerInfo *p, const char *zFormat, ...){ + va_list ap; + p->nErr++; + sqlite3_free(p->zMsg); + va_start(ap, zFormat); + p->zMsg = sqlite3_vmprintf(zFormat, ap); + va_end(ap); +} + +/* +** Each thread runs the following function. +*/ +static void *worker_thread(void *pArg){ + WorkerInfo *p = (WorkerInfo*)pArg; + int iOuter; + int i; + int rc; + sqlite3_stmt *pStmt; + + printf("worker %d startup\n", p->tid); fflush(stdout); + for(iOuter=1; iOuter<=p->nWorker; iOuter++){ + worker_open_connection(p, iOuter); + for(i=0; i<4; i++){ + worker_add_content(p, i*100+1, (i+1)*100, (p->tid+iOuter)%3 + 1); + worker_add_content(p, i*100+1, (i+1)*100, (p->tid+iOuter+1)%3 + 1); + worker_add_content(p, i*100+1, (i+1)*100, (p->tid+iOuter+2)%3 + 1); + } + + pStmt = prep_sql(p->db, "SELECT count(a) FROM t1 WHERE tid=%d", p->tid); + worker_trace(p, "query [%s]", sqlite3_sql(pStmt)); + rc = sqlite3_step(pStmt); + if( rc!=SQLITE_ROW ){ + worker_error(p, "Failed to step: %s", sqlite3_sql(pStmt)); + }else if( sqlite3_column_int(pStmt, 0)!=400 ){ + worker_error(p, "Wrong result: %d", sqlite3_column_int(pStmt,0)); + } + sqlite3_finalize(pStmt); + if( p->nErr ) break; + + if( ((iOuter+p->tid)%3)==0 ){ + sqlite3_db_release_memory(p->db); + p->nTest++; + } + + pthread_mutex_lock(p->pWrMutex); + run_sql(p, "BEGIN;"); + run_sql(p, "UPDATE t1 SET c=NULL WHERE a=55"); + run_sql(p, "UPDATE t2 SET f=NULL WHERE d=42"); + run_sql(p, "UPDATE t3 SET z=NULL WHERE x=31"); + run_sql(p, "ROLLBACK;"); + p->nTest++; + pthread_mutex_unlock(p->pWrMutex); + + + if( iOuter==p->tid ){ + pthread_mutex_lock(p->pWrMutex); + run_sql(p, "VACUUM"); + pthread_mutex_unlock(p->pWrMutex); + } + + pStmt = prep_sql(p->db, + "SELECT t1.rowid, t2.rowid, t3.rowid" + " FROM t1, t2, t3" + " WHERE t1.tid=%d AND t2.tid=%d AND t3.tid=%d" + " AND t1.a<>t2.d AND t2.d<>t3.x" + " ORDER BY 1, 2, 3" + ,p->tid, p->tid, p->tid); + worker_trace(p, "query [%s]", sqlite3_sql(pStmt)); + for(i=0; inWorker; i++){ + rc = sqlite3_step(pStmt); + if( rc!=SQLITE_ROW ){ + worker_error(p, "Failed to step: %s", sqlite3_sql(pStmt)); + break; + } + sched_yield(); + } + sqlite3_finalize(pStmt); + if( p->nErr ) break; + + worker_delete_all_content(p, (p->tid+iOuter)%2); + worker_close_connection(p); + p->db = 0; + } + worker_close_connection(p); + printf("worker %d finished\n", p->tid); fflush(stdout); + return 0; +} + +int main(int argc, char **argv){ + int nWorker = 0; /* Number of worker threads */ + int i; /* Loop counter */ + WorkerInfo *aInfo; /* Information for each worker */ + unsigned wkrFlags = 0; /* Default worker flags */ + int nErr = 0; /* Number of errors */ + int nTest = 0; /* Number of tests */ + int rc; /* Return code */ + sqlite3 *db = 0; /* Main database connection */ + pthread_mutex_t wrMutex; /* The write serialization mutex */ + WorkerInfo infoTop; /* WorkerInfo for the main thread */ + WorkerInfo *p; /* Pointer to infoTop */ + + sqlite3_config(SQLITE_CONFIG_MULTITHREAD); + for(i=1; i='1' && z[0]<='9' && nWorker==0 ){ + nWorker = atoi(z); + if( nWorker<2 ){ + fprintf(stderr, "minimum of 2 threads\n"); + exit(1); + } + }else{ + fprintf(stderr, "extra command-line argument: \"%s\"\n", argv[i]); + exit(1); + } + } + if( nWorker==0 ){ + fprintf(stderr, + "usage: %s ?OPTIONS? N\n" + "N is the number of threads and must be at least 2.\n" + "Options:\n" + " --serialized\n" + " --multithread\n" + " --wal\n" + " --trace\n" + ,argv[0] + ); + exit(1); + } + if( !sqlite3_threadsafe() ){ + fprintf(stderr, "requires a threadsafe build of SQLite\n"); + exit(1); + } + sqlite3_initialize(); + sqlite3_enable_shared_cache(1); + pthread_mutex_init(&wrMutex, 0); + + /* Initialize the test database files */ + (void)unlink("tt4-test1.db"); + (void)unlink("tt4-test2.db"); + (void)unlink("tt4-test3.db"); + rc = sqlite3_open("tt4-test1.db", &db); + if( rc!=SQLITE_OK ){ + fprintf(stderr, "Unable to open test database: tt4-test2.db\n"); + exit(1); + } + memset(&infoTop, 0, sizeof(infoTop)); + infoTop.db = db; + infoTop.wkrFlags = wkrFlags; + p = &infoTop; + if( wkrFlags & TT4_WAL ){ + run_sql(p, "PRAGMA journal_mode=WAL"); + } + run_sql(p, "PRAGMA synchronous=OFF"); + run_sql(p, "CREATE TABLE IF NOT EXISTS t1(tid INTEGER, sp, a, b, c)"); + run_sql(p, "CREATE INDEX t1tid ON t1(tid)"); + run_sql(p, "CREATE INDEX t1ab ON t1(a,b)"); + run_sql(p, "ATTACH 'tt4-test2.db' AS 'test2'"); + run_sql(p, "CREATE TABLE IF NOT EXISTS test2.t2(tid INTEGER, sp, d, e, f)"); + run_sql(p, "CREATE INDEX test2.t2tid ON t2(tid)"); + run_sql(p, "CREATE INDEX test2.t2de ON t2(d,e)"); + run_sql(p, "ATTACH 'tt4-test3.db' AS 'test3'"); + run_sql(p, "CREATE TABLE IF NOT EXISTS test3.t3(tid INTEGER, sp, x, y, z)"); + run_sql(p, "CREATE INDEX test3.t3tid ON t3(tid)"); + run_sql(p, "CREATE INDEX test3.t3xy ON t3(x,y)"); + aInfo = safe_malloc( sizeof(*aInfo)*nWorker ); + memset(aInfo, 0, sizeof(*aInfo)*nWorker); + for(i=0; i=0} @@ -220,4 +221,36 @@ ifcapable compound { } {2} } +reset_db +do_execsql_test 4.1 { + CREATE TABLE t1(a, b); + CREATE TABLE log(x); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + CREATE VIEW v1 AS SELECT a, b FROM t1; + + CREATE TRIGGER tr1 INSTEAD OF DELETE ON v1 BEGIN + INSERT INTO log VALUES('delete'); + END; + + CREATE TRIGGER tr2 INSTEAD OF UPDATE ON v1 BEGIN + INSERT INTO log VALUES('update'); + END; + + CREATE TRIGGER tr3 INSTEAD OF INSERT ON v1 BEGIN + INSERT INTO log VALUES('insert'); + END; +} + +do_execsql_test 4.2 { + DELETE FROM v1 WHERE rowid=1; +} {} + +do_execsql_test 4.3 { + UPDATE v1 SET a=b WHERE rowid=2; +} {} + + + + finish_test diff --git a/test/tt3_checkpoint.c b/test/tt3_checkpoint.c index 3c28f0d3a5..060a698211 100644 --- a/test/tt3_checkpoint.c +++ b/test/tt3_checkpoint.c @@ -1,5 +1,5 @@ /* -** 2001 September 15 +** 2011-02-02 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -66,7 +66,7 @@ static int checkpoint_starvation_walhook( return SQLITE_OK; } -static char *checkpoint_starvation_reader(int iTid, int iArg){ +static char *checkpoint_starvation_reader(int iTid, void *pArg){ Error err = {0}; Sqlite db = {0}; @@ -146,5 +146,3 @@ static void checkpoint_starvation_2(int nMs){ } print_and_free_err(&err); } - - diff --git a/test/tt3_index.c b/test/tt3_index.c new file mode 100644 index 0000000000..62f06ade4a --- /dev/null +++ b/test/tt3_index.c @@ -0,0 +1,74 @@ +/* +** 2014 December 9 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** create_drop_index_1 +*/ + + +static char *create_drop_index_thread(int iTid, void *pArg){ + Error err = {0}; /* Error code and message */ + Sqlite db = {0}; /* SQLite database connection */ + + while( !timetostop(&err) ){ + opendb(&err, &db, "test.db", 0); + + sql_script(&err, &db, + "DROP INDEX IF EXISTS i1;" + "DROP INDEX IF EXISTS i2;" + "DROP INDEX IF EXISTS i3;" + "DROP INDEX IF EXISTS i4;" + + "CREATE INDEX IF NOT EXISTS i1 ON t11(a);" + "CREATE INDEX IF NOT EXISTS i2 ON t11(b);" + "CREATE INDEX IF NOT EXISTS i3 ON t11(c);" + "CREATE INDEX IF NOT EXISTS i4 ON t11(d);" + + "SELECT * FROM t11 ORDER BY a;" + "SELECT * FROM t11 ORDER BY b;" + "SELECT * FROM t11 ORDER BY c;" + "SELECT * FROM t11 ORDER BY d;" + ); + clear_error(&err, SQLITE_LOCKED); + + closedb(&err, &db); + } + + print_and_free_err(&err); + return sqlite3_mprintf("ok"); +} + +static void create_drop_index_1(int nMs){ + Error err = {0}; + Sqlite db = {0}; + Threadset threads = {0}; + + opendb(&err, &db, "test.db", 1); + sql_script(&err, &db, + "CREATE TABLE t11(a, b, c, d);" + "WITH data(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM data WHERE x<100) " + "INSERT INTO t11 SELECT x,x,x,x FROM data;" + ); + closedb(&err, &db); + + setstoptime(&err, nMs); + + sqlite3_enable_shared_cache(1); + launch_thread(&err, &threads, create_drop_index_thread, 0); + launch_thread(&err, &threads, create_drop_index_thread, 0); + launch_thread(&err, &threads, create_drop_index_thread, 0); + launch_thread(&err, &threads, create_drop_index_thread, 0); + launch_thread(&err, &threads, create_drop_index_thread, 0); + + join_all_threads(&err, &threads); + sqlite3_enable_shared_cache(0); + print_and_free_err(&err); +} diff --git a/test/tt3_lookaside1.c b/test/tt3_lookaside1.c new file mode 100644 index 0000000000..e785e7998b --- /dev/null +++ b/test/tt3_lookaside1.c @@ -0,0 +1,99 @@ +/* +** 2014 December 9 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** lookaside1 +*/ + +/* +** The test in this file attempts to expose a specific race condition +** that is suspected to exist at time of writing. +*/ + +static char *lookaside1_thread_reader(int iTid, void *pArg){ + Error err = {0}; /* Error code and message */ + Sqlite db = {0}; /* SQLite database connection */ + + opendb(&err, &db, "test.db", 0); + + while( !timetostop(&err) ){ + sqlite3_stmt *pStmt = 0; + int rc; + + sqlite3_prepare_v2(db.db, "SELECT 1 FROM t1", -1, &pStmt, 0); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + execsql(&err, &db, "SELECT length(x||y||z) FROM t2"); + } + rc = sqlite3_finalize(pStmt); + if( err.rc==SQLITE_OK && rc!=SQLITE_OK ){ + sqlite_error(&err, &db, "finalize"); + } + } + + closedb(&err, &db); + print_and_free_err(&err); + return sqlite3_mprintf("ok"); +} + +static char *lookaside1_thread_writer(int iTid, void *pArg){ + Error err = {0}; /* Error code and message */ + Sqlite db = {0}; /* SQLite database connection */ + + opendb(&err, &db, "test.db", 0); + + do{ + sql_script(&err, &db, + "BEGIN;" + "UPDATE t3 SET i=i+1 WHERE x=1;" + "ROLLBACK;" + ); + }while( !timetostop(&err) ); + + closedb(&err, &db); + print_and_free_err(&err); + return sqlite3_mprintf("ok"); +} + + +static void lookaside1(int nMs){ + Error err = {0}; + Sqlite db = {0}; + Threadset threads = {0}; + + opendb(&err, &db, "test.db", 1); + sql_script(&err, &db, + "CREATE TABLE t1(x PRIMARY KEY) WITHOUT ROWID;" + "WITH data(x,y) AS (" + " SELECT 1, quote(randomblob(750)) UNION ALL " + " SELECT x*2, y||y FROM data WHERE x<5) " + "INSERT INTO t1 SELECT y FROM data;" + + "CREATE TABLE t3(x PRIMARY KEY,i) WITHOUT ROWID;" + "INSERT INTO t3 VALUES(1, 1);" + + "CREATE TABLE t2(x,y,z);" + "INSERT INTO t2 VALUES(randomblob(50), randomblob(50), randomblob(50));" + ); + closedb(&err, &db); + + setstoptime(&err, nMs); + + sqlite3_enable_shared_cache(1); + launch_thread(&err, &threads, lookaside1_thread_reader, 0); + launch_thread(&err, &threads, lookaside1_thread_reader, 0); + launch_thread(&err, &threads, lookaside1_thread_reader, 0); + launch_thread(&err, &threads, lookaside1_thread_reader, 0); + launch_thread(&err, &threads, lookaside1_thread_reader, 0); + launch_thread(&err, &threads, lookaside1_thread_writer, 0); + join_all_threads(&err, &threads); + sqlite3_enable_shared_cache(0); + print_and_free_err(&err); +} diff --git a/test/tt3_stress.c b/test/tt3_stress.c new file mode 100644 index 0000000000..cdfab9c09c --- /dev/null +++ b/test/tt3_stress.c @@ -0,0 +1,368 @@ +/* +** 2014 December 9 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** +*/ + + +/* +** Thread 1. CREATE and DROP a table. +*/ +static char *stress_thread_1(int iTid, void *pArg){ + Error err = {0}; /* Error code and message */ + Sqlite db = {0}; /* SQLite database connection */ + + opendb(&err, &db, "test.db", 0); + while( !timetostop(&err) ){ + sql_script(&err, &db, "CREATE TABLE IF NOT EXISTS t1(a PRIMARY KEY, b)"); + clear_error(&err, SQLITE_LOCKED); + sql_script(&err, &db, "DROP TABLE IF EXISTS t1"); + clear_error(&err, SQLITE_LOCKED); + } + closedb(&err, &db); + print_and_free_err(&err); + return sqlite3_mprintf("ok"); +} + +/* +** Thread 2. Open and close database connections. +*/ +static char *stress_thread_2(int iTid, void *pArg){ + Error err = {0}; /* Error code and message */ + Sqlite db = {0}; /* SQLite database connection */ + while( !timetostop(&err) ){ + opendb(&err, &db, "test.db", 0); + sql_script(&err, &db, "SELECT * FROM sqlite_master;"); + clear_error(&err, SQLITE_LOCKED); + closedb(&err, &db); + } + print_and_free_err(&err); + return sqlite3_mprintf("ok"); +} + +/* +** Thread 3. Attempt many small SELECT statements. +*/ +static char *stress_thread_3(int iTid, void *pArg){ + Error err = {0}; /* Error code and message */ + Sqlite db = {0}; /* SQLite database connection */ + + int i1 = 0; + int i2 = 0; + + opendb(&err, &db, "test.db", 0); + while( !timetostop(&err) ){ + sql_script(&err, &db, "SELECT * FROM t1 ORDER BY a;"); + i1++; + if( err.rc ) i2++; + clear_error(&err, SQLITE_LOCKED); + clear_error(&err, SQLITE_ERROR); + } + closedb(&err, &db); + print_and_free_err(&err); + return sqlite3_mprintf("read t1 %d/%d attempts", i2, i1); +} + +/* +** Thread 5. Attempt INSERT statements. +*/ +static char *stress_thread_4(int iTid, void *pArg){ + Error err = {0}; /* Error code and message */ + Sqlite db = {0}; /* SQLite database connection */ + int i1 = 0; + int i2 = 0; + int iArg = PTR2INT(pArg); + + opendb(&err, &db, "test.db", 0); + while( !timetostop(&err) ){ + if( iArg ){ + closedb(&err, &db); + opendb(&err, &db, "test.db", 0); + } + sql_script(&err, &db, + "WITH loop(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM loop LIMIT 200) " + "INSERT INTO t1 VALUES(randomblob(60), randomblob(60));" + ); + i1++; + if( err.rc ) i2++; + clear_error(&err, SQLITE_LOCKED); + clear_error(&err, SQLITE_ERROR); + } + closedb(&err, &db); + print_and_free_err(&err); + return sqlite3_mprintf("wrote t1 %d/%d attempts", i2, i1); +} + +/* +** Thread 6. Attempt DELETE operations. +*/ +static char *stress_thread_5(int iTid, void *pArg){ + Error err = {0}; /* Error code and message */ + Sqlite db = {0}; /* SQLite database connection */ + int iArg = PTR2INT(pArg); + + int i1 = 0; + int i2 = 0; + + opendb(&err, &db, "test.db", 0); + while( !timetostop(&err) ){ + i64 i = (i1 % 4); + if( iArg ){ + closedb(&err, &db); + opendb(&err, &db, "test.db", 0); + } + execsql(&err, &db, "DELETE FROM t1 WHERE (rowid % 4)==:i", &i); + i1++; + if( err.rc ) i2++; + clear_error(&err, SQLITE_LOCKED); + } + closedb(&err, &db); + print_and_free_err(&err); + return sqlite3_mprintf("deleted from t1 %d/%d attempts", i2, i1); +} + + +static void stress1(int nMs){ + Error err = {0}; + Threadset threads = {0}; + + setstoptime(&err, nMs); + sqlite3_enable_shared_cache(1); + + launch_thread(&err, &threads, stress_thread_1, 0); + launch_thread(&err, &threads, stress_thread_1, 0); + + launch_thread(&err, &threads, stress_thread_2, 0); + launch_thread(&err, &threads, stress_thread_2, 0); + + launch_thread(&err, &threads, stress_thread_3, 0); + launch_thread(&err, &threads, stress_thread_3, 0); + + launch_thread(&err, &threads, stress_thread_4, 0); + launch_thread(&err, &threads, stress_thread_4, 0); + + launch_thread(&err, &threads, stress_thread_5, 0); + launch_thread(&err, &threads, stress_thread_5, (void*)1); + + join_all_threads(&err, &threads); + sqlite3_enable_shared_cache(0); + + print_and_free_err(&err); +} + +/************************************************************************** +*************************************************************************** +** Start of test case "stress2" +*/ + + + +/* +** 1. CREATE TABLE statements. +** 2. DROP TABLE statements. +** 3. Small SELECT statements. +** 4. Big SELECT statements. +** 5. Small INSERT statements. +** 6. Big INSERT statements. +** 7. Small UPDATE statements. +** 8. Big UPDATE statements. +** 9. Small DELETE statements. +** 10. Big DELETE statements. +** 11. VACUUM. +** 14. Integrity-check. +** 17. Switch the journal mode from delete to wal and back again. +** 19. Open and close database connections rapidly. +*/ + +#define STRESS2_TABCNT 5 /* count1 in SDS test */ + +#define STRESS2_COUNT2 200 /* count2 in SDS test */ +#define STRESS2_COUNT3 57 /* count2 in SDS test */ + +static void stress2_workload1(Error *pErr, Sqlite *pDb, int i){ + int iTab = (i % (STRESS2_TABCNT-1)) + 1; + sql_script_printf(pErr, pDb, + "CREATE TABLE IF NOT EXISTS t%d(x PRIMARY KEY, y, z);", iTab + ); +} + +static void stress2_workload2(Error *pErr, Sqlite *pDb, int i){ + int iTab = (i % (STRESS2_TABCNT-1)) + 1; + sql_script_printf(pErr, pDb, "DROP TABLE IF EXISTS t%d;", iTab); +} + +static void stress2_workload3(Error *pErr, Sqlite *pDb, int i){ + sql_script(pErr, pDb, "SELECT * FROM t0 WHERE z = 'small'"); +} + +static void stress2_workload4(Error *pErr, Sqlite *pDb, int i){ + sql_script(pErr, pDb, "SELECT * FROM t0 WHERE z = 'big'"); +} + +static void stress2_workload5(Error *pErr, Sqlite *pDb, int i){ + sql_script(pErr, pDb, + "INSERT INTO t0 VALUES(hex(random()), hex(randomblob(200)), 'small');" + ); +} + +static void stress2_workload6(Error *pErr, Sqlite *pDb, int i){ + sql_script(pErr, pDb, + "INSERT INTO t0 VALUES(hex(random()), hex(randomblob(57)), 'big');" + ); +} + +static void stress2_workload7(Error *pErr, Sqlite *pDb, int i){ + sql_script_printf(pErr, pDb, + "UPDATE t0 SET y = hex(randomblob(200)) " + "WHERE x LIKE hex((%d %% 5)) AND z='small';" + ,i + ); +} +static void stress2_workload8(Error *pErr, Sqlite *pDb, int i){ + sql_script_printf(pErr, pDb, + "UPDATE t0 SET y = hex(randomblob(57)) " + "WHERE x LIKE hex(%d %% 5) AND z='big';" + ,i + ); +} + +static void stress2_workload9(Error *pErr, Sqlite *pDb, int i){ + sql_script_printf(pErr, pDb, + "DELETE FROM t0 WHERE x LIKE hex(%d %% 5) AND z='small';", i + ); +} +static void stress2_workload10(Error *pErr, Sqlite *pDb, int i){ + sql_script_printf(pErr, pDb, + "DELETE FROM t0 WHERE x LIKE hex(%d %% 5) AND z='big';", i + ); +} + +static void stress2_workload11(Error *pErr, Sqlite *pDb, int i){ + sql_script(pErr, pDb, "VACUUM"); +} + +static void stress2_workload14(Error *pErr, Sqlite *pDb, int i){ + sql_script(pErr, pDb, "PRAGMA integrity_check"); +} + +static void stress2_workload17(Error *pErr, Sqlite *pDb, int i){ + sql_script_printf(pErr, pDb, + "PRAGMA journal_mode = %q", (i%2) ? "delete" : "wal" + ); +} + +static char *stress2_workload19(int iTid, void *pArg){ + Error err = {0}; /* Error code and message */ + Sqlite db = {0}; /* SQLite database connection */ + const char *zDb = (const char*)pArg; + while( !timetostop(&err) ){ + opendb(&err, &db, zDb, 0); + sql_script(&err, &db, "SELECT * FROM sqlite_master;"); + clear_error(&err, SQLITE_LOCKED); + closedb(&err, &db); + } + print_and_free_err(&err); + return sqlite3_mprintf("ok"); +} + + +typedef struct Stress2Ctx Stress2Ctx; +struct Stress2Ctx { + const char *zDb; + void (*xProc)(Error*, Sqlite*, int); +}; + +static char *stress2_thread_wrapper(int iTid, void *pArg){ + Stress2Ctx *pCtx = (Stress2Ctx*)pArg; + Error err = {0}; /* Error code and message */ + Sqlite db = {0}; /* SQLite database connection */ + int i1 = 0; + int i2 = 0; + + while( !timetostop(&err) ){ + int cnt; + opendb(&err, &db, pCtx->zDb, 0); + for(cnt=0; err.rc==SQLITE_OK && cntxProc(&err, &db, i1); + i2 += (err.rc==SQLITE_OK); + clear_error(&err, SQLITE_LOCKED); + i1++; + } + closedb(&err, &db); + } + + print_and_free_err(&err); + return sqlite3_mprintf("ok %d/%d", i2, i1); +} + +static void stress2_launch_thread_loop( + Error *pErr, /* IN/OUT: Error code */ + Threadset *pThreads, /* Thread set */ + const char *zDb, /* Database name */ + void (*x)(Error*,Sqlite*,int) /* Run this until error or timeout */ +){ + Stress2Ctx *pCtx = sqlite3_malloc(sizeof(Stress2Ctx)); + pCtx->zDb = zDb; + pCtx->xProc = x; + launch_thread(pErr, pThreads, stress2_thread_wrapper, (void*)pCtx); +} + +static void stress2(int nMs){ + struct Stress2Task { + void (*x)(Error*,Sqlite*,int); + } aTask[] = { + { stress2_workload1 }, + { stress2_workload2 }, + { stress2_workload3 }, + { stress2_workload4 }, + { stress2_workload5 }, + { stress2_workload6 }, + { stress2_workload7 }, + { stress2_workload8 }, + { stress2_workload9 }, + { stress2_workload10 }, + { stress2_workload11 }, + { stress2_workload14 }, + { stress2_workload17 }, + }; + const char *zDb = "test.db"; + + int i; + Error err = {0}; + Sqlite db = {0}; + Threadset threads = {0}; + + /* To make sure the db file is empty before commencing */ + opendb(&err, &db, zDb, 1); + sql_script(&err, &db, + "CREATE TABLE IF NOT EXISTS t0(x PRIMARY KEY, y, z);" + "CREATE INDEX IF NOT EXISTS i0 ON t0(y);" + ); + closedb(&err, &db); + + setstoptime(&err, nMs); + sqlite3_enable_shared_cache(1); + + for(i=0; i-1} { return SQLITE_OK } diff --git a/test/wal5.test b/test/wal5.test index 68750f1479..8c1ec8bcc7 100644 --- a/test/wal5.test +++ b/test/wal5.test @@ -55,7 +55,8 @@ foreach {testprefix do_wal_checkpoint} { if {[lsearch {-mode -db} $key]<0} { error "unknown switch: $key" } } - if {$a(-mode)!="restart" && $a(-mode)!="full"} { set a(-mode) passive } + set vals {restart full truncate} + if {[lsearch -exact $vals $a(-mode)]<0} { set a(-mode) passive } set cmd [list sqlite3_wal_checkpoint_v2 $dbhandle $a(-mode)] if {[info exists a(-db)]} { lappend sql $a(-db) } @@ -278,6 +279,11 @@ foreach {testprefix do_wal_checkpoint} { 9 RESTART 2 {1 4 3} 2 10 RESTART 3 {1 4 4} 3 + 11 TRUNCATE - {0 0 0} 3 + 12 TRUNCATE 1 {1 3 3} 1 + 13 TRUNCATE 2 {1 4 3} 2 + 14 TRUNCATE 3 {1 4 4} 3 + } { do_multiclient_test tn { setup_and_attach_aux @@ -348,6 +354,42 @@ foreach {testprefix do_wal_checkpoint} { do_test 3.$tn.6 { code3 { do_wal_checkpoint db3 } } {0 0 0} } + + # Test SQLITE_CHECKPOINT_TRUNCATE. + # + do_multiclient_test tn { + + code1 $do_wal_checkpoint + code2 $do_wal_checkpoint + code3 $do_wal_checkpoint + + do_test 4.$tn.1 { + sql1 { + PRAGMA page_size = 1024; + PRAGMA auto_vacuum = 0; + PRAGMA journal_mode = WAL; + PRAGMA synchronous = normal; + CREATE TABLE t1(x, y); + CREATE INDEX i1 ON t1(x, y); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + } + file size test.db-wal + } [wal_file_size 8 1024] + + do_test 4.$tn.2 { do_wal_checkpoint db -mode truncate } {0 0 0} + do_test 4.$tn.3 { file size test.db-wal } 0 + + do_test 4.$tn.4 { + sql2 { SELECT * FROM t1 } + } {1 2 3 4} + + do_test 4.$tn.5 { + sql2 { INSERT INTO t1 VALUES('a', 'b') } + file size test.db-wal + } [wal_file_size 2 1024] + + } } diff --git a/test/where2.test b/test/where2.test index d9b2b23de9..367eb0dfea 100644 --- a/test/where2.test +++ b/test/where2.test @@ -751,5 +751,13 @@ do_execsql_test where2-12.1 { } {/.*SEARCH TABLE t12 AS b .*SEARCH TABLE t12 AS b .*/} } +# Verify that all necessary OP_OpenRead opcodes occur in the OR optimization. +# +do_execsql_test where2-13.1 { + CREATE TABLE t13(a,b); + CREATE INDEX t13a ON t13(a); + INSERT INTO t13 VALUES(4,5); + SELECT * FROM t13 WHERE (1=2 AND a=3) OR a=4; +} {4 5} finish_test diff --git a/test/where9.test b/test/where9.test index 1c32ff82f7..d073074d43 100644 --- a/test/where9.test +++ b/test/where9.test @@ -781,7 +781,12 @@ do_test where9-6.8.2 { OR (b NOT NULL AND c NOT NULL AND d IS NULL) } } {1 {no query solution}} + +set solution_possible 0 ifcapable stat4||stat3 { + if {[permutation] != "no_optimization"} { set solution_possible 1 } +} +if $solution_possible { # When STAT3 is enabled, the "b NOT NULL" terms get translated # into b>NULL, which can be satified by the index t1b. It is a very # expensive way to do the query, but it works, and so a solution is possible. diff --git a/test/whereJ.test b/test/whereJ.test index 3cdeae890b..48924d0fcf 100644 --- a/test/whereJ.test +++ b/test/whereJ.test @@ -9,7 +9,7 @@ # #*********************************************************************** # -# This file implements a single regression test for a complex +# This file implements regression tests for a complex # query planning case. # @@ -328,4 +328,351 @@ do_execsql_test whereJ-1.4 { GROUP BY aid; } {/B-TREE/} +############################################################################ +# Ensure that the sorting cost does not swamp the loop costs and cause +# distinctions between individual loop costs to get lost, and hence for +# sub-optimal loops to be chosen. +# +do_execsql_test whereJ-2.1 { + CREATE TABLE tab( + id INTEGER PRIMARY KEY, + minChild INTEGER REFERENCES t1, + maxChild INTEGER REFERENCES t1, + x INTEGER + ); + EXPLAIN QUERY PLAN + SELECT t4.x + FROM tab AS t0, tab AS t1, tab AS t2, tab AS t3, tab AS t4 + WHERE t0.id=0 + AND t1.id BETWEEN t0.minChild AND t0.maxChild + AND t2.id BETWEEN t1.minChild AND t1.maxChild + AND t3.id BETWEEN t2.minChild AND t2.maxChild + AND t4.id BETWEEN t3.minChild AND t3.maxChild + ORDER BY t4.x; +} {~/SCAN/} +do_execsql_test whereJ-2.2 { + EXPLAIN QUERY PLAN + SELECT t4.x + FROM tab AS t0a, tab AS t0b, + tab AS t1a, tab AS t1b, + tab AS t2a, tab AS t2b, + tab AS t3a, tab AS t3b, + tab AS t4 + WHERE 1 + AND t0a.id=1 + AND t1a.id BETWEEN t0a.minChild AND t0a.maxChild + AND t2a.id BETWEEN t1a.minChild AND t1a.maxChild + AND t3a.id BETWEEN t2a.minChild AND t2a.maxChild + AND t0b.id=2 + AND t1b.id BETWEEN t0b.minChild AND t0b.maxChild + AND t2b.id BETWEEN t1b.minChild AND t1b.maxChild + AND t3b.id BETWEEN t2b.minChild AND t2b.maxChild + AND t4.id BETWEEN t3a.minChild AND t3b.maxChild + ORDER BY t4.x; +} {~/SCAN/} + +############################################################################ + +# Create and populate table. +do_execsql_test 3.1 { CREATE TABLE t1(a, b, c) } +for {set i 0} {$i < 32} {incr i 2} { + for {set x 0} {$x < 100} {incr x} { + execsql { INSERT INTO t1 VALUES($i, $x, $c) } + incr c + } + execsql { INSERT INTO t1 VALUES($i+1, 5, $c) } + incr c +} + +do_execsql_test 3.2 { + SELECT a, count(*) FROM t1 GROUP BY a HAVING a < 8; +} { + 0 100 1 1 2 100 3 1 4 100 5 1 6 100 7 1 +} + +do_execsql_test 3.3 { + CREATE INDEX idx_ab ON t1(a, b); + CREATE INDEX idx_c ON t1(c); + ANALYZE; +} {} + +# This one should use index "idx_c". +do_eqp_test 3.4 { + SELECT * FROM t1 WHERE + a = 4 AND b BETWEEN 20 AND 80 -- Matches 80 rows + AND + c BETWEEN 150 AND 160 -- Matches 10 rows +} { + 0 0 0 {SEARCH TABLE t1 USING INDEX idx_c (c>? AND c? AND b=5 AND d<=5) + AND ((e>=7 AND e<=7) OR (f>=8 AND f<=8)) + AND g>0; + }] +} {~/ANY/} +do_test 5.2 {set res} {/USING INDEX t1abe/} +do_test 5.3 {set res} {/USING INDEX t1abf/} + + + finish_test diff --git a/test/without_rowid1.test b/test/without_rowid1.test index bdd03c57bb..9d7a6430fe 100644 --- a/test/without_rowid1.test +++ b/test/without_rowid1.test @@ -213,5 +213,69 @@ do_execsql_test 4.1 { do_execsql_test 4.2 { SELECT t42.rowid FROM t42, t41; } {1} + + +#-------------------------------------------------------------------------- +# The following tests verify that the trailing PK fields added to each +# entry in an index on a WITHOUT ROWID table are used correctly. +# +do_execsql_test 5.0 { + CREATE TABLE t45(a PRIMARY KEY, b, c) WITHOUT ROWID; + CREATE INDEX i45 ON t45(b); + + INSERT INTO t45 VALUES(2, 'one', 'x'); + INSERT INTO t45 VALUES(4, 'one', 'x'); + INSERT INTO t45 VALUES(6, 'one', 'x'); + INSERT INTO t45 VALUES(8, 'one', 'x'); + INSERT INTO t45 VALUES(10, 'one', 'x'); + + INSERT INTO t45 VALUES(1, 'two', 'x'); + INSERT INTO t45 VALUES(3, 'two', 'x'); + INSERT INTO t45 VALUES(5, 'two', 'x'); + INSERT INTO t45 VALUES(7, 'two', 'x'); + INSERT INTO t45 VALUES(9, 'two', 'x'); +} + +do_eqp_test 5.1 { + SELECT * FROM t45 WHERE b=? AND a>? +} {/*USING INDEX i45 (b=? AND a>?)*/} + +do_execsql_test 5.2 { + SELECT * FROM t45 WHERE b='two' AND a>4 +} {5 two x 7 two x 9 two x} + +do_execsql_test 5.3 { + SELECT * FROM t45 WHERE b='one' AND a<8 +} { 2 one x 4 one x 6 one x } + +do_execsql_test 5.4 { + CREATE TABLE t46(a, b, c, d, PRIMARY KEY(a, b)) WITHOUT ROWID; + WITH r(x) AS ( + SELECT 1 UNION ALL SELECT x+1 FROM r WHERE x<100 + ) + INSERT INTO t46 SELECT x / 20, x % 20, x % 10, x FROM r; +} + +set queries { + 1 2 "c = 5 AND a = 1" {/*i46 (c=? AND a=?)*/} + 2 6 "c = 4 AND a < 3" {/*i46 (c=? AND a= 3" {/*i46 (c=? AND a>?)*/} + 4 1 "c = 2 AND a = 1 AND b<10" {/*i46 (c=? AND a=? AND b5" {/*i46 (c=? AND a=? AND b>?)*/} +} + +foreach {tn cnt where eqp} $queries { + do_execsql_test 5.5.$tn.1 "SELECT count(*) FROM t46 WHERE $where" $cnt +} + +do_execsql_test 5.6 { + CREATE INDEX i46 ON t46(c); +} + +foreach {tn cnt where eqp} $queries { + do_execsql_test 5.7.$tn.1 "SELECT count(*) FROM t46 WHERE $where" $cnt + do_eqp_test 5.7.$tn.2 "SELECT count(*) FROM t46 WHERE $where" $eqp +} + finish_test diff --git a/test/without_rowid3.test b/test/without_rowid3.test index c4c2d6f483..a0dc76d3f9 100644 --- a/test/without_rowid3.test +++ b/test/without_rowid3.test @@ -1621,7 +1621,7 @@ ifcapable auth { } } {} - proc auth {args} {eval lappend ::authargs $args ; return SQLITE_OK} + proc auth {args} {eval lappend ::authargs [lrange $args 0 4]; return SQLITE_OK} db auth auth # An insert on the parent table must read the child key of any deferred diff --git a/test/without_rowid5.test b/test/without_rowid5.test index 45e047befe..d163d9c1bc 100644 --- a/test/without_rowid5.test +++ b/test/without_rowid5.test @@ -185,8 +185,7 @@ do_execsql_test without_rowid5-5.9 { # EVIDENCE-OF: R-12643-30541 The incremental blob I/O mechanism does not # work for WITHOUT ROWID tables. # -# EVIDENCE-OF: R-25760-33257 The sqlite3_blob_open() interface will fail -# for a WITHOUT ROWID table. +# EVIDENCE-OF: R-40134-30296 Table zTable is a WITHOUT ROWID table # do_execsql_test without_rowid5-6.1 { CREATE TABLE b1(a INTEGER PRIMARY KEY, b BLOB) WITHOUT ROWID; diff --git a/test/without_rowid6.test b/test/without_rowid6.test new file mode 100644 index 0000000000..e827ccab90 --- /dev/null +++ b/test/without_rowid6.test @@ -0,0 +1,41 @@ +# 2014-12-28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Verify that WITHOUT ROWID tables work correctly when the PRIMARY KEY +# has redundant columns. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +do_execsql_test without_rowid6-100 { + CREATE TABLE t1(a,b,c,d,e, PRIMARY KEY(a,b,c,a,b,c,d,a,b,c)) WITHOUT ROWID; + CREATE INDEX t1a ON t1(b, b); + WITH RECURSIVE + c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<1000) + INSERT INTO t1(a,b,c,d,e) SELECT i, i+1000, printf('x%dy',i), 0, 0 FROM c; + ANALYZE; +} {} +do_execsql_test without_rowid6-110 { + SELECT c FROM t1 WHERE a=123; +} {x123y} +do_execsql_test without_rowid6-120 { + SELECT c FROM t1 WHERE b=1123; +} {x123y} +do_execsql_test without_rowid6-130 { + SELECT c FROM t1 ORDER BY a DESC LIMIT 5; +} {x1000y x999y x998y x997y x996y} +do_execsql_test without_rowid6-140 { + SELECT c FROM t1 ORDER BY b LIMIT 5; +} {x1y x2y x3y x4y x5y} + + +finish_test diff --git a/tool/mkautoconfamal.sh b/tool/mkautoconfamal.sh index c13f7c999a..4829277234 100644 --- a/tool/mkautoconfamal.sh +++ b/tool/mkautoconfamal.sh @@ -62,7 +62,7 @@ mkdir -p tea/generic echo "#ifdef USE_SYSTEM_SQLITE" > tea/generic/tclsqlite3.c echo "# include " >> tea/generic/tclsqlite3.c echo "#else" >> tea/generic/tclsqlite3.c -echo "#include \"../../sqlite3.c\"" >> tea/generic/tclsqlite3.c +echo "#include \"sqlite3.c\"" >> tea/generic/tclsqlite3.c echo "#endif" >> tea/generic/tclsqlite3.c cat $TOP/src/tclsqlite.c >> tea/generic/tclsqlite3.c diff --git a/tool/mkpragmatab.tcl b/tool/mkpragmatab.tcl index 28a1e468b8..21503d16ee 100644 --- a/tool/mkpragmatab.tcl +++ b/tool/mkpragmatab.tcl @@ -246,18 +246,29 @@ set pragma_def { NAME: schema_version TYPE: HEADER_VALUE + ARG: BTREE_SCHEMA_VERSION IF: !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) NAME: user_version TYPE: HEADER_VALUE + ARG: BTREE_USER_VERSION + IF: !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) + + NAME: data_version + TYPE: HEADER_VALUE + ARG: BTREE_DATA_VERSION + FLAG: ReadOnly IF: !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) NAME: freelist_count TYPE: HEADER_VALUE + ARG: BTREE_FREE_PAGE_COUNT + FLAG: ReadOnly IF: !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) NAME: application_id TYPE: HEADER_VALUE + ARG: BTREE_APPLICATION_ID IF: !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) NAME: compile_options @@ -294,6 +305,8 @@ set pragma_def { IF: defined(SQLITE_HAS_CODEC) || defined(SQLITE_ENABLE_CEROD) NAME: soft_heap_limit + + NAME: threads } fconfigure stdout -translation lf set name {} diff --git a/tool/mksqlite3c-noext.tcl b/tool/mksqlite3c-noext.tcl index ecb9cb0439..f54b347be1 100644 --- a/tool/mksqlite3c-noext.tcl +++ b/tool/mksqlite3c-noext.tcl @@ -239,6 +239,7 @@ foreach file { malloc.c printf.c random.c + threads.c utf.c util.c hash.c diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl index 073d67a5dd..bf5b1748c2 100644 --- a/tool/mksqlite3c.tcl +++ b/tool/mksqlite3c.tcl @@ -214,7 +214,7 @@ proc copy_file {filename} { } } elseif {[regexp {^(SQLITE_EXTERN )?void \(\*sqlite3IoTrace\)} $line]} { regsub {^SQLITE_EXTERN } $line {} line - puts $out "SQLITE_PRIVATE $line" + puts $out $line } elseif {[regexp {^void \(\*sqlite3Os} $line]} { puts $out "SQLITE_PRIVATE $line" } else { @@ -255,6 +255,7 @@ foreach file { malloc.c printf.c random.c + threads.c utf.c util.c hash.c diff --git a/tool/showdb.c b/tool/showdb.c index bf4af444b2..82b8c9f14f 100644 --- a/tool/showdb.c +++ b/tool/showdb.c @@ -9,6 +9,8 @@ #if !defined(_MSC_VER) #include +#else +#include #endif #include @@ -66,7 +68,7 @@ static unsigned char *getContent(int ofst, int nByte){ if( aData==0 ) out_of_memory(); memset(aData, 0, nByte+32); lseek(db, ofst, SEEK_SET); - read(db, aData, nByte); + if( read(db, aData, nByte)=nByte ){ sprintf(&zBuf[i], " "); @@ -149,7 +151,7 @@ static void print_decode_line( sprintf(&zBuf[i], " %02x", aData[ofst+j]); val = val*256 + aData[ofst+j]; } - i += strlen(&zBuf[i]); + i += (int)strlen(&zBuf[i]); } sprintf(&zBuf[i], " %9d", val); printf("%s %s\n", zBuf, zMsg); @@ -190,14 +192,14 @@ static void print_db_header(void){ /* ** Describe cell content. */ -static int describeContent( +static i64 describeContent( unsigned char *a, /* Cell content */ - int nLocal, /* Bytes in a[] */ + i64 nLocal, /* Bytes in a[] */ char *zDesc /* Write description here */ ){ - int nDesc = 0; - int n, i, j; - i64 x, v; + i64 nDesc = 0; + int n, j; + i64 i, x, v; const unsigned char *pData; const unsigned char *pLimit; char sep = ' '; @@ -237,15 +239,15 @@ static int describeContent( }else if( x==9 ){ sprintf(zDesc, "1"); }else if( x>=12 ){ - int size = (x-12)/2; + i64 size = (x-12)/2; if( (x&1)==0 ){ - sprintf(zDesc, "blob(%d)", size); + sprintf(zDesc, "blob(%lld)", size); }else{ - sprintf(zDesc, "txt(%d)", size); + sprintf(zDesc, "txt(%lld)", size); } pData += size; } - j = strlen(zDesc); + j = (int)strlen(zDesc); zDesc += j; nDesc += j; } @@ -256,11 +258,11 @@ static int describeContent( ** Compute the local payload size given the total payload size and ** the page size. */ -static int localPayload(i64 nPayload, char cType){ - int maxLocal; - int minLocal; - int surplus; - int nLocal; +static i64 localPayload(i64 nPayload, char cType){ + i64 maxLocal; + i64 minLocal; + i64 surplus; + i64 nLocal; if( cType==13 ){ /* Table leaf */ maxLocal = pagesize-35; @@ -288,19 +290,19 @@ static int localPayload(i64 nPayload, char cType){ ** ** The return value is the local cell size. */ -static int describeCell( +static i64 describeCell( unsigned char cType, /* Page type */ unsigned char *a, /* Cell content */ int showCellContent, /* Show cell content if true */ char **pzDesc /* Store description here */ ){ int i; - int nDesc = 0; + i64 nDesc = 0; int n = 0; int leftChild; i64 nPayload; i64 rowid; - int nLocal; + i64 nLocal; static char zDesc[1000]; i = 0; if( cType<=5 ){ @@ -373,13 +375,14 @@ static void decodeCell( int szPgHdr, /* Size of the page header. 0 or 100 */ int ofst /* Cell begins at a[ofst] */ ){ - int i, j, k; + int i, j; int leftChild; + i64 k; i64 nPayload; i64 rowid; i64 nHdr; i64 iType; - int nLocal; + i64 nLocal; unsigned char *x = a + ofst; unsigned char *end; unsigned char cType = a[0]; @@ -400,10 +403,10 @@ static void decodeCell( printBytes(a, x, i); nLocal = localPayload(nPayload, cType); if( nLocal==nPayload ){ - printf("payload-size: %d\n", (int)nPayload); + printf("payload-size: %lld\n", nPayload); }else{ - printf("payload-size: %d (%d local, %d overflow)\n", - (int)nPayload, nLocal, (int)(nPayload-nLocal)); + printf("payload-size: %lld (%lld local, %lld overflow)\n", + nPayload, nLocal, nPayload-nLocal); } x += i; }else{ @@ -452,7 +455,7 @@ static void decodeCell( } printf("%s\n", zTypeName); szCol[nCol] = sz; - ofstCol[nCol] = k; + ofstCol[nCol] = (int)k; typeCol[nCol] = (int)iType; k += sz; nCol++; @@ -506,7 +509,7 @@ static void decodeCell( } if( j=nByte ){ @@ -74,7 +73,7 @@ static unsigned print_decode_line( ** in global variables. */ static unsigned decode_journal_header(int iOfst){ - char *pHdr = read_content(64, iOfst); + unsigned char *pHdr = read_content(64, iOfst); unsigned nPage; printf("Header at offset %d:\n", iOfst); print_decode_line(pHdr, 0, 4, "Header part 1 (3654616569)"); @@ -101,12 +100,11 @@ static void print_page(int iOfst){ char zTitle[50]; aData = read_content(pageSize+8, iOfst); sprintf(zTitle, "page number for page at offset %d", iOfst); - print_decode_line(aData, 0, 4, zTitle); + print_decode_line(aData-iOfst, iOfst, 4, zTitle); free(aData); } int main(int argc, char **argv){ - int rc; int nPage, cnt; int iOfst; if( argc!=2 ){ @@ -136,4 +134,5 @@ int main(int argc, char **argv){ iOfst = (iOfst/sectorSize + 1)*sectorSize; } fclose(db); + return 0; } diff --git a/tool/showstat4.c b/tool/showstat4.c new file mode 100644 index 0000000000..215962919e --- /dev/null +++ b/tool/showstat4.c @@ -0,0 +1,158 @@ +/* +** This utility program decodes and displays the content of the +** sqlite_stat4 table in the database file named on the command +** line. +*/ +#include +#include +#include +#include +#include "sqlite3.h" + +typedef sqlite3_int64 i64; /* 64-bit signed integer type */ + + +/* +** Convert the var-int format into i64. Return the number of bytes +** in the var-int. Write the var-int value into *pVal. +*/ +static int decodeVarint(const unsigned char *z, i64 *pVal){ + i64 v = 0; + int i; + for(i=0; i<8; i++){ + v = (v<<7) + (z[i]&0x7f); + if( (z[i]&0x80)==0 ){ *pVal = v; return i+1; } + } + v = (v<<8) + (z[i]&0xff); + *pVal = v; + return 9; +} + + + +int main(int argc, char **argv){ + sqlite3 *db; + sqlite3_stmt *pStmt; + char *zIdx = 0; + int rc, j, x, y, mxHdr; + const unsigned char *aSample; + int nSample; + i64 iVal; + const char *zSep; + int iRow = 0; + + if( argc!=2 ){ + fprintf(stderr, "Usage: %s DATABASE-FILE\n", argv[0]); + exit(1); + } + rc = sqlite3_open(argv[1], &db); + if( rc!=SQLITE_OK || db==0 ){ + fprintf(stderr, "Cannot open database file [%s]\n", argv[1]); + exit(1); + } + rc = sqlite3_prepare_v2(db, + "SELECT tbl||'.'||idx, nEq, nLT, nDLt, sample " + "FROM sqlite_stat4 ORDER BY 1", -1, + &pStmt, 0); + if( rc!=SQLITE_OK || pStmt==0 ){ + fprintf(stderr, "%s\n", sqlite3_errmsg(db)); + sqlite3_close(db); + exit(1); + } + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + if( zIdx==0 || strcmp(zIdx, (const char*)sqlite3_column_text(pStmt,0))!=0 ){ + if( zIdx ) printf("\n**************************************" + "**************\n\n"); + sqlite3_free(zIdx); + zIdx = sqlite3_mprintf("%s", sqlite3_column_text(pStmt,0)); + iRow = 0; + } + printf("%s sample %d ------------------------------------\n", zIdx, ++iRow); + printf(" nEq = %s\n", sqlite3_column_text(pStmt,1)); + printf(" nLt = %s\n", sqlite3_column_text(pStmt,2)); + printf(" nDLt = %s\n", sqlite3_column_text(pStmt,3)); + printf(" sample = x'"); + aSample = sqlite3_column_blob(pStmt,4); + nSample = sqlite3_column_bytes(pStmt,4); + for(j=0; jnSample ){ + printf(" \n"); + continue; + } + y = mxHdr = (int)iVal; + while( xmxHdr ) break; + if( iVal<0 ) break; + switch( iVal ){ + case 0: sz = 0; break; + case 1: sz = 1; break; + case 2: sz = 2; break; + case 3: sz = 3; break; + case 4: sz = 4; break; + case 5: sz = 6; break; + case 6: sz = 8; break; + case 7: sz = 8; break; + case 8: sz = 0; break; + case 9: sz = 0; break; + case 10: + case 11: sz = 0; break; + default: sz = (int)(iVal-12)/2; break; + } + if( y+sz>nSample ) break; + if( iVal==0 ){ + printf("%sNULL", zSep); + }else if( iVal==8 || iVal==9 ){ + printf("%s%d", zSep, ((int)iVal)-8); + }else if( iVal<=7 ){ + v = (signed char)aSample[y]; + for(j=1; j #include #include + +#if !defined(_MSC_VER) #include +#else +#include +#endif + #include #include @@ -172,7 +178,7 @@ static void print_decode_line( int val = aData[ofst]; char zBuf[100]; sprintf(zBuf, " %03x: %02x", ofst, aData[ofst]); - i = strlen(zBuf); + i = (int)strlen(zBuf); for(j=1; j<4; j++){ if( j>=nByte ){ sprintf(&zBuf[i], " "); @@ -180,7 +186,7 @@ static void print_decode_line( sprintf(&zBuf[i], " %02x", aData[ofst+j]); val = val*256 + aData[ofst+j]; } - i += strlen(&zBuf[i]); + i += (int)strlen(&zBuf[i]); } if( asHex ){ sprintf(&zBuf[i], " 0x%08x", val); @@ -273,14 +279,14 @@ static void print_wal_header(Cksum *pCksum){ /* ** Describe cell content. */ -static int describeContent( +static i64 describeContent( unsigned char *a, /* Cell content */ - int nLocal, /* Bytes in a[] */ + i64 nLocal, /* Bytes in a[] */ char *zDesc /* Write description here */ ){ int nDesc = 0; - int n, i, j; - i64 x, v; + int n, j; + i64 i, x, v; const unsigned char *pData; const unsigned char *pLimit; char sep = ' '; @@ -320,15 +326,15 @@ static int describeContent( }else if( x==9 ){ sprintf(zDesc, "1"); }else if( x>=12 ){ - int size = (x-12)/2; + i64 size = (x-12)/2; if( (x&1)==0 ){ - sprintf(zDesc, "blob(%d)", size); + sprintf(zDesc, "blob(%lld)", size); }else{ - sprintf(zDesc, "txt(%d)", size); + sprintf(zDesc, "txt(%lld)", size); } pData += size; } - j = strlen(zDesc); + j = (int)strlen(zDesc); zDesc += j; nDesc += j; } @@ -339,11 +345,11 @@ static int describeContent( ** Compute the local payload size given the total payload size and ** the page size. */ -static int localPayload(i64 nPayload, char cType){ - int maxLocal; - int minLocal; - int surplus; - int nLocal; +static i64 localPayload(i64 nPayload, char cType){ + i64 maxLocal; + i64 minLocal; + i64 surplus; + i64 nLocal; if( cType==13 ){ /* Table leaf */ maxLocal = pagesize-35; @@ -370,19 +376,19 @@ static int localPayload(i64 nPayload, char cType){ ** ** The return value is the local cell size. */ -static int describeCell( +static i64 describeCell( unsigned char cType, /* Page type */ unsigned char *a, /* Cell content */ int showCellContent, /* Show cell content if true */ char **pzDesc /* Store description here */ ){ int i; - int nDesc = 0; + i64 nDesc = 0; int n = 0; int leftChild; i64 nPayload; i64 rowid; - int nLocal; + i64 nLocal; static char zDesc[1000]; i = 0; if( cType<=5 ){ @@ -479,17 +485,17 @@ static void decode_btree_page( for(i=0; i +#include +#include + +#if defined(_MSC_VER) || defined(__BORLANDC__) + typedef __int64 i64; + typedef unsigned __int64 u64; +#else + typedef long long int i64; + typedef unsigned long long int u64; +#endif + +static int hexValue(char c){ + if( c>='0' && c<='9' ) return c - '0'; + if( c>='a' && c<='f' ) return c - 'a' + 10; + if( c>='A' && c<='F' ) return c - 'A' + 10; + return -1; +} + +static char toHex(unsigned char c){ + return "0123456789abcdef"[c&0xf]; +} + +static int putVarint(unsigned char *p, u64 v){ + int i, j, n; + unsigned char buf[10]; + if( v & (((u64)0xff000000)<<32) ){ + p[8] = (unsigned char)v; + v >>= 8; + for(i=7; i>=0; i--){ + p[i] = (unsigned char)((v & 0x7f) | 0x80); + v >>= 7; + } + return 9; + } + n = 0; + do{ + buf[n++] = (unsigned char)((v & 0x7f) | 0x80); + v >>= 7; + }while( v!=0 ); + buf[0] &= 0x7f; + for(i=0, j=n-1; j>=0; j--, i++){ + p[i] = buf[j]; + } + return n; +} + + +int main(int argc, char **argv){ + int i; + u64 x; + u64 uX = 0; + i64 iX; + int n; + unsigned char zHex[20]; + + if( argc==1 ){ + fprintf(stderr, + "Usage:\n" + " %s HH HH HH ... Convert varint to decimal\n" + " %s DDDDD Convert decimal to varint\n" + " Add '+' or '-' before DDDDD to disambiguate.\n", + argv[0], argv[0]); + exit(1); + } + if( argc>2 + || (strlen(argv[1])==2 && hexValue(argv[1][0])>=0 && hexValue(argv[1][1])>=0) + ){ + /* Hex to decimal */ + for(i=1; i'9' ){ + fprintf(stderr, "Not a decimal number: %s", argv[1]); + exit(1); + } + uX = uX*10 + z[0] - '0'; + z++; + } + if( sign<0 ){ + memcpy(&iX, &uX, 8); + iX = -iX; + memcpy(&uX, &iX, 8); + } + } + n = putVarint(zHex, uX); + printf("%lld =", (i64)uX); + for(i=0; i>4), toHex(zHex[i]&0x0f)); + } + printf("\n"); + return 0; +} diff --git a/tool/vdbe-compress.tcl b/tool/vdbe-compress.tcl index a349830bcf..9477f4afe6 100644 --- a/tool/vdbe-compress.tcl +++ b/tool/vdbe-compress.tcl @@ -110,6 +110,11 @@ while {![eof stdin]} { foreach v $vlist { regsub -all "(\[^a-zA-Z0-9>.\])${v}(\\W)" $line "\\1u.$sname.$v\\2" line regsub -all "(\[^a-zA-Z0-9>.\])${v}(\\W)" $line "\\1u.$sname.$v\\2" line + + # The expressions above fail to catch instance of variable "abc" in + # expressions like (32>abc). The following expression makes those + # substitutions. + regsub -all "(\[^-\])>${v}(\\W)" $line "\\1>u.$sname.$v\\2" line } append afterUnion [string trimright $line]\n } elseif {$line=="" && [eof stdin]} {