diff --git a/Makefile.in b/Makefile.in index b961070c23..9076ffc085 100644 --- a/Makefile.in +++ b/Makefile.in @@ -566,6 +566,7 @@ SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1 FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ +FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000 FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c DBFUZZ_OPT = @@ -620,7 +621,8 @@ fuzzcheck$(TEXE): $(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(LTLINK) -o $@ $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) sqlite3.c $(TLIBS) ossshell$(TEXE): $(TOP)/test/ossfuzz.c $(TOP)/test/ossshell.c sqlite3.c sqlite3.h - $(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/ossshell.c sqlite3.c sqlite3.h $(TLIBS) + $(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/ossshell.c \ + $(TOP)/test/ossfuzz.c sqlite3.c $(TLIBS) dbfuzz$(TEXE): $(TOP)/test/dbfuzz.c sqlite3.c sqlite3.h $(LTLINK) -o $@ $(DBFUZZ_OPT) $(TOP)/test/dbfuzz.c sqlite3.c $(TLIBS) @@ -1153,6 +1155,10 @@ sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl sqlite3_analyzer$(TEXE): sqlite3_analyzer.c $(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS) +dbdump$(TEXE): $(TOP)/ext/misc/dbdump.c sqlite3.lo + $(LTLINK) -DDBDUMP_STANDALONE -o $@ \ + $(TOP)/ext/misc/dbdump.c sqlite3.lo $(TLIBS) + showdb$(TEXE): $(TOP)/tool/showdb.c sqlite3.lo $(LTLINK) -o $@ $(TOP)/tool/showdb.c sqlite3.lo $(TLIBS) @@ -1177,8 +1183,13 @@ LogEst$(TEXE): $(TOP)/tool/logest.c sqlite3.h wordcount$(TEXE): $(TOP)/test/wordcount.c sqlite3.lo $(LTLINK) -o $@ $(TOP)/test/wordcount.c sqlite3.lo $(TLIBS) -speedtest1$(TEXE): $(TOP)/test/speedtest1.c sqlite3.lo - $(LTLINK) -o $@ $(TOP)/test/speedtest1.c sqlite3.lo $(TLIBS) +speedtest1$(TEXE): $(TOP)/test/speedtest1.c sqlite3.c + $(LTLINK) $(ST_OPT) -o $@ $(TOP)/test/speedtest1.c sqlite3.c $(TLIBS) + +KV_OPT += -DSQLITE_DIRECT_OVERFLOW_READ + +kvtest$(TEXE): $(TOP)/test/kvtest.c sqlite3.c + $(LTLINK) $(KV_OPT) -o $@ $(TOP)/test/kvtest.c sqlite3.c $(TLIBS) rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.lo $(LTLINK) -I. -o $@ $(TOP)/ext/rbu/rbu.c sqlite3.lo $(TLIBS) diff --git a/Makefile.msc b/Makefile.msc index d4aba301fb..1dfc6bc49d 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -24,6 +24,13 @@ USE_AMALGAMATION = 1 USE_FULLWARN = 0 !ENDIF +# Set this non-0 to enable treating warnings as errors (-WX, etc) when +# compiling. +# +!IFNDEF USE_FATAL_WARN +USE_FATAL_WARN = 0 +!ENDIF + # Set this non-0 to enable full runtime error checks (-RTC1, etc). This # has no effect if (any) optimizations are enabled. # @@ -493,6 +500,12 @@ TCC = $(CC) -nologo -W4 -DINCLUDE_MSVC_H=1 $(CCOPTS) $(TCCOPTS) TCC = $(CC) -nologo -W3 $(CCOPTS) $(TCCOPTS) !ENDIF +# Check if warnings should be treated as errors when compiling. +# +!IF $(USE_FATAL_WARN)!=0 +TCC = $(TCC) -WX +!ENDIF + TCC = $(TCC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -I$(TOP)\src -fp:precise RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -I$(TOP)\src $(RCOPTS) $(RCCOPTS) @@ -732,6 +745,7 @@ RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1 !IF $(DEBUG)>2 TCC = $(TCC) -DSQLITE_DEBUG=1 +TCC = $(TCC) -DSQLITE_ENABLE_WHERETRACE -DSQLITE_ENABLE_SELECTTRACE RCC = $(RCC) -DSQLITE_DEBUG=1 !ENDIF @@ -1496,9 +1510,13 @@ SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_ # MPTESTER_COMPILE_OPTS = -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS5 FUZZERSHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ +FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ -DSQLITE_MAX_MEMORY=50000000 FUZZCHECK_SRC = $(TOP)\test\fuzzcheck.c $(TOP)\test\ossfuzz.c OSSSHELL_SRC = $(TOP)\test\ossshell.c $(TOP)\test\ossfuzz.c +DBFUZZ_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION +KV_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_DIRECT_OVERFLOW_READ +DBSELFTEST_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5 +ST_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 # Standard options to testfixture. # @@ -1541,7 +1559,7 @@ $(SQLITE3DLL): $(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP) sqlite3.def: libsqlite3.lib echo EXPORTS > sqlite3.def dumpbin /all libsqlite3.lib \ - | $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3_[^@]*)(?:@\d+)?$$" \1 \ + | $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3(?:session|changeset)?_[^@]*)(?:@\d+)?$$" \1 \ | sort >> sqlite3.def # <> @@ -1568,6 +1586,9 @@ sourcetest: srcck1.exe sqlite3.c fuzzershell.exe: $(TOP)\tool\fuzzershell.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) $(FUZZERSHELL_COMPILE_OPTS) $(TOP)\tool\fuzzershell.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) +dbfuzz.exe: $(TOP)\test\dbfuzz.c $(SQLITE3C) $(SQLITE3H) + $(LTLINK) $(NO_WARN) $(DBFUZZ_COMPILE_OPTS) $(TOP)\test\dbfuzz.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) + fuzzcheck.exe: $(FUZZCHECK_SRC) $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) $(FUZZCHECK_COMPILE_OPTS) $(FUZZCHECK_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) @@ -2157,6 +2178,10 @@ sqlite3_analyzer.exe: sqlite3_analyzer.c $(LIBRESOBJS) $(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_analyzer.c \ /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS) +dbdump.exe: $(TOP)\ext\misc\dbdump.c $(SQLITE3C) $(SQLITE3H) + $(LTLINK) $(NO_WARN) -DDBDUMP_STANDALONE $(TOP)\ext\misc\dbdump.c $(SQLITE3C) \ + /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) + testloadext.lo: $(TOP)\src\test_loadext.c $(LTCOMPILE) $(NO_WARN) -c $(TOP)\src\test_loadext.c @@ -2164,46 +2189,54 @@ testloadext.dll: testloadext.lo $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ testloadext.lo showdb.exe: $(TOP)\tool\showdb.c $(SQLITE3C) $(SQLITE3H) - $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ + $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ $(TOP)\tool\showdb.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) showstat4.exe: $(TOP)\tool\showstat4.c $(SQLITE3C) $(SQLITE3H) - $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ + $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ $(TOP)\tool\showstat4.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) showjournal.exe: $(TOP)\tool\showjournal.c $(SQLITE3C) $(SQLITE3H) - $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ + $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ $(TOP)\tool\showjournal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) showwal.exe: $(TOP)\tool\showwal.c $(SQLITE3C) $(SQLITE3H) - $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ + $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ $(TOP)\tool\showwal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) -changeset.exe: $(TOP)\ext\session\changeset.c $(SQLITE3C) - $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ +changeset.exe: $(TOP)\ext\session\changeset.c $(SQLITE3C) $(SQLITE3H) + $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ + -DSQLITE_ENABLE_SESSION=1 -DSQLITE_ENABLE_PREUPDATE_HOOK=1 \ $(TOP)\ext\session\changeset.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) fts3view.exe: $(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) $(SQLITE3H) - $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ + $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ $(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) rollback-test.exe: $(TOP)\tool\rollback-test.c $(SQLITE3C) $(SQLITE3H) - $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ + $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ $(TOP)\tool\rollback-test.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) LogEst.exe: $(TOP)\tool\logest.c $(SQLITE3H) - $(LTLINK) $(NO_WARN) -Fe$@ $(TOP)\tool\LogEst.c /link $(LDFLAGS) $(LTLINKOPTS) + $(LTLINK) $(NO_WARN) $(TOP)\tool\LogEst.c /link $(LDFLAGS) $(LTLINKOPTS) wordcount.exe: $(TOP)\test\wordcount.c $(SQLITE3C) $(SQLITE3H) - $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ + $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ $(TOP)\test\wordcount.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) speedtest1.exe: $(TOP)\test\speedtest1.c $(SQLITE3C) $(SQLITE3H) - $(LTLINK) $(NO_WARN) -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ + $(LTLINK) $(NO_WARN) $(ST_COMPILE_OPTS) -DSQLITE_OMIT_LOAD_EXTENSION \ $(TOP)\test\speedtest1.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) +kvtest.exe: $(TOP)\test\kvtest.c $(SQLITE3C) $(SQLITE3H) + $(LTLINK) $(NO_WARN) $(KV_COMPILE_OPTS) \ + $(TOP)\test\kvtest.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) + +dbselftest.exe: $(TOP)\test\dbselftest.c $(SQLITE3C) $(SQLITE3H) + $(LTLINK) $(NO_WARN) $(DBSELFTEST_COMPILE_OPTS) $(TOP)\test\dbselftest.c $(SQLITE3C) + rbu.exe: $(TOP)\ext\rbu\rbu.c $(TOP)\ext\rbu\sqlite3rbu.c $(SQLITE3C) $(SQLITE3H) - $(LTLINK) $(NO_WARN) -DSQLITE_ENABLE_RBU -Fe$@ \ + $(LTLINK) $(NO_WARN) -DSQLITE_ENABLE_RBU \ $(TOP)\ext\rbu\rbu.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) moreclean: clean @@ -2227,7 +2260,7 @@ clean: del /Q tclsqlite3.exe $(SQLITETCLH) $(SQLITETCLDECLSH) 2>NUL del /Q testloadext.dll 2>NUL del /Q testfixture.exe test.db 2>NUL - del /Q LogEst.exe fts3view.exe rollback-test.exe showdb.exe 2>NUL + del /Q LogEst.exe fts3view.exe rollback-test.exe showdb.exe dbdump.exe 2>NUL del /Q changeset.exe 2>NUL del /Q showjournal.exe showstat4.exe showwal.exe speedtest1.exe 2>NUL del /Q mptester.exe wordcount.exe rbu.exe srcck1.exe 2>NUL diff --git a/README.md b/README.md index dbc020574e..7b860c211e 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,48 @@ If you are reading this on a Git mirror someplace, you are doing it wrong. The [official repository](https://www.sqlite.org/src/) is better. Go there now. +## Obtaining The Code + +SQLite sources are managed using the +[Fossil](https://www.fossil-scm.org/), a distributed version control system +that was specifically designed to support SQLite development. +If you do not want to use Fossil, you can download tarballs or ZIP +archives as follows: + + * Lastest trunk check-in: + or + . + + * Latest release: + or + . + + * For other check-ins, substitute an appropriate branch name or + tag or hash prefix for "release" in the URLs of the previous + bullet. Or browse the [timeline](https://www.sqlite.org/src/timeline) + to locate the check-in desired, click on its information page link, + then click on the "Tarball" or "ZIP Archive" links on the information + page. + +If you do want to use Fossil to check out the source tree, +first install Fossil version 2.0 or later. +(Source tarballs and precompiled binaries available +[here](https://www.fossil-scm.org/fossil/uv/download.html).) +Then run commands like this: + + mkdir ~/sqlite + cd ~/sqlite + fossil clone https://www.sqlite.org/src sqlite.fossil + fossil open sqlite.fossil + +After setting up a repository using the steps above, you can always +update to the lastest version using: + + fossil update trunk ;# latest trunk check-in + fossil update release ;# latest official release + +Or type "fossil ui" to get a web-based user interface. + ## Compiling First create a directory in which to place @@ -18,13 +60,13 @@ script found at the root of the source tree. Then run "make". For example: - tar xzf sqlite.tar.gz ;# Unpack the source tree into "sqlite" - mkdir bld ;# Build will occur in a sibling directory - cd bld ;# Change to the build directory - ../sqlite/configure ;# Run the configure script - make ;# Run the makefile. - make sqlite3.c ;# Build the "amalgamation" source file - make test ;# Run some tests (requires Tcl) + tar xzf sqlite.tar.gz ;# Unpack the source tree into "sqlite" + mkdir bld ;# Build will occur in a sibling directory + cd bld ;# Change to the build directory + ../sqlite/configure ;# Run the configure script + make ;# Run the makefile. + make sqlite3.c ;# Build the "amalgamation" source file + make test ;# Run some tests (requires Tcl) See the makefile for additional targets. @@ -43,13 +85,13 @@ with the provided "Makefile.msc" to build one of the supported targets. For example: - mkdir bld - cd bld - nmake /f Makefile.msc TOP=..\sqlite - nmake /f Makefile.msc sqlite3.c TOP=..\sqlite - nmake /f Makefile.msc sqlite3.dll TOP=..\sqlite - nmake /f Makefile.msc sqlite3.exe TOP=..\sqlite - nmake /f Makefile.msc test TOP=..\sqlite + mkdir bld + cd bld + nmake /f Makefile.msc TOP=..\sqlite + nmake /f Makefile.msc sqlite3.c TOP=..\sqlite + nmake /f Makefile.msc sqlite3.dll TOP=..\sqlite + nmake /f Makefile.msc sqlite3.exe TOP=..\sqlite + nmake /f Makefile.msc test TOP=..\sqlite There are several build options that can be set via the NMAKE command line. For example, to build for WinRT, simply add "FOR_WINRT=1" argument diff --git a/VERSION b/VERSION index 3f67e25cea..c5b45eb7b1 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.17.0 +3.18.0 diff --git a/autoconf/Makefile.msc b/autoconf/Makefile.msc index b53e2370eb..8c13778fc8 100644 --- a/autoconf/Makefile.msc +++ b/autoconf/Makefile.msc @@ -24,6 +24,13 @@ TOP = . USE_FULLWARN = 0 !ENDIF +# Set this non-0 to enable treating warnings as errors (-WX, etc) when +# compiling. +# +!IFNDEF USE_FATAL_WARN +USE_FATAL_WARN = 0 +!ENDIF + # Set this non-0 to enable full runtime error checks (-RTC1, etc). This # has no effect if (any) optimizations are enabled. # @@ -454,6 +461,12 @@ TCC = $(CC) -nologo -W4 -DINCLUDE_MSVC_H=1 $(CCOPTS) $(TCCOPTS) TCC = $(CC) -nologo -W3 $(CCOPTS) $(TCCOPTS) !ENDIF +# Check if warnings should be treated as errors when compiling. +# +!IF $(USE_FATAL_WARN)!=0 +TCC = $(TCC) -WX +!ENDIF + TCC = $(TCC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -fp:precise RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) $(RCOPTS) $(RCCOPTS) @@ -631,6 +644,7 @@ RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1 !IF $(DEBUG)>2 TCC = $(TCC) -DSQLITE_DEBUG=1 +TCC = $(TCC) -DSQLITE_ENABLE_WHERETRACE -DSQLITE_ENABLE_SELECTTRACE RCC = $(RCC) -DSQLITE_DEBUG=1 !ENDIF @@ -937,7 +951,7 @@ Replace.exe: sqlite3.def: Replace.exe $(LIBOBJ) echo EXPORTS > sqlite3.def dumpbin /all $(LIBOBJ) \ - | .\Replace.exe "^\s+/EXPORT:_?(sqlite3_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \ + | .\Replace.exe "^\s+/EXPORT:_?(sqlite3(?:session|changeset)?_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \ | sort >> sqlite3.def $(SQLITE3EXE): $(TOP)\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H) diff --git a/autoconf/configure.ac b/autoconf/configure.ac index b9a11aac8f..5a607de054 100644 --- a/autoconf/configure.ac +++ b/autoconf/configure.ac @@ -55,9 +55,9 @@ AS_IF([ test x"$enable_editline" != xno ],[ LIBS="" AC_SEARCH_LIBS([readline],[edit],[ AC_DEFINE([HAVE_EDITLINE],1,Define to use BSD editline) - READLINE_LIBS=$LIBS + READLINE_LIBS="$LIBS -ltinfo" enable_readline=no - ]) + ],[],[-ltinfo]) AS_UNSET(ac_cv_search_readline) LIBS=$sLIBS ]) diff --git a/configure b/configure index be7c89143b..628e23f2d0 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for sqlite 3.17.0. +# Generated by GNU Autoconf 2.69 for sqlite 3.18.0. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. @@ -726,8 +726,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.17.0' -PACKAGE_STRING='sqlite 3.17.0' +PACKAGE_VERSION='3.18.0' +PACKAGE_STRING='sqlite 3.18.0' PACKAGE_BUGREPORT='' PACKAGE_URL='' @@ -1463,7 +1463,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.17.0 to adapt to many kinds of systems. +\`configure' configures sqlite 3.18.0 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1528,7 +1528,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.17.0:";; + short | recursive ) echo "Configuration of sqlite 3.18.0:";; esac cat <<\_ACEOF @@ -1652,7 +1652,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.17.0 +sqlite configure 3.18.0 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -2071,7 +2071,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.17.0, which was +It was created by sqlite $as_me 3.18.0, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -12151,7 +12151,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.17.0, which was +This file was extended by sqlite $as_me 3.18.0, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -12217,7 +12217,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -sqlite config.status 3.17.0 +sqlite config.status 3.18.0 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/ext/README.md b/ext/README.md new file mode 100644 index 0000000000..933a33d053 --- /dev/null +++ b/ext/README.md @@ -0,0 +1,8 @@ +## Loadable Extensions + +Various [loadable extensions](https://www.sqlite.org/loadext.html) for +SQLite are found in subfolders. + +Most subfolders are dedicated to a single loadable extension (for +example FTS5, or RTREE). But the misc/ subfolder contains a collection +of smaller single-file extensions. diff --git a/ext/README.txt b/ext/README.txt deleted file mode 100644 index 009495f59d..0000000000 --- a/ext/README.txt +++ /dev/null @@ -1,2 +0,0 @@ -Version loadable extensions to SQLite are found in subfolders -of this folder. diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 748faefec5..6f059bc95d 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -349,8 +349,9 @@ int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){ ** Return the number of bytes read, or 0 on error. ** The value is stored in *v. */ -int sqlite3Fts3GetVarint(const char *p, sqlite_int64 *v){ - const char *pStart = p; +int sqlite3Fts3GetVarint(const char *pBuf, sqlite_int64 *v){ + const unsigned char *p = (const unsigned char*)pBuf; + const unsigned char *pStart = p; u32 a; u64 b; int shift; @@ -492,6 +493,7 @@ static int fts3DisconnectMethod(sqlite3_vtab *pVtab){ assert( p->pSegments==0 ); /* Free any prepared statements held */ + sqlite3_finalize(p->pSeekStmt); for(i=0; iaStmt); i++){ sqlite3_finalize(p->aStmt[i]); } @@ -1363,9 +1365,9 @@ static int fts3InitVtab( p->pTokenizer = pTokenizer; p->nMaxPendingData = FTS3_MAX_PENDING_DATA; p->bHasDocsize = (isFts4 && bNoDocsize==0); - p->bHasStat = isFts4; - p->bFts4 = isFts4; - p->bDescIdx = bDescIdx; + p->bHasStat = (u8)isFts4; + p->bFts4 = (u8)isFts4; + p->bDescIdx = (u8)bDescIdx; p->nAutoincrmerge = 0xff; /* 0xff means setting unknown */ p->zContentTbl = zContent; p->zLanguageid = zLanguageid; @@ -1396,7 +1398,9 @@ static int fts3InitVtab( char *z; int n = 0; z = (char *)sqlite3Fts3NextToken(aCol[iCol], &n); - memcpy(zCsr, z, n); + if( n>0 ){ + memcpy(zCsr, z, n); + } zCsr[n] = '\0'; sqlite3Fts3Dequote(zCsr); p->azColumn[iCol] = zCsr; @@ -1680,6 +1684,26 @@ static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ return SQLITE_OK; } +/* +** Finalize the statement handle at pCsr->pStmt. +** +** Or, if that statement handle is one created by fts3CursorSeekStmt(), +** and the Fts3Table.pSeekStmt slot is currently NULL, save the statement +** pointer there instead of finalizing it. +*/ +static void fts3CursorFinalizeStmt(Fts3Cursor *pCsr){ + if( pCsr->bSeekStmt ){ + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + if( p->pSeekStmt==0 ){ + p->pSeekStmt = pCsr->pStmt; + sqlite3_reset(pCsr->pStmt); + pCsr->pStmt = 0; + } + pCsr->bSeekStmt = 0; + } + sqlite3_finalize(pCsr->pStmt); +} + /* ** Close the cursor. For additional information see the documentation ** on the xClose method of the virtual table interface. @@ -1687,7 +1711,7 @@ static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); - sqlite3_finalize(pCsr->pStmt); + fts3CursorFinalizeStmt(pCsr); sqlite3Fts3ExprFree(pCsr->pExpr); sqlite3Fts3FreeDeferredTokens(pCsr); sqlite3_free(pCsr->aDoclist); @@ -1705,20 +1729,23 @@ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ ** ** (or the equivalent for a content=xxx table) and set pCsr->pStmt to ** it. If an error occurs, return an SQLite error code. -** -** Otherwise, set *ppStmt to point to pCsr->pStmt and return SQLITE_OK. */ -static int fts3CursorSeekStmt(Fts3Cursor *pCsr, sqlite3_stmt **ppStmt){ +static int fts3CursorSeekStmt(Fts3Cursor *pCsr){ int rc = SQLITE_OK; if( pCsr->pStmt==0 ){ Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; char *zSql; - zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist); - if( !zSql ) return SQLITE_NOMEM; - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); - sqlite3_free(zSql); + if( p->pSeekStmt ){ + pCsr->pStmt = p->pSeekStmt; + p->pSeekStmt = 0; + }else{ + zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist); + if( !zSql ) return SQLITE_NOMEM; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); + } + if( rc==SQLITE_OK ) pCsr->bSeekStmt = 1; } - *ppStmt = pCsr->pStmt; return rc; } @@ -1730,9 +1757,7 @@ static int fts3CursorSeekStmt(Fts3Cursor *pCsr, sqlite3_stmt **ppStmt){ static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ int rc = SQLITE_OK; if( pCsr->isRequireSeek ){ - sqlite3_stmt *pStmt = 0; - - rc = fts3CursorSeekStmt(pCsr, &pStmt); + rc = fts3CursorSeekStmt(pCsr); if( rc==SQLITE_OK ){ sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId); pCsr->isRequireSeek = 0; @@ -3190,7 +3215,7 @@ static int fts3FilterMethod( assert( iIdx==nVal ); /* In case the cursor has been used before, clear it now. */ - sqlite3_finalize(pCsr->pStmt); + fts3CursorFinalizeStmt(pCsr); sqlite3_free(pCsr->aDoclist); sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); sqlite3Fts3ExprFree(pCsr->pExpr); @@ -3258,7 +3283,7 @@ static int fts3FilterMethod( rc = SQLITE_NOMEM; } }else if( eSearch==FTS3_DOCID_SEARCH ){ - rc = fts3CursorSeekStmt(pCsr, &pCsr->pStmt); + rc = fts3CursorSeekStmt(pCsr); if( rc==SQLITE_OK ){ rc = sqlite3_bind_value(pCsr->pStmt, 1, pCons); } @@ -3386,8 +3411,10 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){ const u32 nMinMerge = 64; /* Minimum amount of incr-merge work to do */ Fts3Table *p = (Fts3Table*)pVtab; - int rc = sqlite3Fts3PendingTermsFlush(p); + int rc; + i64 iLastRowid = sqlite3_last_insert_rowid(p->db); + rc = sqlite3Fts3PendingTermsFlush(p); if( rc==SQLITE_OK && p->nLeafAdd>(nMinMerge/16) && p->nAutoincrmerge && p->nAutoincrmerge!=0xff @@ -3402,6 +3429,7 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){ if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, p->nAutoincrmerge); } sqlite3Fts3SegmentsClose(p); + sqlite3_set_last_insert_rowid(p->db, iLastRowid); return rc; } @@ -3422,7 +3450,7 @@ static int fts3SetHasStat(Fts3Table *p){ if( rc==SQLITE_OK ){ int bHasStat = (sqlite3_step(pStmt)==SQLITE_ROW); rc = sqlite3_finalize(pStmt); - if( rc==SQLITE_OK ) p->bHasStat = bHasStat; + if( rc==SQLITE_OK ) p->bHasStat = (u8)bHasStat; } sqlite3_free(zSql); }else{ diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 0c86c4217e..c3cab9d821 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -230,6 +230,7 @@ struct Fts3Table { ** statements is run and reset within a single virtual table API call. */ sqlite3_stmt *aStmt[40]; + sqlite3_stmt *pSeekStmt; /* Cache for fts3CursorSeekStmt() */ char *zReadExprlist; char *zWriteExprlist; @@ -299,6 +300,7 @@ struct Fts3Cursor { i16 eSearch; /* Search strategy (see below) */ u8 isEof; /* True if at End Of Results */ u8 isRequireSeek; /* True if must seek pStmt to %_content row */ + u8 bSeekStmt; /* True if pStmt is a seek */ sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ Fts3Expr *pExpr; /* Parsed MATCH query string */ int iLangid; /* Language being queried for */ diff --git a/ext/fts3/fts3_unicode.c b/ext/fts3/fts3_unicode.c index 94fc27b5b4..dfb2680c50 100644 --- a/ext/fts3/fts3_unicode.c +++ b/ext/fts3/fts3_unicode.c @@ -136,16 +136,16 @@ static int unicodeAddExceptions( ){ const unsigned char *z = (const unsigned char *)zIn; const unsigned char *zTerm = &z[nIn]; - int iCode; + unsigned int iCode; int nEntry = 0; assert( bAlnum==0 || bAlnum==1 ); while( zi; j--) aNew[j] = aNew[j-1]; - aNew[i] = iCode; + aNew[i] = (int)iCode; nNew++; } } @@ -318,7 +318,7 @@ static int unicodeNext( ){ unicode_cursor *pCsr = (unicode_cursor *)pC; unicode_tokenizer *p = ((unicode_tokenizer *)pCsr->base.pTokenizer); - int iCode = 0; + unsigned int iCode = 0; char *zOut; const unsigned char *z = &pCsr->aInput[pCsr->iOff]; const unsigned char *zStart = z; @@ -330,7 +330,7 @@ static int unicodeNext( ** the input. */ while( z=zTerm ) return SQLITE_DONE; @@ -350,7 +350,7 @@ static int unicodeNext( /* Write the folded case of the last character read to the output */ zEnd = z; - iOut = sqlite3FtsUnicodeFold(iCode, p->bRemoveDiacritic); + iOut = sqlite3FtsUnicodeFold((int)iCode, p->bRemoveDiacritic); if( iOut ){ WRITE_UTF8(zOut, iOut); } @@ -358,8 +358,8 @@ static int unicodeNext( /* If the cursor is not at EOF, read the next character */ if( z>=zTerm ) break; READ_UTF8(z, zTerm, iCode); - }while( unicodeIsAlnum(p, iCode) - || sqlite3FtsUnicodeIsdiacritic(iCode) + }while( unicodeIsAlnum(p, (int)iCode) + || sqlite3FtsUnicodeIsdiacritic((int)iCode) ); /* Set the output variables and return. */ diff --git a/ext/fts3/fts3_unicode2.c b/ext/fts3/fts3_unicode2.c index 20b7a25dbf..da7251ed0c 100644 --- a/ext/fts3/fts3_unicode2.c +++ b/ext/fts3/fts3_unicode2.c @@ -127,9 +127,9 @@ int sqlite3FtsUnicodeIsalnum(int c){ 0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001, }; - if( c<128 ){ - return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 ); - }else if( c<(1<<22) ){ + if( (unsigned int)c<128 ){ + return ( (aAscii[c >> 5] & ((unsigned int)1 << (c & 0x001F)))==0 ); + }else if( (unsigned int)c<(1<<22) ){ unsigned int key = (((unsigned int)c)<<10) | 0x000003FF; int iRes = 0; int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; @@ -322,16 +322,17 @@ int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){ int ret = c; - assert( c>=0 ); assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 ); if( c<128 ){ if( c>='A' && c<='Z' ) ret = c + ('a' - 'A'); }else if( c<65536 ){ + const struct TableEntry *p; int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; int iLo = 0; int iRes = -1; + assert( c>aEntry[0].iCode ); while( iHi>=iLo ){ int iTest = (iHi + iLo) / 2; int cmp = (c - aEntry[iTest].iCode); @@ -342,14 +343,12 @@ int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){ iHi = iTest-1; } } - assert( iRes<0 || c>=aEntry[iRes].iCode ); - if( iRes>=0 ){ - const struct TableEntry *p = &aEntry[iRes]; - if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){ - ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF; - assert( ret>0 ); - } + assert( iRes>=0 && c>=aEntry[iRes].iCode ); + p = &aEntry[iRes]; + if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){ + ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF; + assert( ret>0 ); } if( bRemoveDiacritic ) ret = remove_diacritic(ret); diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 3ff481b0b0..d700c8403c 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -4956,11 +4956,14 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ ** Convert the text beginning at *pz into an integer and return ** its value. Advance *pz to point to the first character past ** the integer. +** +** This function used for parameters to merge= and incrmerge= +** commands. */ static int fts3Getint(const char **pz){ const char *z = *pz; int i = 0; - while( (*z)>='0' && (*z)<='9' ) i = 10*i + *(z++) - '0'; + while( (*z)>='0' && (*z)<='9' && i<214748363 ) i = 10*i + *(z++) - '0'; *pz = z; return i; } diff --git a/ext/fts3/tool/fts3cov.sh b/ext/fts3/tool/fts3cov.sh new file mode 100644 index 0000000000..b1f34dce76 --- /dev/null +++ b/ext/fts3/tool/fts3cov.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +set -e + +srcdir=`dirname $(dirname $(dirname $(dirname $0)))` +./testfixture $srcdir/test/fts3.test --output=fts3cov-out.txt + +echo "" + +for f in `ls $srcdir/ext/fts3/*.c` +do + f=`basename $f` + echo -ne "$f: " + gcov -b $f | grep Taken | sed 's/Taken at least once://' +done + diff --git a/ext/fts3/unicode/mkunicode.tcl b/ext/fts3/unicode/mkunicode.tcl index aafb4e9f9b..de89099122 100644 --- a/ext/fts3/unicode/mkunicode.tcl +++ b/ext/fts3/unicode/mkunicode.tcl @@ -227,7 +227,7 @@ proc print_isalnum {zFunc lRange} { an_print_ascii_bitmap $lRange puts { if( (unsigned int)c<128 ){ - return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 ); + return ( (aAscii[c >> 5] & ((unsigned int)1 << (c & 0x001F)))==0 ); }else if( (unsigned int)c<(1<<22) ){ unsigned int key = (((unsigned int)c)<<10) | 0x000003FF; int iRes = 0; diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 9ef338e821..d350856573 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -30,7 +30,9 @@ typedef short i16; typedef sqlite3_int64 i64; typedef sqlite3_uint64 u64; -#define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0]))) +#ifndef ArraySize +# define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0]))) +#endif #define testcase(x) #define ALWAYS(x) 1 diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 9dcd88a5bc..e18691ad0b 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -1110,7 +1110,10 @@ static int fts5ExprNodeNext_OR( || (bFromValid && fts5RowidCmp(pExpr, p1->iRowid, iFrom)<0) ){ int rc = fts5ExprNodeNext(pExpr, p1, bFromValid, iFrom); - if( rc!=SQLITE_OK ) return rc; + if( rc!=SQLITE_OK ){ + pNode->bNomatch = 0; + return rc; + } } } } @@ -1141,7 +1144,10 @@ static int fts5ExprNodeTest_AND( if( cmp>0 ){ /* Advance pChild until it points to iLast or laster */ rc = fts5ExprNodeNext(pExpr, pChild, 1, iLast); - if( rc!=SQLITE_OK ) return rc; + if( rc!=SQLITE_OK ){ + pAnd->bNomatch = 0; + return rc; + } } /* If the child node is now at EOF, so is the parent AND node. Otherwise, @@ -1180,6 +1186,8 @@ static int fts5ExprNodeNext_AND( int rc = fts5ExprNodeNext(pExpr, pNode->apChild[0], bFromValid, iFrom); if( rc==SQLITE_OK ){ rc = fts5ExprNodeTest_AND(pExpr, pNode); + }else{ + pNode->bNomatch = 0; } return rc; } @@ -1222,6 +1230,9 @@ static int fts5ExprNodeNext_NOT( if( rc==SQLITE_OK ){ rc = fts5ExprNodeTest_NOT(pExpr, pNode); } + if( rc!=SQLITE_OK ){ + pNode->bNomatch = 0; + } return rc; } @@ -1601,7 +1612,7 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm( rc = fts5ParseStringFromToken(pToken, &z); if( rc==SQLITE_OK ){ - int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_QUERY : 0); + int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_PREFIX : 0); int n; sqlite3Fts5Dequote(z); n = (int)strlen(z); diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index a695887458..c8c26bd1b6 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -545,11 +545,6 @@ int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){ } } - /* Write the averages record */ - if( rc==SQLITE_OK ){ - rc = fts5StorageSaveTotals(p); - } - return rc; } @@ -753,11 +748,6 @@ int sqlite3Fts5StorageIndexInsert( } sqlite3_free(buf.p); - /* Write the averages record */ - if( rc==SQLITE_OK ){ - rc = fts5StorageSaveTotals(p); - } - return rc; } @@ -1092,12 +1082,17 @@ int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow){ ** Flush any data currently held in-memory to disk. */ int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit){ - if( bCommit && p->bTotalsValid ){ - int rc = fts5StorageSaveTotals(p); - p->bTotalsValid = 0; - if( rc!=SQLITE_OK ) return rc; + int rc = SQLITE_OK; + i64 iLastRowid = sqlite3_last_insert_rowid(p->pConfig->db); + if( p->bTotalsValid ){ + rc = fts5StorageSaveTotals(p); + if( bCommit ) p->bTotalsValid = 0; } - return sqlite3Fts5IndexSync(p->pIndex, bCommit); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IndexSync(p->pIndex, bCommit); + } + sqlite3_set_last_insert_rowid(p->pConfig->db, iLastRowid); + return rc; } int sqlite3Fts5StorageRollback(Fts5Storage *p){ diff --git a/ext/fts5/test/fts5faultD.test b/ext/fts5/test/fts5faultD.test new file mode 100644 index 0000000000..e259cbf610 --- /dev/null +++ b/ext/fts5/test/fts5faultD.test @@ -0,0 +1,87 @@ +# 2016 February 2 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# This file is focused on OOM errors. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5faultA + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +foreach_detail_mode $testprefix { + if {"%DETAIL%"=="none"} continue + + do_execsql_test 1.0 { + CREATE VIRTUAL TABLE o1 USING fts5(a, b, c, detail=%DETAIL%); + INSERT INTO o1(o1, rank) VALUES('pgsz', 32); + + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<60 ) + INSERT INTO o1 SELECT 'A', 'B', 'C' FROM s; + + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<60 ) + INSERT INTO o1 SELECT 'C', 'A', 'B' FROM s; + + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<60 ) + INSERT INTO o1 SELECT 'B', 'C', 'A' FROM s; + } + + do_faultsim_test 1 -faults int* -prep { + sqlite3 db test.db + } -body { + execsql { SELECT count(*) FROM o1('a') } + } -test { + faultsim_test_result {0 180} {1 {vtable constructor failed: o1}} + } + + do_faultsim_test 2 -faults int* -prep { + sqlite3 db test.db + } -body { + execsql { SELECT * FROM o1('a:a AND {b c}:b') ORDER BY rank } + expr 1 + } -test { + faultsim_test_result {0 1} {1 {vtable constructor failed: o1}} + } + + do_faultsim_test 3 -faults int* -prep { + sqlite3 db test.db + } -body { + execsql { SELECT * FROM o1('{b c}:b NOT a:a') ORDER BY rank } + expr 1 + } -test { + faultsim_test_result {0 1} {1 {vtable constructor failed: o1}} + } + + do_faultsim_test 4 -faults int* -prep { + sqlite3 db test.db + } -body { + execsql { SELECT * FROM o1('b:b OR a:a') } + expr 1 + } -test { + faultsim_test_result {0 1} {1 {vtable constructor failed: o1}} + } + + do_faultsim_test 5 -faults int* -prep { + sqlite3 db test.db + } -body { + execsql { SELECT count(*) FROM o1('c:b') } + expr 1 + } -test { + faultsim_test_result {0 1} {1 {vtable constructor failed: o1}} + } +} + +finish_test diff --git a/ext/fts5/test/fts5lastrowid.test b/ext/fts5/test/fts5lastrowid.test new file mode 100644 index 0000000000..0f7628b233 --- /dev/null +++ b/ext/fts5/test/fts5lastrowid.test @@ -0,0 +1,73 @@ +# 2017 Feb 27 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests of the last_insert_rowid functionality with fts5. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5lastrowid + +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(str); +} + +do_execsql_test 1.1 { + INSERT INTO t1 VALUES('one string'); + INSERT INTO t1 VALUES('two string'); + INSERT INTO t1 VALUES('three string'); + SELECT last_insert_rowid(); +} {3} + +do_execsql_test 1.2 { + BEGIN; + INSERT INTO t1 VALUES('one string'); + INSERT INTO t1 VALUES('two string'); + INSERT INTO t1 VALUES('three string'); + COMMIT; + SELECT last_insert_rowid(); +} {6} + +do_execsql_test 1.3 { + INSERT INTO t1(rowid, str) VALUES(-22, 'some more text'); + SELECT last_insert_rowid(); +} {-22} + +do_execsql_test 1.4 { + BEGIN; + INSERT INTO t1(rowid, str) VALUES(45, 'some more text'); + INSERT INTO t1(rowid, str) VALUES(46, 'some more text'); + INSERT INTO t1(rowid, str) VALUES(222, 'some more text'); + SELECT last_insert_rowid(); + COMMIT; + SELECT last_insert_rowid(); +} {222 222} + +do_execsql_test 1.5 { + CREATE TABLE x1(x); + INSERT INTO x1 VALUES('john'), ('paul'), ('george'), ('ringo'); + INSERT INTO t1 SELECT x FROM x1; + SELECT last_insert_rowid(); +} {226} + +do_execsql_test 1.6 { + INSERT INTO t1(rowid, str) SELECT rowid+10, x FROM x1; + SELECT last_insert_rowid(); +} {14} + + +finish_test + diff --git a/ext/fts5/test/fts5synonym.test b/ext/fts5/test/fts5synonym.test index 185dda3ff4..6243663123 100644 --- a/ext/fts5/test/fts5synonym.test +++ b/ext/fts5/test/fts5synonym.test @@ -152,7 +152,7 @@ foreach {tn expr res} { 1 {abc} {"abc"} 2 {one} {"one"|"i"|"1"} 3 {3} {"3"|"iii"|"three"} - 4 {3*} {"3"|"iii"|"three" *} + 4 {3*} {"3" *} } { do_execsql_test 4.1.$tn { SELECT fts5_expr($expr, 'tokenize=tclnum') diff --git a/ext/fts5/test/fts5tokenizer.test b/ext/fts5/test/fts5tokenizer.test index 9316d3c234..f68aa19a10 100644 --- a/ext/fts5/test/fts5tokenizer.test +++ b/ext/fts5/test/fts5tokenizer.test @@ -262,5 +262,44 @@ do_execsql_test 8.3 { brown dog fox jump lazi over quick the } +#------------------------------------------------------------------------- +# Check that the FTS5_TOKENIZE_PREFIX flag is passed to the tokenizer +# implementation. +# +reset_db +proc tcl_create {args} { return "tcl_tokenize" } +sqlite3_fts5_create_tokenizer db tcl tcl_create +set ::flags [list] +proc tcl_tokenize {tflags text} { + lappend ::flags $tflags + foreach {w iStart iEnd} [fts5_tokenize_split $text] { + sqlite3_fts5_token $w $iStart $iEnd + } +} + +do_execsql_test 9.1.1 { + CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=tcl); + INSERT INTO t1 VALUES('abc'); + INSERT INTO t1 VALUES('xyz'); +} {} +do_test 9.1.2 { set ::flags } {document document} + +set ::flags [list] +do_execsql_test 9.2.1 { SELECT * FROM t1('abc'); } {abc} +do_test 9.2.2 { set ::flags } {query} + +set ::flags [list] +do_execsql_test 9.3.1 { SELECT * FROM t1('ab*'); } {abc} +do_test 9.3.2 { set ::flags } {prefixquery} + +set ::flags [list] +do_execsql_test 9.4.1 { SELECT * FROM t1('"abc xyz" *'); } {} +do_test 9.4.2 { set ::flags } {prefixquery} + +set ::flags [list] +do_execsql_test 9.5.1 { SELECT * FROM t1('"abc xyz*"'); } {} +do_test 9.5.2 { set ::flags } {query} + + finish_test diff --git a/ext/icu/icu.c b/ext/icu/icu.c index d2beaa3353..7c37812d86 100644 --- a/ext/icu/icu.c +++ b/ext/icu/icu.c @@ -493,38 +493,36 @@ static void icuLoadCollation( ** Register the ICU extension functions with database db. */ int sqlite3IcuInit(sqlite3 *db){ - struct IcuScalar { + static const struct IcuScalar { const char *zName; /* Function name */ - int nArg; /* Number of arguments */ - int enc; /* Optimal text encoding */ - void *pContext; /* sqlite3_user_data() context */ + unsigned char nArg; /* Number of arguments */ + unsigned short enc; /* Optimal text encoding */ + unsigned char iContext; /* sqlite3_user_data() context */ void (*xFunc)(sqlite3_context*,int,sqlite3_value**); } scalars[] = { - {"regexp", 2, SQLITE_ANY|SQLITE_DETERMINISTIC, 0, icuRegexpFunc}, - - {"lower", 1, SQLITE_UTF16|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, - {"lower", 2, SQLITE_UTF16|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, - {"upper", 1, SQLITE_UTF16|SQLITE_DETERMINISTIC, (void*)1, icuCaseFunc16}, - {"upper", 2, SQLITE_UTF16|SQLITE_DETERMINISTIC, (void*)1, icuCaseFunc16}, - - {"lower", 1, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, - {"lower", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, - {"upper", 1, SQLITE_UTF8|SQLITE_DETERMINISTIC, (void*)1, icuCaseFunc16}, - {"upper", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, (void*)1, icuCaseFunc16}, - - {"like", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuLikeFunc}, - {"like", 3, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuLikeFunc}, - - {"icu_load_collation", 2, SQLITE_UTF8, (void*)db, icuLoadCollation}, + {"icu_load_collation", 2, SQLITE_UTF8, 1, icuLoadCollation}, + {"regexp", 2, SQLITE_ANY|SQLITE_DETERMINISTIC, 0, icuRegexpFunc}, + {"lower", 1, SQLITE_UTF16|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, + {"lower", 2, SQLITE_UTF16|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, + {"upper", 1, SQLITE_UTF16|SQLITE_DETERMINISTIC, 1, icuCaseFunc16}, + {"upper", 2, SQLITE_UTF16|SQLITE_DETERMINISTIC, 1, icuCaseFunc16}, + {"lower", 1, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, + {"lower", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, + {"upper", 1, SQLITE_UTF8|SQLITE_DETERMINISTIC, 1, icuCaseFunc16}, + {"upper", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 1, icuCaseFunc16}, + {"like", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuLikeFunc}, + {"like", 3, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuLikeFunc}, }; - int rc = SQLITE_OK; int i; + for(i=0; rc==SQLITE_OK && i<(int)(sizeof(scalars)/sizeof(scalars[0])); i++){ - struct IcuScalar *p = &scalars[i]; + const struct IcuScalar *p = &scalars[i]; rc = sqlite3_create_function( - db, p->zName, p->nArg, p->enc, p->pContext, p->xFunc, 0, 0 + db, p->zName, p->nArg, p->enc, + p->iContext ? (void*)db : (void*)0, + p->xFunc, 0, 0 ); } diff --git a/ext/misc/README.md b/ext/misc/README.md new file mode 100644 index 0000000000..970a09c2e9 --- /dev/null +++ b/ext/misc/README.md @@ -0,0 +1,40 @@ +## Miscellaneous Extensions + +This folder contains a collection of smaller loadable extensions. +See for instructions on how +to compile and use loadable extensions. +Each extension in this folder is implemented in a single file of C code. + +Each source file contains a description in its header comment. See the +header comments for details about each extension. Additional notes are +as follows: + + * **carray.c** — This module implements the + [carray](https://www.sqlite.org/carray.html) table-valued function. + It is a good example of how to go about implementing a custom + [table-valued function](https://www.sqlite.org/vtab.html#tabfunc2). + + * **dbdump.c** — This is not actually a loadable extension, but + rather a library that implements an approximate equivalent to the + ".dump" command of the + [command-line shell](https://www.sqlite.org/cli.html). + + * **memvfs.c** — This file implements a custom + [VFS](https://www.sqlite.org/vfs.html) that stores an entire database + file in a single block of RAM. It serves as a good example of how + to implement a simple custom VFS. + + * **rot13.c** — This file implements the very simple rot13() + substitution function. This file makes a good template for implementing + new custom SQL functions for SQLite. + + * **series.c** — This is an implementation of the + "generate_series" [virtual table](https://www.sqlite.org/vtab.html). + It can make a good template for new custom virtual table implementations. + + * **shathree.c** — An implementation of the sha3() and + sha3_query() SQL functions. The file is named "shathree.c" instead + of "sha3.c" because the default entry point names in SQLite are based + on the source filename with digits removed, so if we used the name + "sha3.c" then the entry point would conflict with the prior "sha1.c" + extension. diff --git a/ext/misc/dbdump.c b/ext/misc/dbdump.c new file mode 100644 index 0000000000..db581a8325 --- /dev/null +++ b/ext/misc/dbdump.c @@ -0,0 +1,717 @@ +/* +** 2016-03-13 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file implements a C-language subroutine that converts the content +** of an SQLite database into UTF-8 text SQL statements that can be used +** to exactly recreate the original database. ROWID values are preserved. +** +** A prototype of the implemented subroutine is this: +** +** int sqlite3_db_dump( +** sqlite3 *db, +** const char *zSchema, +** const char *zTable, +** void (*xCallback)(void*, const char*), +** void *pArg +** ); +** +** The db parameter is the database connection. zSchema is the schema within +** that database which is to be dumped. Usually the zSchema is "main" but +** can also be "temp" or any ATTACH-ed database. If zTable is not NULL, then +** only the content of that one table is dumped. If zTable is NULL, then all +** tables are dumped. +** +** The generate text is passed to xCallback() in multiple calls. The second +** argument to xCallback() is a copy of the pArg parameter. The first +** argument is some of the output text that this routine generates. The +** signature to xCallback() is designed to make it compatible with fputs(). +** +** The sqlite3_db_dump() subroutine returns SQLITE_OK on success or some error +** code if it encounters a problem. +** +** If this file is compiled with -DDBDUMP_STANDALONE then a "main()" routine +** is included so that this routine becomes a command-line utility. The +** command-line utility takes two or three arguments which are the name +** of the database file, the schema, and optionally the table, forming the +** first three arguments of a single call to the library routine. +*/ +#include "sqlite3.h" +#include +#include +#include + +/* +** The state of the dump process. +*/ +typedef struct DState DState; +struct DState { + sqlite3 *db; /* The database connection */ + int nErr; /* Number of errors seen so far */ + int rc; /* Error code */ + int writableSchema; /* True if in writable_schema mode */ + int (*xCallback)(const char*,void*); /* Send output here */ + void *pArg; /* Argument to xCallback() */ +}; + +/* +** A variable length string to which one can append text. +*/ +typedef struct DText DText; +struct DText { + char *z; /* The text */ + int n; /* Number of bytes of content in z[] */ + int nAlloc; /* Number of bytes allocated to z[] */ +}; + +/* +** Initialize and destroy a DText object +*/ +static void initText(DText *p){ + memset(p, 0, sizeof(*p)); +} +static void freeText(DText *p){ + sqlite3_free(p->z); + initText(p); +} + +/* zIn is either a pointer to a NULL-terminated string in memory obtained +** from malloc(), or a NULL pointer. The string pointed to by zAppend is +** added to zIn, and the result returned in memory obtained from malloc(). +** zIn, if it was not NULL, is freed. +** +** If the third argument, quote, is not '\0', then it is used as a +** quote character for zAppend. +*/ +static void appendText(DText *p, char const *zAppend, char quote){ + int len; + int i; + int nAppend = (int)(strlen(zAppend) & 0x3fffffff); + + len = nAppend+p->n+1; + if( quote ){ + len += 2; + for(i=0; in+len>=p->nAlloc ){ + char *zNew; + p->nAlloc = p->nAlloc*2 + len + 20; + zNew = sqlite3_realloc(p->z, p->nAlloc); + if( zNew==0 ){ + freeText(p); + return; + } + p->z = zNew; + } + + if( quote ){ + char *zCsr = p->z+p->n; + *zCsr++ = quote; + for(i=0; in = (int)(zCsr - p->z); + *zCsr = '\0'; + }else{ + memcpy(p->z+p->n, zAppend, nAppend); + p->n += nAppend; + p->z[p->n] = '\0'; + } +} + +/* +** Attempt to determine if identifier zName needs to be quoted, either +** because it contains non-alphanumeric characters, or because it is an +** SQLite keyword. Be conservative in this estimate: When in doubt assume +** that quoting is required. +** +** Return '"' if quoting is required. Return 0 if no quoting is required. +*/ +static char quoteChar(const char *zName){ + /* All SQLite keywords, in alphabetical order */ + static const char *azKeywords[] = { + "ABORT", "ACTION", "ADD", "AFTER", "ALL", "ALTER", "ANALYZE", "AND", "AS", + "ASC", "ATTACH", "AUTOINCREMENT", "BEFORE", "BEGIN", "BETWEEN", "BY", + "CASCADE", "CASE", "CAST", "CHECK", "COLLATE", "COLUMN", "COMMIT", + "CONFLICT", "CONSTRAINT", "CREATE", "CROSS", "CURRENT_DATE", + "CURRENT_TIME", "CURRENT_TIMESTAMP", "DATABASE", "DEFAULT", "DEFERRABLE", + "DEFERRED", "DELETE", "DESC", "DETACH", "DISTINCT", "DROP", "EACH", + "ELSE", "END", "ESCAPE", "EXCEPT", "EXCLUSIVE", "EXISTS", "EXPLAIN", + "FAIL", "FOR", "FOREIGN", "FROM", "FULL", "GLOB", "GROUP", "HAVING", "IF", + "IGNORE", "IMMEDIATE", "IN", "INDEX", "INDEXED", "INITIALLY", "INNER", + "INSERT", "INSTEAD", "INTERSECT", "INTO", "IS", "ISNULL", "JOIN", "KEY", + "LEFT", "LIKE", "LIMIT", "MATCH", "NATURAL", "NO", "NOT", "NOTNULL", + "NULL", "OF", "OFFSET", "ON", "OR", "ORDER", "OUTER", "PLAN", "PRAGMA", + "PRIMARY", "QUERY", "RAISE", "RECURSIVE", "REFERENCES", "REGEXP", + "REINDEX", "RELEASE", "RENAME", "REPLACE", "RESTRICT", "RIGHT", + "ROLLBACK", "ROW", "SAVEPOINT", "SELECT", "SET", "TABLE", "TEMP", + "TEMPORARY", "THEN", "TO", "TRANSACTION", "TRIGGER", "UNION", "UNIQUE", + "UPDATE", "USING", "VACUUM", "VALUES", "VIEW", "VIRTUAL", "WHEN", "WHERE", + "WITH", "WITHOUT", + }; + int i, lwr, upr, mid, c; + if( !isalpha((unsigned char)zName[0]) && zName[0]!='_' ) return '"'; + for(i=0; zName[i]; i++){ + if( !isalnum((unsigned char)zName[i]) && zName[i]!='_' ) return '"'; + } + lwr = 0; + upr = sizeof(azKeywords)/sizeof(azKeywords[0]) - 1; + while( lwr<=upr ){ + mid = (lwr+upr)/2; + c = sqlite3_stricmp(azKeywords[mid], zName); + if( c==0 ) return '"'; + if( c<0 ){ + lwr = mid+1; + }else{ + upr = mid-1; + } + } + return 0; +} + + +/* +** Release memory previously allocated by tableColumnList(). +*/ +static void freeColumnList(char **azCol){ + int i; + for(i=1; azCol[i]; i++){ + sqlite3_free(azCol[i]); + } + /* azCol[0] is a static string */ + sqlite3_free(azCol); +} + +/* +** Return a list of pointers to strings which are the names of all +** columns in table zTab. The memory to hold the names is dynamically +** allocated and must be released by the caller using a subsequent call +** to freeColumnList(). +** +** The azCol[0] entry is usually NULL. However, if zTab contains a rowid +** value that needs to be preserved, then azCol[0] is filled in with the +** name of the rowid column. +** +** The first regular column in the table is azCol[1]. The list is terminated +** by an entry with azCol[i]==0. +*/ +static char **tableColumnList(DState *p, const char *zTab){ + char **azCol = 0; + sqlite3_stmt *pStmt = 0; + char *zSql; + int nCol = 0; + int nAlloc = 0; + int nPK = 0; /* Number of PRIMARY KEY columns seen */ + int isIPK = 0; /* True if one PRIMARY KEY column of type INTEGER */ + int preserveRowid = 1; + int rc; + + zSql = sqlite3_mprintf("PRAGMA table_info=%Q", zTab); + if( zSql==0 ) return 0; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc ) return 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + if( nCol>=nAlloc-2 ){ + char **azNew; + nAlloc = nAlloc*2 + nCol + 10; + azNew = sqlite3_realloc(azCol, nAlloc*sizeof(azCol[0])); + if( azNew==0 ) goto col_oom; + azCol = azNew; + azCol[0] = 0; + } + azCol[++nCol] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1)); + if( azCol[nCol]==0 ) goto col_oom; + if( sqlite3_column_int(pStmt, 5) ){ + nPK++; + if( nPK==1 + && sqlite3_stricmp((const char*)sqlite3_column_text(pStmt,2), + "INTEGER")==0 + ){ + isIPK = 1; + }else{ + isIPK = 0; + } + } + } + sqlite3_finalize(pStmt); + pStmt = 0; + azCol[nCol+1] = 0; + + /* The decision of whether or not a rowid really needs to be preserved + ** is tricky. We never need to preserve a rowid for a WITHOUT ROWID table + ** or a table with an INTEGER PRIMARY KEY. We are unable to preserve + ** rowids on tables where the rowid is inaccessible because there are other + ** columns in the table named "rowid", "_rowid_", and "oid". + */ + if( isIPK ){ + /* If a single PRIMARY KEY column with type INTEGER was seen, then it + ** might be an alise for the ROWID. But it might also be a WITHOUT ROWID + ** table or a INTEGER PRIMARY KEY DESC column, neither of which are + ** ROWID aliases. To distinguish these cases, check to see if + ** there is a "pk" entry in "PRAGMA index_list". There will be + ** no "pk" index if the PRIMARY KEY really is an alias for the ROWID. + */ + zSql = sqlite3_mprintf("SELECT 1 FROM pragma_index_list(%Q)" + " WHERE origin='pk'", zTab); + if( zSql==0 ) goto col_oom; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc ){ + freeColumnList(azCol); + return 0; + } + rc = sqlite3_step(pStmt); + sqlite3_finalize(pStmt); + pStmt = 0; + preserveRowid = rc==SQLITE_ROW; + } + if( preserveRowid ){ + /* Only preserve the rowid if we can find a name to use for the + ** rowid */ + static char *azRowid[] = { "rowid", "_rowid_", "oid" }; + int i, j; + for(j=0; j<3; j++){ + for(i=1; i<=nCol; i++){ + if( sqlite3_stricmp(azRowid[j],azCol[i])==0 ) break; + } + if( i>nCol ){ + /* At this point, we know that azRowid[j] is not the name of any + ** ordinary column in the table. Verify that azRowid[j] is a valid + ** name for the rowid before adding it to azCol[0]. WITHOUT ROWID + ** tables will fail this last check */ + int rc; + rc = sqlite3_table_column_metadata(p->db,0,zTab,azRowid[j],0,0,0,0,0); + if( rc==SQLITE_OK ) azCol[0] = azRowid[j]; + break; + } + } + } + return azCol; + +col_oom: + sqlite3_finalize(pStmt); + freeColumnList(azCol); + p->nErr++; + p->rc = SQLITE_NOMEM; + return 0; +} + +/* +** Send mprintf-formatted content to the output callback. +*/ +static void output_formatted(DState *p, const char *zFormat, ...){ + va_list ap; + char *z; + va_start(ap, zFormat); + z = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + p->xCallback(z, p->pArg); + sqlite3_free(z); +} + +/* +** Output the given string as a quoted string using SQL quoting conventions. +** +** The "\n" and "\r" characters are converted to char(10) and char(13) +** to prevent them from being transformed by end-of-line translators. +*/ +static void output_quoted_string(DState *p, const unsigned char *z){ + int i; + char c; + int inQuote = 0; + int bStarted = 0; + + for(i=0; (c = z[i])!=0 && c!='\'' && c!='\n' && c!='\r'; i++){} + if( c==0 ){ + output_formatted(p, "'%s'", z); + return; + } + while( *z ){ + for(i=0; (c = z[i])!=0 && c!='\n' && c!='\r' && c!='\''; i++){} + if( c=='\'' ) i++; + if( i ){ + if( !inQuote ){ + if( bStarted ) p->xCallback("||", p->pArg); + p->xCallback("'", p->pArg); + inQuote = 1; + } + output_formatted(p, "%.*s", i, z); + z += i; + bStarted = 1; + } + if( c=='\'' ){ + p->xCallback("'", p->pArg); + continue; + } + if( inQuote ){ + p->xCallback("'", p->pArg); + inQuote = 0; + } + if( c==0 ){ + break; + } + for(i=0; (c = z[i])=='\r' || c=='\n'; i++){ + if( bStarted ) p->xCallback("||", p->pArg); + output_formatted(p, "char(%d)", c); + bStarted = 1; + } + z += i; + } + if( inQuote ) p->xCallback("'", p->pArg); +} + +/* +** This is an sqlite3_exec callback routine used for dumping the database. +** Each row received by this callback consists of a table name, +** the table type ("index" or "table") and SQL to create the table. +** This routine should print text sufficient to recreate the table. +*/ +static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ + int rc; + const char *zTable; + const char *zType; + const char *zSql; + DState *p = (DState*)pArg; + sqlite3_stmt *pStmt; + + (void)azCol; + if( nArg!=3 ) return 1; + zTable = azArg[0]; + zType = azArg[1]; + zSql = azArg[2]; + + if( strcmp(zTable, "sqlite_sequence")==0 ){ + p->xCallback("DELETE FROM sqlite_sequence;\n", p->pArg); + }else if( sqlite3_strglob("sqlite_stat?", zTable)==0 ){ + p->xCallback("ANALYZE sqlite_master;\n", p->pArg); + }else if( strncmp(zTable, "sqlite_", 7)==0 ){ + return 0; + }else if( strncmp(zSql, "CREATE VIRTUAL TABLE", 20)==0 ){ + if( !p->writableSchema ){ + p->xCallback("PRAGMA writable_schema=ON;\n", p->pArg); + p->writableSchema = 1; + } + output_formatted(p, + "INSERT INTO sqlite_master(type,name,tbl_name,rootpage,sql)" + "VALUES('table','%q','%q',0,'%q');", + zTable, zTable, zSql); + return 0; + }else{ + if( sqlite3_strglob("CREATE TABLE ['\"]*", zSql)==0 ){ + p->xCallback("CREATE TABLE IF NOT EXISTS ", p->pArg); + p->xCallback(zSql+13, p->pArg); + }else{ + p->xCallback(zSql, p->pArg); + } + p->xCallback(";\n", p->pArg); + } + + if( strcmp(zType, "table")==0 ){ + DText sSelect; + DText sTable; + char **azCol; + int i; + int nCol; + + azCol = tableColumnList(p, zTable); + if( azCol==0 ) return 0; + + initText(&sTable); + appendText(&sTable, "INSERT INTO ", 0); + + /* Always quote the table name, even if it appears to be pure ascii, + ** in case it is a keyword. Ex: INSERT INTO "table" ... */ + appendText(&sTable, zTable, quoteChar(zTable)); + + /* If preserving the rowid, add a column list after the table name. + ** In other words: "INSERT INTO tab(rowid,a,b,c,...) VALUES(...)" + ** instead of the usual "INSERT INTO tab VALUES(...)". + */ + if( azCol[0] ){ + appendText(&sTable, "(", 0); + appendText(&sTable, azCol[0], 0); + for(i=1; azCol[i]; i++){ + appendText(&sTable, ",", 0); + appendText(&sTable, azCol[i], quoteChar(azCol[i])); + } + appendText(&sTable, ")", 0); + } + appendText(&sTable, " VALUES(", 0); + + /* Build an appropriate SELECT statement */ + initText(&sSelect); + appendText(&sSelect, "SELECT ", 0); + if( azCol[0] ){ + appendText(&sSelect, azCol[0], 0); + appendText(&sSelect, ",", 0); + } + for(i=1; azCol[i]; i++){ + appendText(&sSelect, azCol[i], quoteChar(azCol[i])); + if( azCol[i+1] ){ + appendText(&sSelect, ",", 0); + } + } + nCol = i; + if( azCol[0]==0 ) nCol--; + freeColumnList(azCol); + appendText(&sSelect, " FROM ", 0); + appendText(&sSelect, zTable, quoteChar(zTable)); + + rc = sqlite3_prepare_v2(p->db, sSelect.z, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + p->nErr++; + if( p->rc==SQLITE_OK ) p->rc = rc; + }else{ + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + p->xCallback(sTable.z, p->pArg); + for(i=0; ixCallback(",", p->pArg); + switch( sqlite3_column_type(pStmt,i) ){ + case SQLITE_INTEGER: { + output_formatted(p, "%lld", sqlite3_column_int64(pStmt,i)); + break; + } + case SQLITE_FLOAT: { + double r = sqlite3_column_double(pStmt,i); + output_formatted(p, "%!.20g", r); + break; + } + case SQLITE_NULL: { + p->xCallback("NULL", p->pArg); + break; + } + case SQLITE_TEXT: { + output_quoted_string(p, sqlite3_column_text(pStmt,i)); + break; + } + case SQLITE_BLOB: { + int nByte = sqlite3_column_bytes(pStmt,i); + unsigned char *a = (unsigned char*)sqlite3_column_blob(pStmt,i); + int j; + p->xCallback("x'", p->pArg); + for(j=0; j>4)&15]; + zWord[1] = "0123456789abcdef"[a[j]&15]; + zWord[2] = 0; + p->xCallback(zWord, p->pArg); + } + p->xCallback("'", p->pArg); + break; + } + } + } + p->xCallback(");\n", p->pArg); + } + } + sqlite3_finalize(pStmt); + freeText(&sTable); + freeText(&sSelect); + } + return 0; +} + + +/* +** Execute a query statement that will generate SQL output. Print +** the result columns, comma-separated, on a line and then add a +** semicolon terminator to the end of that line. +** +** If the number of columns is 1 and that column contains text "--" +** then write the semicolon on a separate line. That way, if a +** "--" comment occurs at the end of the statement, the comment +** won't consume the semicolon terminator. +*/ +static void output_sql_from_query( + DState *p, /* Query context */ + const char *zSelect, /* SELECT statement to extract content */ + ... +){ + sqlite3_stmt *pSelect; + int rc; + int nResult; + int i; + const char *z; + char *zSql; + va_list ap; + va_start(ap, zSelect); + zSql = sqlite3_vmprintf(zSelect, ap); + va_end(ap); + if( zSql==0 ){ + p->rc = SQLITE_NOMEM; + p->nErr++; + return; + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pSelect, 0); + sqlite3_free(zSql); + if( rc!=SQLITE_OK || !pSelect ){ + output_formatted(p, "/**** ERROR: (%d) %s *****/\n", rc, + sqlite3_errmsg(p->db)); + p->nErr++; + return; + } + rc = sqlite3_step(pSelect); + nResult = sqlite3_column_count(pSelect); + while( rc==SQLITE_ROW ){ + z = (const char*)sqlite3_column_text(pSelect, 0); + p->xCallback(z, p->pArg); + for(i=1; ixCallback(",", p->pArg); + p->xCallback((const char*)sqlite3_column_text(pSelect,i), p->pArg); + } + if( z==0 ) z = ""; + while( z[0] && (z[0]!='-' || z[1]!='-') ) z++; + if( z[0] ){ + p->xCallback("\n;\n", p->pArg); + }else{ + p->xCallback(";\n", p->pArg); + } + rc = sqlite3_step(pSelect); + } + rc = sqlite3_finalize(pSelect); + if( rc!=SQLITE_OK ){ + output_formatted(p, "/**** ERROR: (%d) %s *****/\n", rc, + sqlite3_errmsg(p->db)); + if( (rc&0xff)!=SQLITE_CORRUPT ) p->nErr++; + } +} + +/* +** Run zQuery. Use dump_callback() as the callback routine so that +** the contents of the query are output as SQL statements. +** +** If we get a SQLITE_CORRUPT error, rerun the query after appending +** "ORDER BY rowid DESC" to the end. +*/ +static void run_schema_dump_query( + DState *p, + const char *zQuery, + ... +){ + char *zErr = 0; + char *z; + va_list ap; + va_start(ap, zQuery); + z = sqlite3_vmprintf(zQuery, ap); + va_end(ap); + sqlite3_exec(p->db, z, dump_callback, p, &zErr); + sqlite3_free(z); + if( zErr ){ + output_formatted(p, "/****** %s ******/\n", zErr); + sqlite3_free(zErr); + p->nErr++; + zErr = 0; + } +} + +/* +** Convert an SQLite database into SQL statements that will recreate that +** database. +*/ +int sqlite3_db_dump( + sqlite3 *db, /* The database connection */ + const char *zSchema, /* Which schema to dump. Usually "main". */ + const char *zTable, /* Which table to dump. NULL means everything. */ + int (*xCallback)(const char*,void*), /* Output sent to this callback */ + void *pArg /* Second argument of the callback */ +){ + DState x; + memset(&x, 0, sizeof(x)); + x.rc = sqlite3_exec(db, "BEGIN", 0, 0, 0); + if( x.rc ) return x.rc; + x.db = db; + x.xCallback = xCallback; + x.pArg = pArg; + xCallback("PRAGMA foreign_keys=OFF;\nBEGIN TRANSACTION;\n", pArg); + if( zTable==0 ){ + run_schema_dump_query(&x, + "SELECT name, type, sql FROM \"%w\".sqlite_master " + "WHERE sql NOT NULL AND type=='table' AND name!='sqlite_sequence'", + zSchema + ); + run_schema_dump_query(&x, + "SELECT name, type, sql FROM \"%w\".sqlite_master " + "WHERE name=='sqlite_sequence'", zSchema + ); + output_sql_from_query(&x, + "SELECT sql FROM sqlite_master " + "WHERE sql NOT NULL AND type IN ('index','trigger','view')", 0 + ); + }else{ + run_schema_dump_query(&x, + "SELECT name, type, sql FROM \"%w\".sqlite_master " + "WHERE tbl_name=%Q COLLATE nocase AND type=='table'" + " AND sql NOT NULL", + zSchema, zTable + ); + output_sql_from_query(&x, + "SELECT sql FROM \"%w\".sqlite_master " + "WHERE sql NOT NULL" + " AND type IN ('index','trigger','view')" + " AND tbl_name=%Q COLLATE nocase", + zSchema, zTable + ); + } + if( x.writableSchema ){ + xCallback("PRAGMA writable_schema=OFF;\n", pArg); + } + xCallback(x.nErr ? "ROLLBACK; -- due to errors\n" : "COMMIT;\n", pArg); + sqlite3_exec(db, "COMMIT", 0, 0, 0); + return x.rc; +} + + + +/* The generic subroutine is above. The code the follows implements +** the command-line interface. +*/ +#ifdef DBDUMP_STANDALONE +#include + +/* +** Command-line interface +*/ +int main(int argc, char **argv){ + sqlite3 *db; + const char *zDb; + const char *zSchema; + const char *zTable = 0; + int rc; + + if( argc<2 || argc>4 ){ + fprintf(stderr, "Usage: %s DATABASE ?SCHEMA? ?TABLE?\n", argv[0]); + return 1; + } + zDb = argv[1]; + zSchema = argc>=3 ? argv[2] : "main"; + zTable = argc==4 ? argv[3] : 0; + + rc = sqlite3_open(zDb, &db); + if( rc ){ + fprintf(stderr, "Cannot open \"%s\": %s\n", zDb, sqlite3_errmsg(db)); + sqlite3_close(db); + return 1; + } + rc = sqlite3_db_dump(db, zSchema, zTable, + (int(*)(const char*,void*))fputs, (void*)stdout); + if( rc ){ + fprintf(stderr, "Error: sqlite3_db_dump() returns %d\n", rc); + } + sqlite3_close(db); + return rc!=SQLITE_OK; +} +#endif /* DBDUMP_STANDALONE */ diff --git a/ext/misc/json1.c b/ext/misc/json1.c index 327ad57500..2689726302 100644 --- a/ext/misc/json1.c +++ b/ext/misc/json1.c @@ -22,7 +22,7 @@ ** how JSONB might improve on that.) */ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_JSON1) -#if !defined(_SQLITEINT_H_) +#if !defined(SQLITEINT_H) #include "sqlite3ext.h" #endif SQLITE_EXTENSION_INIT1 @@ -138,9 +138,10 @@ static const char * const jsonType[] = { #define JNODE_RAW 0x01 /* Content is raw, not JSON encoded */ #define JNODE_ESCAPE 0x02 /* Content is text with \ escapes */ #define JNODE_REMOVE 0x04 /* Do not output */ -#define JNODE_REPLACE 0x08 /* Replace with JsonNode.iVal */ -#define JNODE_APPEND 0x10 /* More ARRAY/OBJECT entries at u.iAppend */ -#define JNODE_LABEL 0x20 /* Is a label of an object */ +#define JNODE_REPLACE 0x08 /* Replace with JsonNode.u.iReplace */ +#define JNODE_PATCH 0x10 /* Patch with JsonNode.u.pPatch */ +#define JNODE_APPEND 0x20 /* More ARRAY/OBJECT entries at u.iAppend */ +#define JNODE_LABEL 0x40 /* Is a label of an object */ /* A single node of parsed JSON @@ -148,12 +149,13 @@ static const char * const jsonType[] = { struct JsonNode { u8 eType; /* One of the JSON_ type values */ u8 jnFlags; /* JNODE flags */ - u8 iVal; /* Replacement value when JNODE_REPLACE */ u32 n; /* Bytes of content, or number of sub-nodes */ union { const char *zJContent; /* Content for INT, REAL, and STRING */ u32 iAppend; /* More terms for ARRAY and OBJECT */ u32 iKey; /* Key for ARRAY objects in json_tree() */ + u32 iReplace; /* Replacement content for JNODE_REPLACE */ + JsonNode *pPatch; /* Node chain of patch for JNODE_PATCH */ } u; }; @@ -410,6 +412,13 @@ static void jsonRenderNode( JsonString *pOut, /* Write JSON here */ sqlite3_value **aReplace /* Replacement values */ ){ + if( pNode->jnFlags & (JNODE_REPLACE|JNODE_PATCH) ){ + if( pNode->jnFlags & JNODE_REPLACE ){ + jsonAppendValue(pOut, aReplace[pNode->u.iReplace]); + return; + } + pNode = pNode->u.pPatch; + } switch( pNode->eType ){ default: { assert( pNode->eType==JSON_NULL ); @@ -441,12 +450,7 @@ static void jsonRenderNode( jsonAppendChar(pOut, '['); for(;;){ while( j<=pNode->n ){ - if( pNode[j].jnFlags & (JNODE_REMOVE|JNODE_REPLACE) ){ - if( pNode[j].jnFlags & JNODE_REPLACE ){ - jsonAppendSeparator(pOut); - jsonAppendValue(pOut, aReplace[pNode[j].iVal]); - } - }else{ + if( (pNode[j].jnFlags & JNODE_REMOVE)==0 ){ jsonAppendSeparator(pOut); jsonRenderNode(&pNode[j], pOut, aReplace); } @@ -468,11 +472,7 @@ static void jsonRenderNode( jsonAppendSeparator(pOut); jsonRenderNode(&pNode[j], pOut, aReplace); jsonAppendChar(pOut, ':'); - if( pNode[j+1].jnFlags & JNODE_REPLACE ){ - jsonAppendValue(pOut, aReplace[pNode[j+1].iVal]); - }else{ - jsonRenderNode(&pNode[j+1], pOut, aReplace); - } + jsonRenderNode(&pNode[j+1], pOut, aReplace); } j += 1 + jsonNodeSize(&pNode[j+1]); } @@ -699,7 +699,6 @@ static int jsonParseAddNode( p = &pParse->aNode[pParse->nNode]; p->eType = (u8)eType; p->jnFlags = 0; - p->iVal = 0; p->n = n; p->u.zJContent = zContent; return pParse->nNode++; @@ -1165,6 +1164,25 @@ static void jsonWrongNumArgs( sqlite3_free(zMsg); } +/* +** Mark all NULL entries in the Object passed in as JNODE_REMOVE. +*/ +static void jsonRemoveAllNulls(JsonNode *pNode){ + int i, n; + assert( pNode->eType==JSON_OBJECT ); + n = pNode->n; + for(i=2; i<=n; i += jsonNodeSize(&pNode[i])+1){ + switch( pNode[i].eType ){ + case JSON_NULL: + pNode[i].jnFlags |= JNODE_REMOVE; + break; + case JSON_OBJECT: + jsonRemoveAllNulls(&pNode[i]); + break; + } + } +} + /**************************************************************************** ** SQL functions used for testing and debugging @@ -1357,6 +1375,104 @@ static void jsonExtractFunc( jsonParseReset(&x); } +/* This is the RFC 7396 MergePatch algorithm. +*/ +static JsonNode *jsonMergePatch( + JsonParse *pParse, /* The JSON parser that contains the TARGET */ + int iTarget, /* Node of the TARGET in pParse */ + JsonNode *pPatch /* The PATCH */ +){ + u32 i, j; + u32 iRoot; + JsonNode *pTarget; + if( pPatch->eType!=JSON_OBJECT ){ + return pPatch; + } + assert( iTarget>=0 && iTargetnNode ); + pTarget = &pParse->aNode[iTarget]; + assert( (pPatch->jnFlags & JNODE_APPEND)==0 ); + if( pTarget->eType!=JSON_OBJECT ){ + jsonRemoveAllNulls(pPatch); + return pPatch; + } + iRoot = iTarget; + for(i=1; in; i += jsonNodeSize(&pPatch[i+1])+1){ + int nKey; + const char *zKey; + assert( pPatch[i].eType==JSON_STRING ); + assert( pPatch[i].jnFlags & JNODE_LABEL ); + nKey = pPatch[i].n; + zKey = pPatch[i].u.zJContent; + assert( (pPatch[i].jnFlags & JNODE_RAW)==0 ); + for(j=1; jn; j += jsonNodeSize(&pTarget[j+1])+1 ){ + assert( pTarget[j].eType==JSON_STRING ); + assert( pTarget[j].jnFlags & JNODE_LABEL ); + assert( (pPatch[i].jnFlags & JNODE_RAW)==0 ); + if( pTarget[j].n==nKey && strncmp(pTarget[j].u.zJContent,zKey,nKey)==0 ){ + if( pTarget[j+1].jnFlags & (JNODE_REMOVE|JNODE_PATCH) ) break; + if( pPatch[i+1].eType==JSON_NULL ){ + pTarget[j+1].jnFlags |= JNODE_REMOVE; + }else{ + JsonNode *pNew = jsonMergePatch(pParse, iTarget+j+1, &pPatch[i+1]); + if( pNew==0 ) return 0; + pTarget = &pParse->aNode[iTarget]; + if( pNew!=&pTarget[j+1] ){ + pTarget[j+1].u.pPatch = pNew; + pTarget[j+1].jnFlags |= JNODE_PATCH; + } + } + break; + } + } + if( j>=pTarget->n && pPatch[i+1].eType!=JSON_NULL ){ + int iStart, iPatch; + iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); + jsonParseAddNode(pParse, JSON_STRING, nKey, zKey); + iPatch = jsonParseAddNode(pParse, JSON_TRUE, 0, 0); + if( pParse->oom ) return 0; + jsonRemoveAllNulls(pPatch); + pTarget = &pParse->aNode[iTarget]; + pParse->aNode[iRoot].jnFlags |= JNODE_APPEND; + pParse->aNode[iRoot].u.iAppend = iStart - iRoot; + iRoot = iStart; + pParse->aNode[iPatch].jnFlags |= JNODE_PATCH; + pParse->aNode[iPatch].u.pPatch = &pPatch[i+1]; + } + } + return pTarget; +} + +/* +** Implementation of the json_mergepatch(JSON1,JSON2) function. Return a JSON +** object that is the result of running the RFC 7396 MergePatch() algorithm +** on the two arguments. +*/ +static void jsonPatchFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse x; /* The JSON that is being patched */ + JsonParse y; /* The patch */ + JsonNode *pResult; /* The result of the merge */ + + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; + if( jsonParse(&y, ctx, (const char*)sqlite3_value_text(argv[1])) ){ + jsonParseReset(&x); + return; + } + pResult = jsonMergePatch(&x, 0, y.aNode); + assert( pResult!=0 || x.oom ); + if( pResult ){ + jsonReturnJson(pResult, ctx, 0); + }else{ + sqlite3_result_error_nomem(ctx); + } + jsonParseReset(&x); + jsonParseReset(&y); +} + + /* ** Implementation of the json_object(NAME,VALUE,...) function. Return a JSON ** object that contains all name/value given in arguments. Or if any name @@ -1460,11 +1576,11 @@ static void jsonReplaceFunc( if( x.nErr ) goto replace_err; if( pNode ){ pNode->jnFlags |= (u8)JNODE_REPLACE; - pNode->iVal = (u8)(i+1); + pNode->u.iReplace = i + 1; } } if( x.aNode[0].jnFlags & JNODE_REPLACE ){ - sqlite3_result_value(ctx, argv[x.aNode[0].iVal]); + sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]); }else{ jsonReturnJson(x.aNode, ctx, argv); } @@ -1514,11 +1630,11 @@ static void jsonSetFunc( goto jsonSetDone; }else if( pNode && (bApnd || bIsSet) ){ pNode->jnFlags |= (u8)JNODE_REPLACE; - pNode->iVal = (u8)(i+1); + pNode->u.iReplace = i + 1; } } if( x.aNode[0].jnFlags & JNODE_REPLACE ){ - sqlite3_result_value(ctx, argv[x.aNode[0].iVal]); + sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]); }else{ jsonReturnJson(x.aNode, ctx, argv); } @@ -2161,6 +2277,7 @@ int sqlite3Json1Init(sqlite3 *db){ { "json_extract", -1, 0, jsonExtractFunc }, { "json_insert", -1, 0, jsonSetFunc }, { "json_object", -1, 0, jsonObjectFunc }, + { "json_patch", 2, 0, jsonPatchFunc }, { "json_quote", 1, 0, jsonQuoteFunc }, { "json_remove", -1, 0, jsonRemoveFunc }, { "json_replace", -1, 0, jsonReplaceFunc }, diff --git a/ext/misc/sha1.c b/ext/misc/sha1.c new file mode 100644 index 0000000000..e2843bdefa --- /dev/null +++ b/ext/misc/sha1.c @@ -0,0 +1,407 @@ +/* +** 2017-01-27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements a functions that compute SHA1 hashes. +** Two SQL functions are implemented: +** +** sha1(X) +** sha1_query(Y) +** +** The sha1(X) function computes the SHA1 hash of the input X, or NULL if +** X is NULL. +** +** The sha1_query(Y) function evalutes all queries in the SQL statements of Y +** and returns a hash of their results. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include +#include + +/****************************************************************************** +** The Hash Engine +*/ +/* Context for the SHA1 hash */ +typedef struct SHA1Context SHA1Context; +struct SHA1Context { + unsigned int state[5]; + unsigned int count[2]; + unsigned char buffer[64]; +}; + + +#if __GNUC__ && (defined(__i386__) || defined(__x86_64__)) +/* + * GCC by itself only generates left rotates. Use right rotates if + * possible to be kinder to dinky implementations with iterative rotate + * instructions. + */ +#define SHA_ROT(op, x, k) \ + ({ unsigned int y; asm(op " %1,%0" : "=r" (y) : "I" (k), "0" (x)); y; }) +#define rol(x,k) SHA_ROT("roll", x, k) +#define ror(x,k) SHA_ROT("rorl", x, k) + +#else +/* Generic C equivalent */ +#define SHA_ROT(x,l,r) ((x) << (l) | (x) >> (r)) +#define rol(x,k) SHA_ROT(x,k,32-(k)) +#define ror(x,k) SHA_ROT(x,32-(k),k) +#endif + + +#define blk0le(i) (block[i] = (ror(block[i],8)&0xFF00FF00) \ + |(rol(block[i],8)&0x00FF00FF)) +#define blk0be(i) block[i] +#define blk(i) (block[i&15] = rol(block[(i+13)&15]^block[(i+8)&15] \ + ^block[(i+2)&15]^block[i&15],1)) + +/* + * (R0+R1), R2, R3, R4 are the different operations (rounds) used in SHA1 + * + * Rl0() for little-endian and Rb0() for big-endian. Endianness is + * determined at run-time. + */ +#define Rl0(v,w,x,y,z,i) \ + z+=((w&(x^y))^y)+blk0le(i)+0x5A827999+rol(v,5);w=ror(w,2); +#define Rb0(v,w,x,y,z,i) \ + z+=((w&(x^y))^y)+blk0be(i)+0x5A827999+rol(v,5);w=ror(w,2); +#define R1(v,w,x,y,z,i) \ + z+=((w&(x^y))^y)+blk(i)+0x5A827999+rol(v,5);w=ror(w,2); +#define R2(v,w,x,y,z,i) \ + z+=(w^x^y)+blk(i)+0x6ED9EBA1+rol(v,5);w=ror(w,2); +#define R3(v,w,x,y,z,i) \ + z+=(((w|x)&y)|(w&x))+blk(i)+0x8F1BBCDC+rol(v,5);w=ror(w,2); +#define R4(v,w,x,y,z,i) \ + z+=(w^x^y)+blk(i)+0xCA62C1D6+rol(v,5);w=ror(w,2); + +/* + * Hash a single 512-bit block. This is the core of the algorithm. + */ +void SHA1Transform(unsigned int state[5], const unsigned char buffer[64]){ + unsigned int qq[5]; /* a, b, c, d, e; */ + static int one = 1; + unsigned int block[16]; + memcpy(block, buffer, 64); + memcpy(qq,state,5*sizeof(unsigned int)); + +#define a qq[0] +#define b qq[1] +#define c qq[2] +#define d qq[3] +#define e qq[4] + + /* Copy p->state[] to working vars */ + /* + a = state[0]; + b = state[1]; + c = state[2]; + d = state[3]; + e = state[4]; + */ + + /* 4 rounds of 20 operations each. Loop unrolled. */ + if( 1 == *(unsigned char*)&one ){ + Rl0(a,b,c,d,e, 0); Rl0(e,a,b,c,d, 1); Rl0(d,e,a,b,c, 2); Rl0(c,d,e,a,b, 3); + Rl0(b,c,d,e,a, 4); Rl0(a,b,c,d,e, 5); Rl0(e,a,b,c,d, 6); Rl0(d,e,a,b,c, 7); + Rl0(c,d,e,a,b, 8); Rl0(b,c,d,e,a, 9); Rl0(a,b,c,d,e,10); Rl0(e,a,b,c,d,11); + Rl0(d,e,a,b,c,12); Rl0(c,d,e,a,b,13); Rl0(b,c,d,e,a,14); Rl0(a,b,c,d,e,15); + }else{ + Rb0(a,b,c,d,e, 0); Rb0(e,a,b,c,d, 1); Rb0(d,e,a,b,c, 2); Rb0(c,d,e,a,b, 3); + Rb0(b,c,d,e,a, 4); Rb0(a,b,c,d,e, 5); Rb0(e,a,b,c,d, 6); Rb0(d,e,a,b,c, 7); + Rb0(c,d,e,a,b, 8); Rb0(b,c,d,e,a, 9); Rb0(a,b,c,d,e,10); Rb0(e,a,b,c,d,11); + Rb0(d,e,a,b,c,12); Rb0(c,d,e,a,b,13); Rb0(b,c,d,e,a,14); Rb0(a,b,c,d,e,15); + } + R1(e,a,b,c,d,16); R1(d,e,a,b,c,17); R1(c,d,e,a,b,18); R1(b,c,d,e,a,19); + R2(a,b,c,d,e,20); R2(e,a,b,c,d,21); R2(d,e,a,b,c,22); R2(c,d,e,a,b,23); + R2(b,c,d,e,a,24); R2(a,b,c,d,e,25); R2(e,a,b,c,d,26); R2(d,e,a,b,c,27); + R2(c,d,e,a,b,28); R2(b,c,d,e,a,29); R2(a,b,c,d,e,30); R2(e,a,b,c,d,31); + R2(d,e,a,b,c,32); R2(c,d,e,a,b,33); R2(b,c,d,e,a,34); R2(a,b,c,d,e,35); + R2(e,a,b,c,d,36); R2(d,e,a,b,c,37); R2(c,d,e,a,b,38); R2(b,c,d,e,a,39); + R3(a,b,c,d,e,40); R3(e,a,b,c,d,41); R3(d,e,a,b,c,42); R3(c,d,e,a,b,43); + R3(b,c,d,e,a,44); R3(a,b,c,d,e,45); R3(e,a,b,c,d,46); R3(d,e,a,b,c,47); + R3(c,d,e,a,b,48); R3(b,c,d,e,a,49); R3(a,b,c,d,e,50); R3(e,a,b,c,d,51); + R3(d,e,a,b,c,52); R3(c,d,e,a,b,53); R3(b,c,d,e,a,54); R3(a,b,c,d,e,55); + R3(e,a,b,c,d,56); R3(d,e,a,b,c,57); R3(c,d,e,a,b,58); R3(b,c,d,e,a,59); + R4(a,b,c,d,e,60); R4(e,a,b,c,d,61); R4(d,e,a,b,c,62); R4(c,d,e,a,b,63); + R4(b,c,d,e,a,64); R4(a,b,c,d,e,65); R4(e,a,b,c,d,66); R4(d,e,a,b,c,67); + R4(c,d,e,a,b,68); R4(b,c,d,e,a,69); R4(a,b,c,d,e,70); R4(e,a,b,c,d,71); + R4(d,e,a,b,c,72); R4(c,d,e,a,b,73); R4(b,c,d,e,a,74); R4(a,b,c,d,e,75); + R4(e,a,b,c,d,76); R4(d,e,a,b,c,77); R4(c,d,e,a,b,78); R4(b,c,d,e,a,79); + + /* Add the working vars back into context.state[] */ + state[0] += a; + state[1] += b; + state[2] += c; + state[3] += d; + state[4] += e; + +#undef a +#undef b +#undef c +#undef d +#undef e +} + + +/* Initialize a SHA1 context */ +static void hash_init(SHA1Context *p){ + /* SHA1 initialization constants */ + p->state[0] = 0x67452301; + p->state[1] = 0xEFCDAB89; + p->state[2] = 0x98BADCFE; + p->state[3] = 0x10325476; + p->state[4] = 0xC3D2E1F0; + p->count[0] = p->count[1] = 0; +} + +/* Add new content to the SHA1 hash */ +static void hash_step( + SHA1Context *p, /* Add content to this context */ + const unsigned char *data, /* Data to be added */ + unsigned int len /* Number of bytes in data */ +){ + unsigned int i, j; + + j = p->count[0]; + if( (p->count[0] += len << 3) < j ){ + p->count[1] += (len>>29)+1; + } + j = (j >> 3) & 63; + if( (j + len) > 63 ){ + (void)memcpy(&p->buffer[j], data, (i = 64-j)); + SHA1Transform(p->state, p->buffer); + for(; i + 63 < len; i += 64){ + SHA1Transform(p->state, &data[i]); + } + j = 0; + }else{ + i = 0; + } + (void)memcpy(&p->buffer[j], &data[i], len - i); +} + +/* Compute a string using sqlite3_vsnprintf() and hash it */ +static void hash_step_vformat( + SHA1Context *p, /* Add content to this context */ + const char *zFormat, + ... +){ + va_list ap; + int n; + char zBuf[50]; + va_start(ap, zFormat); + sqlite3_vsnprintf(sizeof(zBuf),zBuf,zFormat,ap); + va_end(ap); + n = (int)strlen(zBuf); + hash_step(p, (unsigned char*)zBuf, n); +} + + +/* Add padding and compute the message digest. Render the +** message digest as lower-case hexadecimal and put it into +** zOut[]. zOut[] must be at least 41 bytes long. */ +static void hash_finish( + SHA1Context *p, /* The SHA1 context to finish and render */ + char *zOut /* Store hexadecimal hash here */ +){ + unsigned int i; + unsigned char finalcount[8]; + unsigned char digest[20]; + static const char zEncode[] = "0123456789abcdef"; + + for (i = 0; i < 8; i++){ + finalcount[i] = (unsigned char)((p->count[(i >= 4 ? 0 : 1)] + >> ((3-(i & 3)) * 8) ) & 255); /* Endian independent */ + } + hash_step(p, (const unsigned char *)"\200", 1); + while ((p->count[0] & 504) != 448){ + hash_step(p, (const unsigned char *)"\0", 1); + } + hash_step(p, finalcount, 8); /* Should cause a SHA1Transform() */ + for (i = 0; i < 20; i++){ + digest[i] = (unsigned char)((p->state[i>>2] >> ((3-(i & 3)) * 8) ) & 255); + } + for(i=0; i<20; i++){ + zOut[i*2] = zEncode[(digest[i]>>4)&0xf]; + zOut[i*2+1] = zEncode[digest[i] & 0xf]; + } + zOut[i*2]= 0; +} +/* End of the hashing logic +*****************************************************************************/ + +/* +** Implementation of the sha1(X) function. +** +** Return a lower-case hexadecimal rendering of the SHA1 hash of the +** argument X. If X is a BLOB, it is hashed as is. For all other +** types of input, X is converted into a UTF-8 string and the string +** is hash without the trailing 0x00 terminator. The hash of a NULL +** value is NULL. +*/ +static void sha1Func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + SHA1Context cx; + int eType = sqlite3_value_type(argv[0]); + int nByte = sqlite3_value_bytes(argv[0]); + char zOut[44]; + + assert( argc==1 ); + if( eType==SQLITE_NULL ) return; + hash_init(&cx); + if( eType==SQLITE_BLOB ){ + hash_step(&cx, sqlite3_value_blob(argv[0]), nByte); + }else{ + hash_step(&cx, sqlite3_value_text(argv[0]), nByte); + } + hash_finish(&cx, zOut); + sqlite3_result_text(context, zOut, 40, SQLITE_TRANSIENT); +} + +/* +** Implementation of the sha1_query(SQL) function. +** +** This function compiles and runs the SQL statement(s) given in the +** argument. The results are hashed using SHA1 and that hash is returned. +** +** The original SQL text is included as part of the hash. +** +** The hash is not just a concatenation of the outputs. Each query +** is delimited and each row and value within the query is delimited, +** with all values being marked with their datatypes. +*/ +static void sha1QueryFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + const char *zSql = (const char*)sqlite3_value_text(argv[0]); + sqlite3_stmt *pStmt = 0; + int nCol; /* Number of columns in the result set */ + int i; /* Loop counter */ + int rc; + int n; + const char *z; + SHA1Context cx; + char zOut[44]; + + assert( argc==1 ); + if( zSql==0 ) return; + hash_init(&cx); + while( zSql[0] ){ + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zSql); + if( rc ){ + char *zMsg = sqlite3_mprintf("error SQL statement [%s]: %s", + zSql, sqlite3_errmsg(db)); + sqlite3_finalize(pStmt); + sqlite3_result_error(context, zMsg, -1); + sqlite3_free(zMsg); + return; + } + if( !sqlite3_stmt_readonly(pStmt) ){ + char *zMsg = sqlite3_mprintf("non-query: [%s]", sqlite3_sql(pStmt)); + sqlite3_finalize(pStmt); + sqlite3_result_error(context, zMsg, -1); + sqlite3_free(zMsg); + return; + } + nCol = sqlite3_column_count(pStmt); + z = sqlite3_sql(pStmt); + n = (int)strlen(z); + hash_step_vformat(&cx,"S%d:",n); + hash_step(&cx,(unsigned char*)z,n); + + /* Compute a hash over the result of the query */ + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + hash_step(&cx,(const unsigned char*)"R",1); + for(i=0; i=1; j--){ + x[j] = u & 0xff; + u >>= 8; + } + x[0] = 'I'; + hash_step(&cx, x, 9); + break; + } + case SQLITE_FLOAT: { + sqlite3_uint64 u; + int j; + unsigned char x[9]; + double r = sqlite3_column_double(pStmt,i); + memcpy(&u, &r, 8); + for(j=8; j>=1; j--){ + x[j] = u & 0xff; + u >>= 8; + } + x[0] = 'F'; + hash_step(&cx,x,9); + break; + } + case SQLITE_TEXT: { + int n2 = sqlite3_column_bytes(pStmt, i); + const unsigned char *z2 = sqlite3_column_text(pStmt, i); + hash_step_vformat(&cx,"T%d:",n2); + hash_step(&cx, z2, n2); + break; + } + case SQLITE_BLOB: { + int n2 = sqlite3_column_bytes(pStmt, i); + const unsigned char *z2 = sqlite3_column_blob(pStmt, i); + hash_step_vformat(&cx,"B%d:",n2); + hash_step(&cx, z2, n2); + break; + } + } + } + } + sqlite3_finalize(pStmt); + } + hash_finish(&cx, zOut); + sqlite3_result_text(context, zOut, 40, SQLITE_TRANSIENT); +} + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_sha_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + rc = sqlite3_create_function(db, "sha1", 1, SQLITE_UTF8, 0, + sha1Func, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sha1_query", 1, SQLITE_UTF8, 0, + sha1QueryFunc, 0, 0); + } + return rc; +} diff --git a/ext/misc/shathree.c b/ext/misc/shathree.c new file mode 100644 index 0000000000..612e395ccb --- /dev/null +++ b/ext/misc/shathree.c @@ -0,0 +1,714 @@ +/* +** 2017-03-08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements a functions that compute SHA1 hashes. +** Two SQL functions are implemented: +** +** sha3(X,SIZE) +** sha3_query(Y,SIZE) +** +** The sha3(X) function computes the SHA3 hash of the input X, or NULL if +** X is NULL. +** +** The sha3_query(Y) function evalutes all queries in the SQL statements of Y +** and returns a hash of their results. +** +** The SIZE argument is optional. If omitted, the SHA3-256 hash algorithm +** is used. If SIZE is included it must be one of the integers 224, 256, +** 384, or 512, to determine SHA3 hash variant that is computed. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include +#include +typedef sqlite3_uint64 u64; + +/****************************************************************************** +** The Hash Engine +*/ +/* +** Macros to determine whether the machine is big or little endian, +** and whether or not that determination is run-time or compile-time. +** +** For best performance, an attempt is made to guess at the byte-order +** using C-preprocessor macros. If that is unsuccessful, or if +** -DSHA3_BYTEORDER=0 is set, then byte-order is determined +** at run-time. +*/ +#ifndef SHA3_BYTEORDER +# if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ + defined(__arm__) +# define SHA3_BYTEORDER 1234 +# elif defined(sparc) || defined(__ppc__) +# define SHA3_BYTEORDER 4321 +# else +# define SHA3_BYTEORDER 0 +# endif +#endif + + +/* +** State structure for a SHA3 hash in progress +*/ +typedef struct SHA3Context SHA3Context; +struct SHA3Context { + union { + u64 s[25]; /* Keccak state. 5x5 lines of 64 bits each */ + unsigned char x[1600]; /* ... or 1600 bytes */ + } u; + unsigned nRate; /* Bytes of input accepted per Keccak iteration */ + unsigned nLoaded; /* Input bytes loaded into u.x[] so far this cycle */ + unsigned ixMask; /* Insert next input into u.x[nLoaded^ixMask]. */ +}; + +/* +** A single step of the Keccak mixing function for a 1600-bit state +*/ +static void KeccakF1600Step(SHA3Context *p){ + int i; + u64 B0, B1, B2, B3, B4; + u64 C0, C1, C2, C3, C4; + u64 D0, D1, D2, D3, D4; + static const u64 RC[] = { + 0x0000000000000001ULL, 0x0000000000008082ULL, + 0x800000000000808aULL, 0x8000000080008000ULL, + 0x000000000000808bULL, 0x0000000080000001ULL, + 0x8000000080008081ULL, 0x8000000000008009ULL, + 0x000000000000008aULL, 0x0000000000000088ULL, + 0x0000000080008009ULL, 0x000000008000000aULL, + 0x000000008000808bULL, 0x800000000000008bULL, + 0x8000000000008089ULL, 0x8000000000008003ULL, + 0x8000000000008002ULL, 0x8000000000000080ULL, + 0x000000000000800aULL, 0x800000008000000aULL, + 0x8000000080008081ULL, 0x8000000000008080ULL, + 0x0000000080000001ULL, 0x8000000080008008ULL + }; +# define A00 (p->u.s[0]) +# define A01 (p->u.s[1]) +# define A02 (p->u.s[2]) +# define A03 (p->u.s[3]) +# define A04 (p->u.s[4]) +# define A10 (p->u.s[5]) +# define A11 (p->u.s[6]) +# define A12 (p->u.s[7]) +# define A13 (p->u.s[8]) +# define A14 (p->u.s[9]) +# define A20 (p->u.s[10]) +# define A21 (p->u.s[11]) +# define A22 (p->u.s[12]) +# define A23 (p->u.s[13]) +# define A24 (p->u.s[14]) +# define A30 (p->u.s[15]) +# define A31 (p->u.s[16]) +# define A32 (p->u.s[17]) +# define A33 (p->u.s[18]) +# define A34 (p->u.s[19]) +# define A40 (p->u.s[20]) +# define A41 (p->u.s[21]) +# define A42 (p->u.s[22]) +# define A43 (p->u.s[23]) +# define A44 (p->u.s[24]) +# define ROL64(a,x) ((a<>(64-x))) + + for(i=0; i<24; i+=4){ + C0 = A00^A10^A20^A30^A40; + C1 = A01^A11^A21^A31^A41; + C2 = A02^A12^A22^A32^A42; + C3 = A03^A13^A23^A33^A43; + C4 = A04^A14^A24^A34^A44; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A11^D1), 44); + B2 = ROL64((A22^D2), 43); + B3 = ROL64((A33^D3), 21); + B4 = ROL64((A44^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i]; + A11 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A20^D0), 3); + B3 = ROL64((A31^D1), 45); + B4 = ROL64((A42^D2), 61); + B0 = ROL64((A03^D3), 28); + B1 = ROL64((A14^D4), 20); + A20 = B0 ^((~B1)& B2 ); + A31 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A40^D0), 18); + B0 = ROL64((A01^D1), 1); + B1 = ROL64((A12^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A34^D4), 8); + A40 = B0 ^((~B1)& B2 ); + A01 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A10^D0), 36); + B2 = ROL64((A21^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A43^D3), 56); + B0 = ROL64((A04^D4), 27); + A10 = B0 ^((~B1)& B2 ); + A21 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A30^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A02^D2), 62); + B1 = ROL64((A13^D3), 55); + B2 = ROL64((A24^D4), 39); + A30 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + C0 = A00^A20^A40^A10^A30; + C1 = A11^A31^A01^A21^A41; + C2 = A22^A42^A12^A32^A02; + C3 = A33^A03^A23^A43^A13; + C4 = A44^A14^A34^A04^A24; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A31^D1), 44); + B2 = ROL64((A12^D2), 43); + B3 = ROL64((A43^D3), 21); + B4 = ROL64((A24^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i+1]; + A31 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A40^D0), 3); + B3 = ROL64((A21^D1), 45); + B4 = ROL64((A02^D2), 61); + B0 = ROL64((A33^D3), 28); + B1 = ROL64((A14^D4), 20); + A40 = B0 ^((~B1)& B2 ); + A21 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A30^D0), 18); + B0 = ROL64((A11^D1), 1); + B1 = ROL64((A42^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A04^D4), 8); + A30 = B0 ^((~B1)& B2 ); + A11 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A20^D0), 36); + B2 = ROL64((A01^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A13^D3), 56); + B0 = ROL64((A44^D4), 27); + A20 = B0 ^((~B1)& B2 ); + A01 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A10^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A22^D2), 62); + B1 = ROL64((A03^D3), 55); + B2 = ROL64((A34^D4), 39); + A10 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + C0 = A00^A40^A30^A20^A10; + C1 = A31^A21^A11^A01^A41; + C2 = A12^A02^A42^A32^A22; + C3 = A43^A33^A23^A13^A03; + C4 = A24^A14^A04^A44^A34; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A21^D1), 44); + B2 = ROL64((A42^D2), 43); + B3 = ROL64((A13^D3), 21); + B4 = ROL64((A34^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i+2]; + A21 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A30^D0), 3); + B3 = ROL64((A01^D1), 45); + B4 = ROL64((A22^D2), 61); + B0 = ROL64((A43^D3), 28); + B1 = ROL64((A14^D4), 20); + A30 = B0 ^((~B1)& B2 ); + A01 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A10^D0), 18); + B0 = ROL64((A31^D1), 1); + B1 = ROL64((A02^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A44^D4), 8); + A10 = B0 ^((~B1)& B2 ); + A31 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A40^D0), 36); + B2 = ROL64((A11^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A03^D3), 56); + B0 = ROL64((A24^D4), 27); + A40 = B0 ^((~B1)& B2 ); + A11 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A20^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A12^D2), 62); + B1 = ROL64((A33^D3), 55); + B2 = ROL64((A04^D4), 39); + A20 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + C0 = A00^A30^A10^A40^A20; + C1 = A21^A01^A31^A11^A41; + C2 = A42^A22^A02^A32^A12; + C3 = A13^A43^A23^A03^A33; + C4 = A34^A14^A44^A24^A04; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A01^D1), 44); + B2 = ROL64((A02^D2), 43); + B3 = ROL64((A03^D3), 21); + B4 = ROL64((A04^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i+3]; + A01 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A10^D0), 3); + B3 = ROL64((A11^D1), 45); + B4 = ROL64((A12^D2), 61); + B0 = ROL64((A13^D3), 28); + B1 = ROL64((A14^D4), 20); + A10 = B0 ^((~B1)& B2 ); + A11 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A20^D0), 18); + B0 = ROL64((A21^D1), 1); + B1 = ROL64((A22^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A24^D4), 8); + A20 = B0 ^((~B1)& B2 ); + A21 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A30^D0), 36); + B2 = ROL64((A31^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A33^D3), 56); + B0 = ROL64((A34^D4), 27); + A30 = B0 ^((~B1)& B2 ); + A31 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A40^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A42^D2), 62); + B1 = ROL64((A43^D3), 55); + B2 = ROL64((A44^D4), 39); + A40 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + } +} + +/* +** Initialize a new hash. iSize determines the size of the hash +** in bits and should be one of 224, 256, 384, or 512. Or iSize +** can be zero to use the default hash size of 256 bits. +*/ +static void SHA3Init(SHA3Context *p, int iSize){ + memset(p, 0, sizeof(*p)); + if( iSize>=128 && iSize<=512 ){ + p->nRate = (1600 - ((iSize + 31)&~31)*2)/8; + }else{ + p->nRate = (1600 - 2*256)/8; + } +#if SHA3_BYTEORDER==1234 + /* Known to be little-endian at compile-time. No-op */ +#elif SHA3_BYTEORDER==4321 + p->ixMask = 7; /* Big-endian */ +#else + { + static unsigned int one = 1; + if( 1==*(unsigned char*)&one ){ + /* Little endian. No byte swapping. */ + p->ixMask = 0; + }else{ + /* Big endian. Byte swap. */ + p->ixMask = 7; + } + } +#endif +} + +/* +** Make consecutive calls to the SHA3Update function to add new content +** to the hash +*/ +static void SHA3Update( + SHA3Context *p, + const unsigned char *aData, + unsigned int nData +){ + unsigned int i = 0; +#if SHA3_BYTEORDER==1234 + if( (p->nLoaded % 8)==0 && ((aData - (const unsigned char*)0)&7)==0 ){ + for(; i+7u.s[p->nLoaded/8] ^= *(u64*)&aData[i]; + p->nLoaded += 8; + if( p->nLoaded>=p->nRate ){ + KeccakF1600Step(p); + p->nLoaded = 0; + } + } + } +#endif + for(; iu.x[p->nLoaded] ^= aData[i]; +#elif SHA3_BYTEORDER==4321 + p->u.x[p->nLoaded^0x07] ^= aData[i]; +#else + p->u.x[p->nLoaded^p->ixMask] ^= aData[i]; +#endif + p->nLoaded++; + if( p->nLoaded==p->nRate ){ + KeccakF1600Step(p); + p->nLoaded = 0; + } + } +} + +/* +** After all content has been added, invoke SHA3Final() to compute +** the final hash. The function returns a pointer to the binary +** hash value. +*/ +static unsigned char *SHA3Final(SHA3Context *p){ + unsigned int i; + if( p->nLoaded==p->nRate-1 ){ + const unsigned char c1 = 0x86; + SHA3Update(p, &c1, 1); + }else{ + const unsigned char c2 = 0x06; + const unsigned char c3 = 0x80; + SHA3Update(p, &c2, 1); + p->nLoaded = p->nRate - 1; + SHA3Update(p, &c3, 1); + } + for(i=0; inRate; i++){ + p->u.x[i+p->nRate] = p->u.x[i^p->ixMask]; + } + return &p->u.x[p->nRate]; +} +/* End of the hashing logic +*****************************************************************************/ + +/* +** Implementation of the sha3(X,SIZE) function. +** +** Return a BLOB which is the SIZE-bit SHA3 hash of X. The default +** size is 256. If X is a BLOB, it is hashed as is. +** For all other non-NULL types of input, X is converted into a UTF-8 string +** and the string is hashed without the trailing 0x00 terminator. The hash +** of a NULL value is NULL. +*/ +static void sha3Func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + SHA3Context cx; + int eType = sqlite3_value_type(argv[0]); + int nByte = sqlite3_value_bytes(argv[0]); + int iSize; + if( argc==1 ){ + iSize = 256; + }else{ + iSize = sqlite3_value_int(argv[1]); + if( iSize!=224 && iSize!=256 && iSize!=384 && iSize!=512 ){ + sqlite3_result_error(context, "SHA3 size should be one of: 224 256 " + "384 512", -1); + return; + } + } + if( eType==SQLITE_NULL ) return; + SHA3Init(&cx, iSize); + if( eType==SQLITE_BLOB ){ + SHA3Update(&cx, sqlite3_value_blob(argv[0]), nByte); + }else{ + SHA3Update(&cx, sqlite3_value_text(argv[0]), nByte); + } + sqlite3_result_blob(context, SHA3Final(&cx), iSize/8, SQLITE_TRANSIENT); +} + +/* Compute a string using sqlite3_vsnprintf() with a maximum length +** of 50 bytes and add it to the hash. +*/ +static void hash_step_vformat( + SHA3Context *p, /* Add content to this context */ + const char *zFormat, + ... +){ + va_list ap; + int n; + char zBuf[50]; + va_start(ap, zFormat); + sqlite3_vsnprintf(sizeof(zBuf),zBuf,zFormat,ap); + va_end(ap); + n = (int)strlen(zBuf); + SHA3Update(p, (unsigned char*)zBuf, n); +} + +/* +** Implementation of the sha3_query(SQL,SIZE) function. +** +** This function compiles and runs the SQL statement(s) given in the +** argument. The results are hashed using a SIZE-bit SHA3. The default +** size is 256. +** +** The format of the byte stream that is hashed is summarized as follows: +** +** S: +** R +** N +** I +** F +** B: +** T: +** +** is the original SQL text for each statement run and is +** the size of that text. The SQL text is UTF-8. A single R character +** occurs before the start of each row. N means a NULL value. +** I mean an 8-byte little-endian integer . F is a floating point +** number with an 8-byte little-endian IEEE floating point value . +** B means blobs of bytes. T means text rendered as +** bytes of UTF-8. The and values are expressed as an ASCII +** text integers. +** +** For each SQL statement in the X input, there is one S segment. Each +** S segment is followed by zero or more R segments, one for each row in the +** result set. After each R, there are one or more N, I, F, B, or T segments, +** one for each column in the result set. Segments are concatentated directly +** with no delimiters of any kind. +*/ +static void sha3QueryFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + const char *zSql = (const char*)sqlite3_value_text(argv[0]); + sqlite3_stmt *pStmt = 0; + int nCol; /* Number of columns in the result set */ + int i; /* Loop counter */ + int rc; + int n; + const char *z; + SHA3Context cx; + int iSize; + + if( argc==1 ){ + iSize = 256; + }else{ + iSize = sqlite3_value_int(argv[1]); + if( iSize!=224 && iSize!=256 && iSize!=384 && iSize!=512 ){ + sqlite3_result_error(context, "SHA3 size should be one of: 224 256 " + "384 512", -1); + return; + } + } + if( zSql==0 ) return; + SHA3Init(&cx, iSize); + while( zSql[0] ){ + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zSql); + if( rc ){ + char *zMsg = sqlite3_mprintf("error SQL statement [%s]: %s", + zSql, sqlite3_errmsg(db)); + sqlite3_finalize(pStmt); + sqlite3_result_error(context, zMsg, -1); + sqlite3_free(zMsg); + return; + } + if( !sqlite3_stmt_readonly(pStmt) ){ + char *zMsg = sqlite3_mprintf("non-query: [%s]", sqlite3_sql(pStmt)); + sqlite3_finalize(pStmt); + sqlite3_result_error(context, zMsg, -1); + sqlite3_free(zMsg); + return; + } + nCol = sqlite3_column_count(pStmt); + z = sqlite3_sql(pStmt); + n = (int)strlen(z); + hash_step_vformat(&cx,"S%d:",n); + SHA3Update(&cx,(unsigned char*)z,n); + + /* Compute a hash over the result of the query */ + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + SHA3Update(&cx,(const unsigned char*)"R",1); + for(i=0; i=1; j--){ + x[j] = u & 0xff; + u >>= 8; + } + x[0] = 'I'; + SHA3Update(&cx, x, 9); + break; + } + case SQLITE_FLOAT: { + sqlite3_uint64 u; + int j; + unsigned char x[9]; + double r = sqlite3_column_double(pStmt,i); + memcpy(&u, &r, 8); + for(j=8; j>=1; j--){ + x[j] = u & 0xff; + u >>= 8; + } + x[0] = 'F'; + SHA3Update(&cx,x,9); + break; + } + case SQLITE_TEXT: { + int n2 = sqlite3_column_bytes(pStmt, i); + const unsigned char *z2 = sqlite3_column_text(pStmt, i); + hash_step_vformat(&cx,"T%d:",n2); + SHA3Update(&cx, z2, n2); + break; + } + case SQLITE_BLOB: { + int n2 = sqlite3_column_bytes(pStmt, i); + const unsigned char *z2 = sqlite3_column_blob(pStmt, i); + hash_step_vformat(&cx,"B%d:",n2); + SHA3Update(&cx, z2, n2); + break; + } + } + } + } + sqlite3_finalize(pStmt); + } + sqlite3_result_blob(context, SHA3Final(&cx), iSize/8, SQLITE_TRANSIENT); +} + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_shathree_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + rc = sqlite3_create_function(db, "sha3", 1, SQLITE_UTF8, 0, + sha3Func, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sha3", 2, SQLITE_UTF8, 0, + sha3Func, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sha3_query", 1, SQLITE_UTF8, 0, + sha3QueryFunc, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sha3_query", 2, SQLITE_UTF8, 0, + sha3QueryFunc, 0, 0); + } + return rc; +} diff --git a/ext/misc/vtshim.c b/ext/misc/vtshim.c index 01348e8d3e..0709a26a7f 100644 --- a/ext/misc/vtshim.c +++ b/ext/misc/vtshim.c @@ -95,6 +95,7 @@ static int vtshimCreate( if( rc ){ sqlite3_free(pNew); *ppVtab = 0; + return rc; } pNew->pAux = pAux; pNew->ppPrev = &pAux->pAllVtab; @@ -133,6 +134,7 @@ static int vtshimConnect( if( rc ){ sqlite3_free(pNew); *ppVtab = 0; + return rc; } pNew->pAux = pAux; pNew->ppPrev = &pAux->pAllVtab; diff --git a/ext/rbu/rbucrash2.test b/ext/rbu/rbucrash2.test new file mode 100644 index 0000000000..5f2ba604d1 --- /dev/null +++ b/ext/rbu/rbucrash2.test @@ -0,0 +1,106 @@ +# 2017 March 02 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl +set ::testprefix rbucrash2 + +db close +forcedelete test.db-oal rbu.db +sqlite3_shutdown +sqlite3_config_uri 1 +reset_db + +# Set up a target database and an rbu update database. The target +# db is the usual "test.db", the rbu db is "test.db2". +# +forcedelete test.db2 +do_execsql_test 1.0 { + CREATE TABLE t1(a, b, c, PRIMARY KEY(a), UNIQUE(b)); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); + INSERT INTO t1 VALUES(7, 8, 9); + + ATTACH 'test.db2' AS rbu; + CREATE TABLE rbu.data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES('one', randomblob(3500), NULL, 0); + INSERT INTO data_t1 VALUES('two', randomblob(3500), NULL, 0); + INSERT INTO data_t1 VALUES('three', randomblob(3500), NULL, 0); + INSERT INTO data_t1 VALUES('four', randomblob(3500), NULL, 0); + INSERT INTO data_t1 VALUES('five', randomblob(3500), NULL, 0); + INSERT INTO data_t1 VALUES('six', randomblob(3500), NULL, 0); +} +db_save_and_close + +proc do_rbu_crash_test2 {tn script} { + + foreach {f blksz} { + test.db 512 + test.db2 512 + test.db 4096 + test.db2 4096 + } { + set bDone 0 + for {set iDelay 1} {$bDone==0} {incr iDelay} { + forcedelete test.db2 test.db2-journal test.db test.db-oal test.db-wal + db_restore + + set res [ + crashsql -file $f -delay $iDelay -tclbody $script -dflt 1 -opendb {} \ + -blocksize $blksz {} + ] + + set bDone 1 + if {$res == "1 {child process exited abnormally}"} { + set bDone 0 + } elseif {$res != "0 {}"} { + error "unexected catchsql result: $res" + } + + sqlite3rbu rbu test.db test.db2 + while {[rbu step]=="SQLITE_OK"} {} + rbu close + + sqlite3 db test.db + do_execsql_test $tn.delay=$iDelay.f=$f.blksz=$blksz { + PRAGMA integrity_check; + } {ok} + db close + } + } +} + +for {set x 1} {$x < 10} {incr x} { + do_rbu_crash_test2 1.$x { + sqlite3rbu rbu test.db test.db2 + while {[rbu step]=="SQLITE_OK"} { + rbu savestate + } + rbu close + } +} + +for {set x 1} {$x < 2} {incr x} { + do_rbu_crash_test2 2.$x { + sqlite3rbu rbu test.db test.db2 + while {[rbu step]=="SQLITE_OK"} { + rbu close + sqlite3rbu rbu test.db test.db2 + } + rbu close + } +} + +finish_test + diff --git a/ext/rbu/rbufault4.test b/ext/rbu/rbufault4.test new file mode 100644 index 0000000000..fe6cc90dc9 --- /dev/null +++ b/ext/rbu/rbufault4.test @@ -0,0 +1,66 @@ +# 2014 October 22 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl +source $testdir/malloc_common.tcl +set ::testprefix rbufault4 + +for {set tn 1} {1} {incr tn} { + reset_db + do_execsql_test 1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1b ON t1(b); + CREATE INDEX i1c ON t1(c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); + } + + forcedelete test.db2 + sqlite3rbu_vacuum rbu test.db test.db2 + for {set i 0} {$i < $tn} {incr i} { rbu step } + set rc [rbu close] + if {$rc!="SQLITE_OK"} { + if {$rc!="SQLITE_DONE"} {error $rc} + break + } + faultsim_save + + do_faultsim_test $tn -faults oom-t* -prep { + faultsim_restore + } -body { + sqlite3rbu_vacuum rbu test.db test.db2 + while 1 { + set rc [rbu step] + if {$rc=="SQLITE_DONE"} break + if {$rc!="SQLITE_OK"} { error $rc } + } + } -test { + catch {rbu close} + faultsim_test_result {0 {}} {1 SQLITE_NOMEM} {1 SQLITE_IOERR_NOMEM} + + sqlite3rbu_vacuum rbu test.db test.db2 + while {[rbu step]=="SQLITE_OK"} {} + set trc [rbu close] + if {$trc!="SQLITE_DONE"} { error "Got $trc instead of SQLITE_DONE!" } + + set rc [db one {PRAGMA integrity_check}] + if {$rc!="ok"} { error "Got $rc instead of ok!" } + } +} + + + +finish_test + diff --git a/ext/rbu/rbuprogress.test b/ext/rbu/rbuprogress.test index af202829c1..078b3b0d30 100644 --- a/ext/rbu/rbuprogress.test +++ b/ext/rbu/rbuprogress.test @@ -40,6 +40,7 @@ proc create_rbu1 {filename} { do_execsql_test 1.0 { + PRAGMA page_size = 4096; CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); } @@ -266,6 +267,7 @@ foreach bReopen {0 1} { do_test 3.$bReopen.1.0 { reset_db execsql { + PRAGMA page_size = 4096; CREATE TABLE t1(a INTEGER PRIMARY KEY, b); CREATE TABLE t2(a INTEGER PRIMARY KEY, b); CREATE TABLE t3(a INTEGER PRIMARY KEY, b); diff --git a/ext/rbu/rburesume.test b/ext/rbu/rburesume.test new file mode 100644 index 0000000000..d03894e135 --- /dev/null +++ b/ext/rbu/rburesume.test @@ -0,0 +1,254 @@ +# 2017 January 13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for resumption of RBU operations in the +# case where the previous RBU process crashed. +# + +source [file join [file dirname [info script]] rbu_common.tcl] +set ::testprefix rburesume + +forcedelete test.db-shm test.db-oal +do_execsql_test 1.0 { + CREATE TABLE t1(a PRIMARY KEY, b, c); + CREATE INDEX t1a ON t1(a); + CREATE INDEX t1b ON t1(b); + CREATE INDEX t1c ON t1(c); + WITH s(i) AS ( + VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<50 + ) + INSERT INTO t1 SELECT randomblob(50), randomblob(75), randomblob(100) FROM s; +} +db_save_and_close + +do_test 1.1 { + list [file exists test.db] \ + [file exists test.db-wal] \ + [file exists test.db-shm] \ + [file exists test.db-oal] +} {1 0 0 0} + +# Each iteration of the following loop: +# +# 1. Restores the db to the state it was in following test case 1.0 +# 2. Opens an RBU vacuum and steps it $n times. +# 3. Closes the RBU vacuum handled opened in (2). +# 4. Opens a second RBU vacuum handle, resumes and completes the vacuum op. +# +# The loop runs until $n is large enough that step (2) vacuums the entire +# database. +# +for {set n 1} {$n < 5000} {incr n} { + db_restore + forcedelete state.db + sqlite3rbu_vacuum rbu test.db state.db + for {set i 0} {$i<$n} {incr i} { + set rc [rbu step] + if {$rc == "SQLITE_DONE"} break + } + rbu close + if {$rc == "SQLITE_DONE"} break + + do_test 1.2.$n.1 { + sqlite3rbu_vacuum rbu test.db state.db + while {[rbu step]=="SQLITE_OK"} {} + rbu close + } {SQLITE_DONE} + + do_test 1.2.$n.2 { + sqlite3 db2 test.db + db2 eval { + SELECT count(*) FROM t1; + PRAGMA integrity_check; + } + } {50 ok} + db2 close +} + +# Each iteration of this loop: +# +# 1. Restores the db to the state it was in following test case 1.0 +# 2. Opens an RBU vacuum and steps it $n times. +# 3. Takes a copy of all database files and the state db. +# 4. Opens a second RBU vacuum handle on the copy, resumes and completes the +# vacuum op. +# +# The loop runs until $n is large enough that step (2) vacuums the entire +# database. +# +for {set n 1} {$n < 5000} {incr n} { + db_restore + forcedelete state.db state.db-shm state.db-oal state.db-wal + sqlite3rbu_vacuum rbu test.db state.db + for {set i 0} {$i<$n} {incr i} { + set rc [rbu step] + if {$rc == "SQLITE_DONE"} break + } + if {$rc == "SQLITE_DONE"} { + rbu close + break + } + + foreach f {test.db test.db-oal test.db-wal test.db-shm test.db-vacuum} { + set f2 [string map [list test.db test.db2] $f] + if {[file exists $f]} { + forcecopy $f $f2 + } else { + forcedelete $f2 + } + } + forcecopy state.db state.db2 + rbu close + + do_test 1.3.$n.1 { + sqlite3rbu_vacuum rbu test.db2 state.db2 + while {[rbu step]=="SQLITE_OK"} {} + rbu close + } {SQLITE_DONE} + + do_test 1.3.$n.2 { + sqlite3 db2 test.db2 + db2 eval { + SELECT count(*) FROM t1; + PRAGMA integrity_check; + } + } {50 ok} + db2 close +} + +# Each iteration of this loop: +# +# 1. Restores the db to the state it was in following test case 1.0 +# 2. Opens an RBU vacuum and steps it 10 times. Then closes it. +# 2. Opens an RBU vacuum and steps it $n times. +# 3. Takes a copy of all database files and the state db. +# 4. Opens a second RBU vacuum handle on the copy, resumes and completes the +# vacuum op. +# +# The loop runs until $n is large enough that step (3) vacuums the entire +# database. +# +for {set n 1} {$n < 5000} {incr n} { + db_restore + forcedelete state.db state.db-shm state.db-oal state.db-wal + + sqlite3rbu_vacuum rbu test.db state.db + for {set i 0} {$i<10} {incr i} { + rbu step + } + rbu close + + sqlite3rbu_vacuum rbu test.db state.db + for {set i 0} {$i<$n} {incr i} { + set rc [rbu step] + if {$rc == "SQLITE_DONE"} break + } + if {$rc == "SQLITE_DONE"} { + rbu close + break + } + + foreach f {test.db test.db-oal test.db-wal test.db-shm test.db-vacuum} { + set f2 [string map [list test.db test.db2] $f] + if {[file exists $f]} { + forcecopy $f $f2 + } else { + forcedelete $f2 + } + } + forcecopy state.db state.db2 + rbu close + + do_test 1.4.$n.1 { + sqlite3rbu_vacuum rbu test.db2 state.db2 + while {[rbu step]=="SQLITE_OK"} {} + rbu close + } {SQLITE_DONE} + + do_test 1.4.$n.2 { + sqlite3 db2 test.db2 + db2 eval { + SELECT count(*) FROM t1; + PRAGMA integrity_check; + } + } {50 ok} + db2 close +} + +forcedelete rbu.db +do_test 2.0 { + sqlite3 db2 rbu.db + db2 eval { + CREATE TABLE data_t1(a, b, c, rbu_control); + WITH s(i) AS ( + VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<10 + ) + INSERT INTO data_t1 + SELECT randomblob(50), randomblob(75), randomblob(100), 0 FROM s; + } + db2 close +} {} + +# Each iteration of this loop: +# +# 1. Restores the db to the state it was in following test case 1.0 +# 2. Opens an RBU handle to apply the RBU update created in test case 2.0. +# 3. Steps the RBU handle $n times. +# 4. Takes a copy of all database files and the state db. +# 5. Opens a second RBU handle on the copy, resumes and completes the +# RBU op. Checks it worked as expected. +# +# The loop runs until $n is large enough that step (3) applies the entire +# update. +# +for {set n 1} {$n < 5000} {incr n} { + db_restore + forcedelete state.db state.db-shm state.db-oal state.db-wal + sqlite3rbu rbu test.db rbu.db state.db + + for {set i 0} {$i<$n} {incr i} { + set rc [rbu step] + if {$rc == "SQLITE_DONE"} break + } + if {$rc == "SQLITE_DONE"} { + rbu close + break + } + + foreach f {test.db test.db-oal test.db-wal test.db-shm test.db-vacuum} { + set f2 [string map [list test.db test.db2] $f] + if {[file exists $f]} { + forcecopy $f $f2 + } else { + forcedelete $f2 + } + } + forcecopy state.db state.db2 + rbu close + + do_test 2.$n.1 { + sqlite3rbu rbu test.db2 rbu.db state.db2 + while {[rbu step]=="SQLITE_OK"} {} + rbu close + } {SQLITE_DONE} + + do_test 2.$n.2 { + sqlite3 db2 test.db2 + db2 eval { + SELECT count(*) FROM t1; + PRAGMA integrity_check; + } + } {60 ok} + db2 close +} + +finish_test + diff --git a/ext/rbu/rbuvacuum2.test b/ext/rbu/rbuvacuum2.test index bd38660320..0a1fe3da94 100644 --- a/ext/rbu/rbuvacuum2.test +++ b/ext/rbu/rbuvacuum2.test @@ -199,6 +199,37 @@ if {$::tcl_platform(platform)=="unix"} { } } +#------------------------------------------------------------------------- +# Test the outcome of some other connection running a checkpoint while +# the incremental checkpoint is suspended. +# +reset_db +do_execsql_test 6.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1b ON t1(b); + CREATE INDEX i1c ON t1(c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); +} +forcedelete test.db2 + +do_test 6.1 { + sqlite3rbu_vacuum rbu test.db test.db2 + while {[rbu state]!="checkpoint"} { rbu step } + rbu close +} {SQLITE_OK} + +do_execsql_test 6.2 { + SELECT 1 FROM sqlite_master LIMIT 1; + PRAGMA wal_checkpoint; +} {1 0 4 4} + +do_test 6.3 { + sqlite3rbu_vacuum rbu test.db test.db2 + while {[rbu step]!="SQLITE_DONE"} { rbu step } + rbu close + execsql { PRAGMA integrity_check } +} {ok} finish_test diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 746469a8af..7dd232b4b5 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -356,6 +356,7 @@ struct sqlite3rbu { RbuObjIter objiter; /* Iterator for skipping through tbl/idx */ const char *zVfsName; /* Name of automatically created rbu vfs */ rbu_file *pTargetFd; /* File handle open on target db */ + int nPagePerSector; /* Pages per sector for pTargetFd */ i64 iOalSz; i64 nPhaseOneStep; @@ -2333,7 +2334,7 @@ static RbuState *rbuLoadState(sqlite3rbu *p){ ** Open the database handle and attach the RBU database as "rbu". If an ** error occurs, leave an error code and message in the RBU handle. */ -static void rbuOpenDatabase(sqlite3rbu *p){ +static void rbuOpenDatabase(sqlite3rbu *p, int *pbRetry){ assert( p->rc || (p->dbMain==0 && p->dbRbu==0) ); assert( p->rc || rbuIsVacuum(p) || p->zTarget!=0 ); @@ -2408,7 +2409,7 @@ static void rbuOpenDatabase(sqlite3rbu *p){ }else{ RbuState *pState = rbuLoadState(p); if( pState ){ - bOpen = (pState->eStage>RBU_STAGE_MOVE); + bOpen = (pState->eStage>=RBU_STAGE_MOVE); rbuFreeState(pState); } } @@ -2420,6 +2421,15 @@ static void rbuOpenDatabase(sqlite3rbu *p){ if( !rbuIsVacuum(p) ){ p->dbMain = rbuOpenDbhandle(p, p->zTarget, 1); }else if( p->pRbuFd->pWalFd ){ + if( pbRetry ){ + p->pRbuFd->bNolock = 0; + sqlite3_close(p->dbRbu); + sqlite3_close(p->dbMain); + p->dbMain = 0; + p->dbRbu = 0; + *pbRetry = 1; + return; + } p->rc = SQLITE_ERROR; p->zErrmsg = sqlite3_mprintf("cannot vacuum wal mode database"); }else{ @@ -2600,16 +2610,35 @@ static void rbuSetupCheckpoint(sqlite3rbu *p, RbuState *pState){ if( rc2!=SQLITE_INTERNAL ) p->rc = rc2; } - if( p->rc==SQLITE_OK ){ + if( p->rc==SQLITE_OK && p->nFrame>0 ){ p->eStage = RBU_STAGE_CKPT; p->nStep = (pState ? pState->nRow : 0); p->aBuf = rbuMalloc(p, p->pgsz); p->iWalCksum = rbuShmChecksum(p); } - if( p->rc==SQLITE_OK && pState && pState->iWalCksum!=p->iWalCksum ){ - p->rc = SQLITE_DONE; - p->eStage = RBU_STAGE_DONE; + if( p->rc==SQLITE_OK ){ + if( p->nFrame==0 || (pState && pState->iWalCksum!=p->iWalCksum) ){ + p->rc = SQLITE_DONE; + p->eStage = RBU_STAGE_DONE; + }else{ + int nSectorSize; + sqlite3_file *pDb = p->pTargetFd->pReal; + sqlite3_file *pWal = p->pTargetFd->pWalFd->pReal; + assert( p->nPagePerSector==0 ); + nSectorSize = pDb->pMethods->xSectorSize(pDb); + if( nSectorSize>p->pgsz ){ + p->nPagePerSector = nSectorSize / p->pgsz; + }else{ + p->nPagePerSector = 1; + } + + /* Call xSync() on the wal file. This causes SQLite to sync the + ** directory in which the target database and the wal file reside, in + ** case it has not been synced since the rename() call in + ** rbuMoveOalFile(). */ + p->rc = pWal->pMethods->xSync(pWal, SQLITE_SYNC_NORMAL); + } } } @@ -2782,7 +2811,7 @@ static void rbuMoveOalFile(sqlite3rbu *p){ #endif if( p->rc==SQLITE_OK ){ - rbuOpenDatabase(p); + rbuOpenDatabase(p, 0); rbuSetupCheckpoint(p, 0); } } @@ -3264,9 +3293,26 @@ int sqlite3rbu_step(sqlite3rbu *p){ p->rc = SQLITE_DONE; } }else{ - RbuFrame *pFrame = &p->aFrame[p->nStep]; - rbuCheckpointFrame(p, pFrame); - p->nStep++; + /* At one point the following block copied a single frame from the + ** wal file to the database file. So that one call to sqlite3rbu_step() + ** checkpointed a single frame. + ** + ** However, if the sector-size is larger than the page-size, and the + ** application calls sqlite3rbu_savestate() or close() immediately + ** after this step, then rbu_step() again, then a power failure occurs, + ** then the database page written here may be damaged. Work around + ** this by checkpointing frames until the next page in the aFrame[] + ** lies on a different disk sector to the current one. */ + u32 iSector; + do{ + RbuFrame *pFrame = &p->aFrame[p->nStep]; + iSector = (pFrame->iDbPage-1) / p->nPagePerSector; + rbuCheckpointFrame(p, pFrame); + p->nStep++; + }while( p->nStepnFrame + && iSector==((p->aFrame[p->nStep].iDbPage-1) / p->nPagePerSector) + && p->rc==SQLITE_OK + ); } p->nProgress++; } @@ -3493,6 +3539,7 @@ static sqlite3rbu *openRbuHandle( /* Open the target, RBU and state databases */ if( p->rc==SQLITE_OK ){ char *pCsr = (char*)&p[1]; + int bRetry = 0; if( zTarget ){ p->zTarget = pCsr; memcpy(p->zTarget, zTarget, nTarget+1); @@ -3504,7 +3551,18 @@ static sqlite3rbu *openRbuHandle( if( zState ){ p->zState = rbuMPrintf(p, "%s", zState); } - rbuOpenDatabase(p); + + /* If the first attempt to open the database file fails and the bRetry + ** flag it set, this means that the db was not opened because it seemed + ** to be a wal-mode db. But, this may have happened due to an earlier + ** RBU vacuum operation leaving an old wal file in the directory. + ** If this is the case, it will have been checkpointed and deleted + ** when the handle was closed and a second attempt to open the + ** database may succeed. */ + rbuOpenDatabase(p, &bRetry); + if( bRetry ){ + rbuOpenDatabase(p, 0); + } } if( p->rc==SQLITE_OK ){ @@ -3695,6 +3753,12 @@ int sqlite3rbu_close(sqlite3rbu *p, char **pzErrmsg){ p->rc = sqlite3_exec(p->dbMain, "COMMIT", 0, 0, &p->zErrmsg); } + /* Sync the db file if currently doing an incremental checkpoint */ + if( p->rc==SQLITE_OK && p->eStage==RBU_STAGE_CKPT ){ + sqlite3_file *pDb = p->pTargetFd->pReal; + p->rc = pDb->pMethods->xSync(pDb, SQLITE_SYNC_NORMAL); + } + rbuSaveState(p, p->eStage); if( p->rc==SQLITE_OK && p->eStage==RBU_STAGE_OAL ){ @@ -3819,6 +3883,12 @@ int sqlite3rbu_savestate(sqlite3rbu *p){ if( rc==SQLITE_OK ) rc = sqlite3_exec(p->dbMain, "COMMIT", 0, 0, 0); } + /* Sync the db file */ + if( rc==SQLITE_OK && p->eStage==RBU_STAGE_CKPT ){ + sqlite3_file *pDb = p->pTargetFd->pReal; + rc = pDb->pMethods->xSync(pDb, SQLITE_SYNC_NORMAL); + } + p->rc = rc; rbuSaveState(p, p->eStage); rc = p->rc; diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index c7de6ac123..42f08a9628 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -68,6 +68,7 @@ #ifndef SQLITE_AMALGAMATION #include "sqlite3rtree.h" typedef sqlite3_int64 i64; +typedef sqlite3_uint64 u64; typedef unsigned char u8; typedef unsigned short u16; typedef unsigned int u32; @@ -116,13 +117,16 @@ struct Rtree { sqlite3 *db; /* Host database connection */ int iNodeSize; /* Size in bytes of each node in the node table */ u8 nDim; /* Number of dimensions */ + u8 nDim2; /* Twice the number of dimensions */ u8 eCoordType; /* RTREE_COORD_REAL32 or RTREE_COORD_INT32 */ u8 nBytesPerCell; /* Bytes consumed per cell */ + u8 inWrTrans; /* True if inside write transaction */ int iDepth; /* Current depth of the r-tree structure */ char *zDb; /* Name of database containing r-tree table */ char *zName; /* Name of r-tree table */ - int nBusy; /* Current number of users of this structure */ + u32 nBusy; /* Current number of users of this structure */ i64 nRowEst; /* Estimated number of rows in this table */ + u32 nCursor; /* Number of open cursors */ /* List of nodes removed during a CondenseTree operation. List is ** linked together via the pointer normally used for hash chains - @@ -132,8 +136,10 @@ struct Rtree { RtreeNode *pDeleted; int iReinsertHeight; /* Height of sub-trees Reinsert() has run on */ + /* Blob I/O on xxx_node */ + sqlite3_blob *pNodeBlob; + /* Statements to read/write/delete a record from xxx_node */ - sqlite3_stmt *pReadNode; sqlite3_stmt *pWriteNode; sqlite3_stmt *pDeleteNode; @@ -362,6 +368,58 @@ struct RtreeMatchArg { # define MIN(x,y) ((x) > (y) ? (y) : (x)) #endif +/* What version of GCC is being used. 0 means GCC is not being used . +** Note that the GCC_VERSION macro will also be set correctly when using +** clang, since clang works hard to be gcc compatible. So the gcc +** optimizations will also work when compiling with clang. +*/ +#ifndef GCC_VERSION +#if defined(__GNUC__) && !defined(SQLITE_DISABLE_INTRINSIC) +# define GCC_VERSION (__GNUC__*1000000+__GNUC_MINOR__*1000+__GNUC_PATCHLEVEL__) +#else +# define GCC_VERSION 0 +#endif +#endif + +/* The testcase() macro should already be defined in the amalgamation. If +** it is not, make it a no-op. +*/ +#ifndef SQLITE_AMALGAMATION +# define testcase(X) +#endif + +/* +** Macros to determine whether the machine is big or little endian, +** and whether or not that determination is run-time or compile-time. +** +** For best performance, an attempt is made to guess at the byte-order +** using C-preprocessor macros. If that is unsuccessful, or if +** -DSQLITE_RUNTIME_BYTEORDER=1 is set, then byte-order is determined +** at run-time. +*/ +#ifndef SQLITE_BYTEORDER +#if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ + defined(__arm__) +# define SQLITE_BYTEORDER 1234 +#elif defined(sparc) || defined(__ppc__) +# define SQLITE_BYTEORDER 4321 +#else +# define SQLITE_BYTEORDER 0 /* 0 means "unknown at compile-time" */ +#endif +#endif + + +/* What version of MSVC is being used. 0 means MSVC is not being used */ +#ifndef MSVC_VERSION +#if defined(_MSC_VER) && !defined(SQLITE_DISABLE_INTRINSIC) +# define MSVC_VERSION _MSC_VER +#else +# define MSVC_VERSION 0 +#endif +#endif + /* ** Functions to deserialize a 16 bit integer, 32 bit real number and ** 64 bit integer. The deserialized value is returned. @@ -370,24 +428,47 @@ static int readInt16(u8 *p){ return (p[0]<<8) + p[1]; } static void readCoord(u8 *p, RtreeCoord *pCoord){ + assert( ((((char*)p) - (char*)0)&3)==0 ); /* p is always 4-byte aligned */ +#if SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 + pCoord->u = _byteswap_ulong(*(u32*)p); +#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 + pCoord->u = __builtin_bswap32(*(u32*)p); +#elif SQLITE_BYTEORDER==4321 + pCoord->u = *(u32*)p; +#else pCoord->u = ( (((u32)p[0]) << 24) + (((u32)p[1]) << 16) + (((u32)p[2]) << 8) + (((u32)p[3]) << 0) ); +#endif } static i64 readInt64(u8 *p){ - return ( - (((i64)p[0]) << 56) + - (((i64)p[1]) << 48) + - (((i64)p[2]) << 40) + - (((i64)p[3]) << 32) + - (((i64)p[4]) << 24) + - (((i64)p[5]) << 16) + - (((i64)p[6]) << 8) + - (((i64)p[7]) << 0) +#if SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 + u64 x; + memcpy(&x, p, 8); + return (i64)_byteswap_uint64(x); +#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 + u64 x; + memcpy(&x, p, 8); + return (i64)__builtin_bswap64(x); +#elif SQLITE_BYTEORDER==4321 + i64 x; + memcpy(&x, p, 8); + return x; +#else + return (i64)( + (((u64)p[0]) << 56) + + (((u64)p[1]) << 48) + + (((u64)p[2]) << 40) + + (((u64)p[3]) << 32) + + (((u64)p[4]) << 24) + + (((u64)p[5]) << 16) + + (((u64)p[6]) << 8) + + (((u64)p[7]) << 0) ); +#endif } /* @@ -395,23 +476,43 @@ static i64 readInt64(u8 *p){ ** 64 bit integer. The value returned is the number of bytes written ** to the argument buffer (always 2, 4 and 8 respectively). */ -static int writeInt16(u8 *p, int i){ +static void writeInt16(u8 *p, int i){ p[0] = (i>> 8)&0xFF; p[1] = (i>> 0)&0xFF; - return 2; } static int writeCoord(u8 *p, RtreeCoord *pCoord){ u32 i; + assert( ((((char*)p) - (char*)0)&3)==0 ); /* p is always 4-byte aligned */ assert( sizeof(RtreeCoord)==4 ); assert( sizeof(u32)==4 ); +#if SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 + i = __builtin_bswap32(pCoord->u); + memcpy(p, &i, 4); +#elif SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 + i = _byteswap_ulong(pCoord->u); + memcpy(p, &i, 4); +#elif SQLITE_BYTEORDER==4321 + i = pCoord->u; + memcpy(p, &i, 4); +#else i = pCoord->u; p[0] = (i>>24)&0xFF; p[1] = (i>>16)&0xFF; p[2] = (i>> 8)&0xFF; p[3] = (i>> 0)&0xFF; +#endif return 4; } static int writeInt64(u8 *p, i64 i){ +#if SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 + i = (i64)__builtin_bswap64((u64)i); + memcpy(p, &i, 8); +#elif SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 + i = (i64)_byteswap_uint64((u64)i); + memcpy(p, &i, 8); +#elif SQLITE_BYTEORDER==4321 + memcpy(p, &i, 8); +#else p[0] = (i>>56)&0xFF; p[1] = (i>>48)&0xFF; p[2] = (i>>40)&0xFF; @@ -420,6 +521,7 @@ static int writeInt64(u8 *p, i64 i){ p[5] = (i>>16)&0xFF; p[6] = (i>> 8)&0xFF; p[7] = (i>> 0)&0xFF; +#endif return 8; } @@ -502,6 +604,17 @@ static RtreeNode *nodeNew(Rtree *pRtree, RtreeNode *pParent){ return pNode; } +/* +** Clear the Rtree.pNodeBlob object +*/ +static void nodeBlobReset(Rtree *pRtree){ + if( pRtree->pNodeBlob && pRtree->inWrTrans==0 && pRtree->nCursor==0 ){ + sqlite3_blob *pBlob = pRtree->pNodeBlob; + pRtree->pNodeBlob = 0; + sqlite3_blob_close(pBlob); + } +} + /* ** Obtain a reference to an r-tree node. */ @@ -511,9 +624,8 @@ static int nodeAcquire( RtreeNode *pParent, /* Either the parent node or NULL */ RtreeNode **ppNode /* OUT: Acquired node */ ){ - int rc; - int rc2 = SQLITE_OK; - RtreeNode *pNode; + int rc = SQLITE_OK; + RtreeNode *pNode = 0; /* Check if the requested node is already in the hash table. If so, ** increase its reference count and return it. @@ -529,28 +641,45 @@ static int nodeAcquire( return SQLITE_OK; } - sqlite3_bind_int64(pRtree->pReadNode, 1, iNode); - rc = sqlite3_step(pRtree->pReadNode); - if( rc==SQLITE_ROW ){ - const u8 *zBlob = sqlite3_column_blob(pRtree->pReadNode, 0); - if( pRtree->iNodeSize==sqlite3_column_bytes(pRtree->pReadNode, 0) ){ - pNode = (RtreeNode *)sqlite3_malloc(sizeof(RtreeNode)+pRtree->iNodeSize); - if( !pNode ){ - rc2 = SQLITE_NOMEM; - }else{ - pNode->pParent = pParent; - pNode->zData = (u8 *)&pNode[1]; - pNode->nRef = 1; - pNode->iNode = iNode; - pNode->isDirty = 0; - pNode->pNext = 0; - memcpy(pNode->zData, zBlob, pRtree->iNodeSize); - nodeReference(pParent); - } + if( pRtree->pNodeBlob ){ + sqlite3_blob *pBlob = pRtree->pNodeBlob; + pRtree->pNodeBlob = 0; + rc = sqlite3_blob_reopen(pBlob, iNode); + pRtree->pNodeBlob = pBlob; + if( rc ){ + nodeBlobReset(pRtree); + if( rc==SQLITE_NOMEM ) return SQLITE_NOMEM; + } + } + if( pRtree->pNodeBlob==0 ){ + char *zTab = sqlite3_mprintf("%s_node", pRtree->zName); + if( zTab==0 ) return SQLITE_NOMEM; + rc = sqlite3_blob_open(pRtree->db, pRtree->zDb, zTab, "data", iNode, 0, + &pRtree->pNodeBlob); + sqlite3_free(zTab); + } + if( rc ){ + nodeBlobReset(pRtree); + *ppNode = 0; + /* If unable to open an sqlite3_blob on the desired row, that can only + ** be because the shadow tables hold erroneous data. */ + if( rc==SQLITE_ERROR ) rc = SQLITE_CORRUPT_VTAB; + }else if( pRtree->iNodeSize==sqlite3_blob_bytes(pRtree->pNodeBlob) ){ + pNode = (RtreeNode *)sqlite3_malloc(sizeof(RtreeNode)+pRtree->iNodeSize); + if( !pNode ){ + rc = SQLITE_NOMEM; + }else{ + pNode->pParent = pParent; + pNode->zData = (u8 *)&pNode[1]; + pNode->nRef = 1; + pNode->iNode = iNode; + pNode->isDirty = 0; + pNode->pNext = 0; + rc = sqlite3_blob_read(pRtree->pNodeBlob, pNode->zData, + pRtree->iNodeSize, 0); + nodeReference(pParent); } } - rc = sqlite3_reset(pRtree->pReadNode); - if( rc==SQLITE_OK ) rc = rc2; /* If the root node was just loaded, set pRtree->iDepth to the height ** of the r-tree structure. A height of zero means all data is stored on @@ -602,7 +731,7 @@ static void nodeOverwriteCell( int ii; u8 *p = &pNode->zData[4 + pRtree->nBytesPerCell*iCell]; p += writeInt64(p, pCell->iRowid); - for(ii=0; ii<(pRtree->nDim*2); ii++){ + for(ii=0; iinDim2; ii++){ p += writeCoord(p, &pCell->aCoord[ii]); } pNode->isDirty = 1; @@ -736,13 +865,16 @@ static void nodeGetCell( ){ u8 *pData; RtreeCoord *pCoord; - int ii; + int ii = 0; pCell->iRowid = nodeGetRowid(pRtree, pNode, iCell); pData = pNode->zData + (12 + pRtree->nBytesPerCell*iCell); pCoord = pCell->aCoord; - for(ii=0; iinDim*2; ii++){ - readCoord(&pData[ii*4], &pCoord[ii]); - } + do{ + readCoord(pData, &pCoord[ii]); + readCoord(pData+4, &pCoord[ii+1]); + pData += 8; + ii += 2; + }while( iinDim2 ); } @@ -793,7 +925,9 @@ static void rtreeReference(Rtree *pRtree){ static void rtreeRelease(Rtree *pRtree){ pRtree->nBusy--; if( pRtree->nBusy==0 ){ - sqlite3_finalize(pRtree->pReadNode); + pRtree->inWrTrans = 0; + pRtree->nCursor = 0; + nodeBlobReset(pRtree); sqlite3_finalize(pRtree->pWriteNode); sqlite3_finalize(pRtree->pDeleteNode); sqlite3_finalize(pRtree->pReadRowid); @@ -831,6 +965,7 @@ static int rtreeDestroy(sqlite3_vtab *pVtab){ if( !zCreate ){ rc = SQLITE_NOMEM; }else{ + nodeBlobReset(pRtree); rc = sqlite3_exec(pRtree->db, zCreate, 0, 0, 0); sqlite3_free(zCreate); } @@ -846,6 +981,7 @@ static int rtreeDestroy(sqlite3_vtab *pVtab){ */ static int rtreeOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ int rc = SQLITE_NOMEM; + Rtree *pRtree = (Rtree *)pVTab; RtreeCursor *pCsr; pCsr = (RtreeCursor *)sqlite3_malloc(sizeof(RtreeCursor)); @@ -853,6 +989,7 @@ static int rtreeOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ memset(pCsr, 0, sizeof(RtreeCursor)); pCsr->base.pVtab = pVTab; rc = SQLITE_OK; + pRtree->nCursor++; } *ppCursor = (sqlite3_vtab_cursor *)pCsr; @@ -885,10 +1022,13 @@ static int rtreeClose(sqlite3_vtab_cursor *cur){ Rtree *pRtree = (Rtree *)(cur->pVtab); int ii; RtreeCursor *pCsr = (RtreeCursor *)cur; + assert( pRtree->nCursor>0 ); freeCursorConstraints(pCsr); sqlite3_free(pCsr->aPoint); for(ii=0; iiaNode[ii]); sqlite3_free(pCsr); + pRtree->nCursor--; + nodeBlobReset(pRtree); return SQLITE_OK; } @@ -911,15 +1051,22 @@ static int rtreeEof(sqlite3_vtab_cursor *cur){ ** false. a[] is the four bytes of the on-disk record to be decoded. ** Store the results in "r". ** -** There are three versions of this macro, one each for little-endian and -** big-endian processors and a third generic implementation. The endian- -** specific implementations are much faster and are preferred if the -** processor endianness is known at compile-time. The SQLITE_BYTEORDER -** macro is part of sqliteInt.h and hence the endian-specific -** implementation will only be used if this module is compiled as part -** of the amalgamation. +** There are five versions of this macro. The last one is generic. The +** other four are various architectures-specific optimizations. */ -#if defined(SQLITE_BYTEORDER) && SQLITE_BYTEORDER==1234 +#if SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + c.u = _byteswap_ulong(*(u32*)a); \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ +} +#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + c.u = __builtin_bswap32(*(u32*)a); \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ +} +#elif SQLITE_BYTEORDER==1234 #define RTREE_DECODE_COORD(eInt, a, r) { \ RtreeCoord c; /* Coordinate decoded */ \ memcpy(&c.u,a,4); \ @@ -927,7 +1074,7 @@ static int rtreeEof(sqlite3_vtab_cursor *cur){ ((c.u&0xff)<<24)|((c.u&0xff00)<<8); \ r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ } -#elif defined(SQLITE_BYTEORDER) && SQLITE_BYTEORDER==4321 +#elif SQLITE_BYTEORDER==4321 #define RTREE_DECODE_COORD(eInt, a, r) { \ RtreeCoord c; /* Coordinate decoded */ \ memcpy(&c.u,a,4); \ @@ -954,10 +1101,10 @@ static int rtreeCallbackConstraint( sqlite3_rtree_dbl *prScore, /* OUT: score for the cell */ int *peWithin /* OUT: visibility of the cell */ ){ - int i; /* Loop counter */ sqlite3_rtree_query_info *pInfo = pConstraint->pInfo; /* Callback info */ int nCoord = pInfo->nCoord; /* No. of coordinates */ int rc; /* Callback return code */ + RtreeCoord c; /* Translator union */ sqlite3_rtree_dbl aCoord[RTREE_MAX_DIMENSIONS*2]; /* Decoded coordinates */ assert( pConstraint->op==RTREE_MATCH || pConstraint->op==RTREE_QUERY ); @@ -967,13 +1114,41 @@ static int rtreeCallbackConstraint( pInfo->iRowid = readInt64(pCellData); } pCellData += 8; - for(i=0; iop==RTREE_MATCH ){ + int eWithin = 0; rc = pConstraint->u.xGeom((sqlite3_rtree_geometry*)pInfo, - nCoord, aCoord, &i); - if( i==0 ) *peWithin = NOT_WITHIN; + nCoord, aCoord, &eWithin); + if( eWithin==0 ) *peWithin = NOT_WITHIN; *prScore = RTREE_ZERO; }else{ pInfo->aCoord = aCoord; @@ -1009,6 +1184,7 @@ static void rtreeNonleafConstraint( assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE || p->op==RTREE_GT || p->op==RTREE_EQ ); + assert( ((((char*)pCellData) - (char*)0)&3)==0 ); /* 4-byte aligned */ switch( p->op ){ case RTREE_LE: case RTREE_LT: @@ -1049,6 +1225,7 @@ static void rtreeLeafConstraint( assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE || p->op==RTREE_GT || p->op==RTREE_EQ ); pCellData += 8 + p->iCoord*4; + assert( ((((char*)pCellData) - (char*)0)&3)==0 ); /* 4-byte aligned */ RTREE_DECODE_COORD(eInt, pCellData, xN); switch( p->op ){ case RTREE_LE: if( xN <= p->u.rValue ) return; break; @@ -1117,7 +1294,7 @@ static int rtreeSearchPointCompare( } /* -** Interchange to search points in a cursor. +** Interchange two search points in a cursor. */ static void rtreeSearchPointSwap(RtreeCursor *p, int i, int j){ RtreeSearchPoint t = p->aPoint[i]; @@ -1365,7 +1542,7 @@ static int rtreeStepToLeaf(RtreeCursor *pCur){ if( rScoreeWithin = eWithin; + p->eWithin = (u8)eWithin; p->id = x.id; p->iCell = x.iCell; RTREE_QUEUE_TRACE(pCur, "PUSH-S:"); @@ -1424,7 +1601,6 @@ static int rtreeColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ if( i==0 ){ sqlite3_result_int64(ctx, nodeGetRowid(pRtree, pNode, p->iCell)); }else{ - if( rc ) return rc; nodeGetCoord(pRtree, pNode, p->iCell, i-1, &c); #ifndef SQLITE_RTREE_INT_ONLY if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ @@ -1553,7 +1729,7 @@ static int rtreeFilter( p->id = iNode; p->eWithin = PARTLY_WITHIN; rc = nodeRowidIndex(pRtree, pLeaf, iRowid, &iCell); - p->iCell = iCell; + p->iCell = (u8)iCell; RTREE_QUEUE_TRACE(pCsr, "PUSH-F1:"); }else{ pCsr->atEOF = 1; @@ -1586,7 +1762,7 @@ static int rtreeFilter( if( rc!=SQLITE_OK ){ break; } - p->pInfo->nCoord = pRtree->nDim*2; + p->pInfo->nCoord = pRtree->nDim2; p->pInfo->anQueue = pCsr->anQueue; p->pInfo->mxLevel = pRtree->iDepth + 1; }else{ @@ -1601,7 +1777,7 @@ static int rtreeFilter( } if( rc==SQLITE_OK ){ RtreeSearchPoint *pNew; - pNew = rtreeSearchPointNew(pCsr, RTREE_ZERO, pRtree->iDepth+1); + pNew = rtreeSearchPointNew(pCsr, RTREE_ZERO, (u8)(pRtree->iDepth+1)); if( pNew==0 ) return SQLITE_NOMEM; pNew->id = 1; pNew->iCell = 0; @@ -1619,19 +1795,6 @@ static int rtreeFilter( return rc; } -/* -** Set the pIdxInfo->estimatedRows variable to nRow. Unless this -** extension is currently being used by a version of SQLite too old to -** support estimatedRows. In that case this function is a no-op. -*/ -static void setEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){ -#if SQLITE_VERSION_NUMBER>=3008002 - if( sqlite3_libversion_number()>=3008002 ){ - pIdxInfo->estimatedRows = nRow; - } -#endif -} - /* ** Rtree virtual table module xBestIndex method. There are three ** table scan strategies to choose from (in order from most to @@ -1711,7 +1874,7 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ ** a single row. */ pIdxInfo->estimatedCost = 30.0; - setEstimatedRows(pIdxInfo, 1); + pIdxInfo->estimatedRows = 1; return SQLITE_OK; } @@ -1729,7 +1892,7 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ break; } zIdxStr[iIdx++] = op; - zIdxStr[iIdx++] = p->iColumn - 1 + '0'; + zIdxStr[iIdx++] = (char)(p->iColumn - 1 + '0'); pIdxInfo->aConstraintUsage[ii].argvIndex = (iIdx/2); pIdxInfo->aConstraintUsage[ii].omit = 1; } @@ -1743,7 +1906,7 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ nRow = pRtree->nRowEst >> (iIdx/2); pIdxInfo->estimatedCost = (double)6.0 * (double)nRow; - setEstimatedRows(pIdxInfo, nRow); + pIdxInfo->estimatedRows = nRow; return rc; } @@ -1753,9 +1916,26 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ */ static RtreeDValue cellArea(Rtree *pRtree, RtreeCell *p){ RtreeDValue area = (RtreeDValue)1; - int ii; - for(ii=0; ii<(pRtree->nDim*2); ii+=2){ - area = (area * (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii]))); + assert( pRtree->nDim>=1 && pRtree->nDim<=5 ); +#ifndef SQLITE_RTREE_INT_ONLY + if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ + switch( pRtree->nDim ){ + case 5: area = p->aCoord[9].f - p->aCoord[8].f; + case 4: area *= p->aCoord[7].f - p->aCoord[6].f; + case 3: area *= p->aCoord[5].f - p->aCoord[4].f; + case 2: area *= p->aCoord[3].f - p->aCoord[2].f; + default: area *= p->aCoord[1].f - p->aCoord[0].f; + } + }else +#endif + { + switch( pRtree->nDim ){ + case 5: area = p->aCoord[9].i - p->aCoord[8].i; + case 4: area *= p->aCoord[7].i - p->aCoord[6].i; + case 3: area *= p->aCoord[5].i - p->aCoord[4].i; + case 2: area *= p->aCoord[3].i - p->aCoord[2].i; + default: area *= p->aCoord[1].i - p->aCoord[0].i; + } } return area; } @@ -1765,11 +1945,12 @@ static RtreeDValue cellArea(Rtree *pRtree, RtreeCell *p){ ** of the objects size in each dimension. */ static RtreeDValue cellMargin(Rtree *pRtree, RtreeCell *p){ - RtreeDValue margin = (RtreeDValue)0; - int ii; - for(ii=0; ii<(pRtree->nDim*2); ii+=2){ + RtreeDValue margin = 0; + int ii = pRtree->nDim2 - 2; + do{ margin += (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii])); - } + ii -= 2; + }while( ii>=0 ); return margin; } @@ -1777,17 +1958,19 @@ static RtreeDValue cellMargin(Rtree *pRtree, RtreeCell *p){ ** Store the union of cells p1 and p2 in p1. */ static void cellUnion(Rtree *pRtree, RtreeCell *p1, RtreeCell *p2){ - int ii; + int ii = 0; if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ - for(ii=0; ii<(pRtree->nDim*2); ii+=2){ + do{ p1->aCoord[ii].f = MIN(p1->aCoord[ii].f, p2->aCoord[ii].f); p1->aCoord[ii+1].f = MAX(p1->aCoord[ii+1].f, p2->aCoord[ii+1].f); - } + ii += 2; + }while( iinDim2 ); }else{ - for(ii=0; ii<(pRtree->nDim*2); ii+=2){ + do{ p1->aCoord[ii].i = MIN(p1->aCoord[ii].i, p2->aCoord[ii].i); p1->aCoord[ii+1].i = MAX(p1->aCoord[ii+1].i, p2->aCoord[ii+1].i); - } + ii += 2; + }while( iinDim2 ); } } @@ -1798,7 +1981,7 @@ static void cellUnion(Rtree *pRtree, RtreeCell *p1, RtreeCell *p2){ static int cellContains(Rtree *pRtree, RtreeCell *p1, RtreeCell *p2){ int ii; int isInt = (pRtree->eCoordType==RTREE_COORD_INT32); - for(ii=0; ii<(pRtree->nDim*2); ii+=2){ + for(ii=0; iinDim2; ii+=2){ RtreeCoord *a1 = &p1->aCoord[ii]; RtreeCoord *a2 = &p2->aCoord[ii]; if( (!isInt && (a2[0].fa1[1].f)) @@ -1833,7 +2016,7 @@ static RtreeDValue cellOverlap( for(ii=0; iinDim*2); jj+=2){ + for(jj=0; jjnDim2; jj+=2){ RtreeDValue x1, x2; x1 = MAX(DCOORD(p->aCoord[jj]), DCOORD(aCell[ii].aCoord[jj])); x2 = MIN(DCOORD(p->aCoord[jj+1]), DCOORD(aCell[ii].aCoord[jj+1])); @@ -2889,7 +3072,7 @@ static int rtreeUpdate( ** This problem was discovered after years of use, so we silently ignore ** these kinds of misdeclared tables to avoid breaking any legacy. */ - assert( nData<=(pRtree->nDim*2 + 3) ); + assert( nData<=(pRtree->nDim2 + 3) ); #ifndef SQLITE_RTREE_INT_ONLY if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ @@ -2979,6 +3162,27 @@ constraint: return rc; } +/* +** Called when a transaction starts. +*/ +static int rtreeBeginTransaction(sqlite3_vtab *pVtab){ + Rtree *pRtree = (Rtree *)pVtab; + assert( pRtree->inWrTrans==0 ); + pRtree->inWrTrans++; + return SQLITE_OK; +} + +/* +** Called when a transaction completes (either by COMMIT or ROLLBACK). +** The sqlite3_blob object should be released at this point. +*/ +static int rtreeEndTransaction(sqlite3_vtab *pVtab){ + Rtree *pRtree = (Rtree *)pVtab; + pRtree->inWrTrans = 0; + nodeBlobReset(pRtree); + return SQLITE_OK; +} + /* ** The xRename method for rtree module virtual tables. */ @@ -3000,6 +3204,7 @@ static int rtreeRename(sqlite3_vtab *pVtab, const char *zNewName){ return rc; } + /* ** This function populates the pRtree->nRowEst variable with an estimate ** of the number of rows in the virtual table. If possible, this is based @@ -3059,15 +3264,15 @@ static sqlite3_module rtreeModule = { rtreeColumn, /* xColumn - read data */ rtreeRowid, /* xRowid - read data */ rtreeUpdate, /* xUpdate - write data */ - 0, /* xBegin - begin transaction */ - 0, /* xSync - sync transaction */ - 0, /* xCommit - commit transaction */ - 0, /* xRollback - rollback transaction */ + rtreeBeginTransaction, /* xBegin - begin transaction */ + rtreeEndTransaction, /* xSync - sync transaction */ + rtreeEndTransaction, /* xCommit - commit transaction */ + rtreeEndTransaction, /* xRollback - rollback transaction */ 0, /* xFindFunction - function overloading */ rtreeRename, /* xRename - rename the table */ 0, /* xSavepoint */ 0, /* xRelease */ - 0 /* xRollbackTo */ + 0, /* xRollbackTo */ }; static int rtreeSqlInit( @@ -3079,10 +3284,9 @@ static int rtreeSqlInit( ){ int rc = SQLITE_OK; - #define N_STATEMENT 9 + #define N_STATEMENT 8 static const char *azSql[N_STATEMENT] = { - /* Read and write the xxx_node table */ - "SELECT data FROM '%q'.'%q_node' WHERE nodeno = :1", + /* Write the xxx_node table */ "INSERT OR REPLACE INTO '%q'.'%q_node' VALUES(:1, :2)", "DELETE FROM '%q'.'%q_node' WHERE nodeno = :1", @@ -3120,15 +3324,14 @@ static int rtreeSqlInit( } } - appStmt[0] = &pRtree->pReadNode; - appStmt[1] = &pRtree->pWriteNode; - appStmt[2] = &pRtree->pDeleteNode; - appStmt[3] = &pRtree->pReadRowid; - appStmt[4] = &pRtree->pWriteRowid; - appStmt[5] = &pRtree->pDeleteRowid; - appStmt[6] = &pRtree->pReadParent; - appStmt[7] = &pRtree->pWriteParent; - appStmt[8] = &pRtree->pDeleteParent; + appStmt[0] = &pRtree->pWriteNode; + appStmt[1] = &pRtree->pDeleteNode; + appStmt[2] = &pRtree->pReadRowid; + appStmt[3] = &pRtree->pWriteRowid; + appStmt[4] = &pRtree->pDeleteRowid; + appStmt[5] = &pRtree->pReadParent; + appStmt[6] = &pRtree->pWriteParent; + appStmt[7] = &pRtree->pDeleteParent; rc = rtreeQueryStat1(db, pRtree); for(i=0; ibase.pModule = &rtreeModule; pRtree->zDb = (char *)&pRtree[1]; pRtree->zName = &pRtree->zDb[nDb+1]; - pRtree->nDim = (argc-4)/2; - pRtree->nBytesPerCell = 8 + pRtree->nDim*4*2; - pRtree->eCoordType = eCoordType; + pRtree->nDim = (u8)((argc-4)/2); + pRtree->nDim2 = pRtree->nDim*2; + pRtree->nBytesPerCell = 8 + pRtree->nDim2*4; + pRtree->eCoordType = (u8)eCoordType; memcpy(pRtree->zDb, argv[1], nDb); memcpy(pRtree->zName, argv[2], nName); @@ -3341,7 +3545,8 @@ static void rtreenode(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ UNUSED_PARAMETER(nArg); memset(&node, 0, sizeof(RtreeNode)); memset(&tree, 0, sizeof(Rtree)); - tree.nDim = sqlite3_value_int(apArg[0]); + tree.nDim = (u8)sqlite3_value_int(apArg[0]); + tree.nDim2 = tree.nDim*2; tree.nBytesPerCell = 8 + 8 * tree.nDim; node.zData = (u8 *)sqlite3_value_blob(apArg[1]); @@ -3354,7 +3559,7 @@ static void rtreenode(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ nodeGetCell(&tree, &node, ii, &cell); sqlite3_snprintf(512-nCell,&zCell[nCell],"%lld", cell.iRowid); nCell = (int)strlen(zCell); - for(jj=0; jj +#include +#include +#include +#include + +/************************************************************************* +** Start of generic command line parser. +*/ +#define CMDLINE_BARE 0 +#define CMDLINE_INTEGER 1 +#define CMDLINE_STRING 2 +#define CMDLINE_BOOLEAN 3 + +typedef struct CmdLineOption CmdLineOption; +struct CmdLineOption { + const char *zText; /* Name of command line option */ + const char *zHelp; /* Help text for option */ + int eType; /* One of the CMDLINE_* values */ + int iOff; /* Offset of output variable */ +}; + +#define CMDLINE_INT32(x,y,z) {x, y, CMDLINE_INTEGER, z} +#define CMDLINE_BOOL(x,y,z) {x, y, CMDLINE_BOOLEAN, z} +#define CMDLINE_TEXT(x,y,z) {x, y, CMDLINE_STRING, z} +#define CMDLINE_NONE(x,y,z) {x, y, CMDLINE_BARE, z} + +static void option_requires_argument_error(CmdLineOption *pOpt){ + fprintf(stderr, "Option requires a%s argument: %s\n", + pOpt->eType==CMDLINE_INTEGER ? "n integer" : + pOpt->eType==CMDLINE_STRING ? " string" : " boolean", + pOpt->zText + ); + exit(1); +} + +static void ambiguous_option_error(const char *zArg){ + fprintf(stderr, "Option is ambiguous: %s\n", zArg); + exit(1); +} + +static void unknown_option_error( + const char *zArg, + CmdLineOption *aOpt, + const char *zHelp +){ + int i; + fprintf(stderr, "Unknown option: %s\n", zArg); + fprintf(stderr, "\nOptions are:\n"); + fprintf(stderr, " % -30sEcho command line options\n", "-cmdline:verbose"); + for(i=0; aOpt[i].zText; i++){ + int eType = aOpt[i].eType; + char *zOpt = sqlite3_mprintf("%s %s", aOpt[i].zText, + eType==CMDLINE_BARE ? "" : + eType==CMDLINE_INTEGER ? "N" : + eType==CMDLINE_BOOLEAN ? "BOOLEAN" : "TEXT" + ); + fprintf(stderr, " % -30s%s\n", zOpt, aOpt[i].zHelp); + sqlite3_free(zOpt); + } + if( zHelp ){ + fprintf(stderr, "\n%s\n", zHelp); + } + exit(1); +} + +static int get_integer_option(CmdLineOption *pOpt, const char *zArg){ + int i = 0; + int iRet = 0; + int bSign = 1; + if( zArg[0]=='-' ){ + bSign = -1; + i = 1; + } + while( zArg[i] ){ + if( zArg[i]<'0' || zArg[i]>'9' ) option_requires_argument_error(pOpt); + iRet = iRet*10 + (zArg[i] - '0'); + i++; + } + return (iRet*bSign); +} + +static int get_boolean_option(CmdLineOption *pOpt, const char *zArg){ + if( 0==sqlite3_stricmp(zArg, "true") ) return 1; + if( 0==sqlite3_stricmp(zArg, "1") ) return 1; + if( 0==sqlite3_stricmp(zArg, "0") ) return 0; + if( 0==sqlite3_stricmp(zArg, "false") ) return 0; + option_requires_argument_error(pOpt); + return 0; +} + +static void parse_command_line( + int argc, + char **argv, + int iStart, + CmdLineOption *aOpt, + void *pStruct, + const char *zHelp +){ + char *pOut = (char*)pStruct; + int bVerbose = 0; + int iArg; + + for(iArg=iStart; iArgzText, zArg, nArg) ){ + if( nMatch ){ + ambiguous_option_error(zArg); + } + nMatch++; + if( pOpt->eType==CMDLINE_BARE ){ + *(int*)(&pOut[pOpt->iOff]) = 1; + }else{ + iArg++; + if( iArg==argc ){ + option_requires_argument_error(pOpt); + } + switch( pOpt->eType ){ + case CMDLINE_INTEGER: + *(int*)(&pOut[pOpt->iOff]) = get_integer_option(pOpt, argv[iArg]); + break; + case CMDLINE_STRING: + *(const char**)(&pOut[pOpt->iOff]) = argv[iArg]; + break; + case CMDLINE_BOOLEAN: + *(int*)(&pOut[pOpt->iOff]) = get_boolean_option(pOpt, argv[iArg]); + break; + } + } + } + } + + if( nMatch==0 && 0==sqlite3_strnicmp("-cmdline:verbose", zArg, nArg) ){ + bVerbose = 1; + nMatch = 1; + } + + if( nMatch==0 ){ + unknown_option_error(zArg, aOpt, zHelp); + } + } + + if( bVerbose ){ + int iOpt; + fprintf(stdout, "Options are: "); + for(iOpt=0; aOpt[iOpt].zText; iOpt++){ + CmdLineOption *pOpt = &aOpt[iOpt]; + if( pOpt->eType!=CMDLINE_BARE || *(int*)(&pOut[pOpt->iOff]) ){ + fprintf(stdout, "%s ", pOpt->zText); + } + switch( pOpt->eType ){ + case CMDLINE_INTEGER: + fprintf(stdout, "%d ", *(int*)(&pOut[pOpt->iOff])); + break; + case CMDLINE_BOOLEAN: + fprintf(stdout, "%d ", *(int*)(&pOut[pOpt->iOff])); + break; + case CMDLINE_STRING: + fprintf(stdout, "%s ", *(const char**)(&pOut[pOpt->iOff])); + break; + } + } + fprintf(stdout, "\n"); + } +} +/* +** End of generic command line parser. +*************************************************************************/ + +static void abort_due_to_error(int rc){ + fprintf(stderr, "Error: %d\n"); + exit(-1); +} + +static void execsql(sqlite3 *db, const char *zSql){ + int rc = sqlite3_exec(db, zSql, 0, 0, 0); + if( rc!=SQLITE_OK ) abort_due_to_error(rc); +} + +static int xConflict(void *pCtx, int eConflict, sqlite3_changeset_iter *p){ + return SQLITE_CHANGESET_ABORT; +} + +static void run_test( + sqlite3 *db, + sqlite3 *db2, + int nRow, + const char *zSql +){ + sqlite3_session *pSession = 0; + sqlite3_stmt *pStmt = 0; + int rc; + int i; + int nChangeset; + void *pChangeset; + + /* Attach a session object to database db */ + rc = sqlite3session_create(db, "main", &pSession); + if( rc!=SQLITE_OK ) abort_due_to_error(rc); + + /* Configure the session to capture changes on all tables */ + rc = sqlite3session_attach(pSession, 0); + if( rc!=SQLITE_OK ) abort_due_to_error(rc); + + /* Prepare the SQL statement */ + rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ) abort_due_to_error(rc); + + /* Open a transaction */ + execsql(db, "BEGIN"); + + /* Execute the SQL statement nRow times */ + for(i=0; inCol = nCol; pNew->bPk = bPk; pNew->aiCol = (int*)&pNew[1]; diff --git a/ext/session/sqlite3session.c b/ext/session/sqlite3session.c index efb7e37659..34eaac0842 100644 --- a/ext/session/sqlite3session.c +++ b/ext/session/sqlite3session.c @@ -3049,7 +3049,7 @@ int sqlite3changeset_conflict( if( !pIter->pConflict ){ return SQLITE_MISUSE; } - if( iVal<0 || iVal>=sqlite3_column_count(pIter->pConflict) ){ + if( iVal<0 || iVal>=pIter->nCol ){ return SQLITE_RANGE; } *ppValue = sqlite3_column_value(pIter->pConflict, iVal); @@ -3516,7 +3516,13 @@ static int sessionInsertRow( sessionAppendStr(&buf, "INSERT INTO main.", &rc); sessionAppendIdent(&buf, zTab, &rc); - sessionAppendStr(&buf, " VALUES(?", &rc); + sessionAppendStr(&buf, "(", &rc); + for(i=0; inCol; i++){ + if( i!=0 ) sessionAppendStr(&buf, ", ", &rc); + sessionAppendIdent(&buf, p->azCol[i], &rc); + } + + sessionAppendStr(&buf, ") VALUES(?", &rc); for(i=1; inCol; i++){ sessionAppendStr(&buf, ", ?", &rc); } @@ -4062,11 +4068,17 @@ static int sessionChangesetApply( nTab = (int)strlen(zTab); sApply.azCol = (const char **)zTab; }else{ + int nMinCol = 0; + int i; + sqlite3changeset_pk(pIter, &abPK, 0); rc = sessionTableInfo( db, "main", zNew, &sApply.nCol, &zTab, &sApply.azCol, &sApply.abPK ); if( rc!=SQLITE_OK ) break; + for(i=0; i For each row (primary key) that exists in both tables, but features -** different in each, an UPDATE record is added to the session. +** different non-PK values in each, an UPDATE record is added to the +** session. ** ** ** To clarify, if this function is called and then a changeset constructed @@ -917,7 +918,7 @@ void sqlite3changegroup_delete(sqlite3_changegroup*); **
    **
  • The table has the same name as the name recorded in the ** changeset, and -**
  • The table has the same number of columns as recorded in the +**
  • The table has at least as many columns as recorded in the ** changeset, and **
  • The table has primary key columns in the same position as ** recorded in the changeset. @@ -962,7 +963,11 @@ void sqlite3changegroup_delete(sqlite3_changegroup*); ** If a row with matching primary key values is found, but one or more of ** the non-primary key fields contains a value different from the original ** row value stored in the changeset, the conflict-handler function is -** invoked with [SQLITE_CHANGESET_DATA] as the second argument. +** invoked with [SQLITE_CHANGESET_DATA] as the second argument. If the +** database table has more columns than are recorded in the changeset, +** only the values of those non-primary key fields are compared against +** the current database contents - any trailing database table columns +** are ignored. ** ** If no row with matching primary key values is found in the database, ** the conflict-handler function is invoked with [SQLITE_CHANGESET_NOTFOUND] @@ -977,7 +982,9 @@ void sqlite3changegroup_delete(sqlite3_changegroup*); ** **
    INSERT Changes
    ** For each INSERT change, an attempt is made to insert the new row into -** the database. +** the database. If the changeset row contains fewer fields than the +** database table, the trailing fields are populated with their default +** values. ** ** If the attempt to insert the row fails because the database already ** contains a row with the same primary key values, the conflict handler @@ -995,13 +1002,13 @@ void sqlite3changegroup_delete(sqlite3_changegroup*); ** For each UPDATE change, this function checks if the target database ** contains a row with the same primary key value (or values) as the ** original row values stored in the changeset. If it does, and the values -** stored in all non-primary key columns also match the values stored in -** the changeset the row is updated within the target database. +** stored in all modified non-primary key columns also match the values +** stored in the changeset the row is updated within the target database. ** ** If a row with matching primary key values is found, but one or more of -** the non-primary key fields contains a value different from an original -** row value stored in the changeset, the conflict-handler function is -** invoked with [SQLITE_CHANGESET_DATA] as the second argument. Since +** the modified non-primary key fields contains a value different from an +** original row value stored in the changeset, the conflict-handler function +** is invoked with [SQLITE_CHANGESET_DATA] as the second argument. Since ** UPDATE changes only contain values for non-primary key fields that are ** to be modified, only those fields need to match the original values to ** avoid the SQLITE_CHANGESET_DATA conflict-handler callback. diff --git a/ext/userauth/sqlite3userauth.h b/ext/userauth/sqlite3userauth.h index 619477cac9..c7d23b9399 100644 --- a/ext/userauth/sqlite3userauth.h +++ b/ext/userauth/sqlite3userauth.h @@ -21,6 +21,10 @@ */ #ifdef SQLITE_USER_AUTHENTICATION +#ifdef __cplusplus +extern "C" { +#endif + /* ** If a database contains the SQLITE_USER table, then the ** sqlite3_user_authenticate() interface must be invoked with an @@ -85,4 +89,8 @@ int sqlite3_user_delete( const char *zUsername /* Username to remove */ ); +#ifdef __cplusplus +} /* end of the 'extern "C"' block */ +#endif + #endif /* SQLITE_USER_AUTHENTICATION */ diff --git a/ext/userauth/userauth.c b/ext/userauth/userauth.c index 6ce99053d3..a3d346c05c 100644 --- a/ext/userauth/userauth.c +++ b/ext/userauth/userauth.c @@ -22,7 +22,7 @@ ** directory as this file for additional information. */ #ifdef SQLITE_USER_AUTHENTICATION -#ifndef _SQLITEINT_H_ +#ifndef SQLITEINT_H # include "sqliteInt.h" #endif diff --git a/main.mk b/main.mk index 305d3a61a7..1a2700999b 100644 --- a/main.mk +++ b/main.mk @@ -478,7 +478,10 @@ SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1 FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 +FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000 DBFUZZ_OPT = +KV_OPT = -DSQLITE_THREADSAFE=0 -DSQLITE_DIRECT_OVERFLOW_READ +ST_OPT = -DSQLITE_THREADSAFE=0 # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. @@ -759,6 +762,10 @@ sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl sqlite3_analyzer$(EXE): sqlite3_analyzer.c $(TCCX) $(TCL_FLAGS) sqlite3_analyzer.c -o $@ $(LIBTCL) $(THREADLIB) +dbdump$(EXE): $(TOP)/ext/misc/dbdump.c sqlite3.o + $(TCCX) -DDBDUMP_STANDALONE -o dbdump$(EXE) \ + $(TOP)/ext/misc/dbdump.c sqlite3.o $(THREADLIB) + # Rules to build the 'testfixture' application. # TESTFIXTURE_FLAGS = -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 @@ -896,8 +903,11 @@ wordcount$(EXE): $(TOP)/test/wordcount.c sqlite3.c $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o wordcount$(EXE) \ $(TOP)/test/wordcount.c sqlite3.c -speedtest1$(EXE): $(TOP)/test/speedtest1.c sqlite3.o - $(TCC) -I. $(OTAFLAGS) -o speedtest1$(EXE) $(TOP)/test/speedtest1.c sqlite3.o $(THREADLIB) +speedtest1$(EXE): $(TOP)/test/speedtest1.c sqlite3.c + $(TCCX) -I. $(ST_OPT) -o speedtest1$(EXE) $(TOP)/test/speedtest1.c sqlite3.c $(THREADLIB) + +kvtest$(EXE): $(TOP)/test/kvtest.c sqlite3.c + $(TCCX) -I. $(KV_OPT) -o kvtest$(EXE) $(TOP)/test/kvtest.c sqlite3.c $(THREADLIB) rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.o $(TCC) -I. -o rbu$(EXE) $(TOP)/ext/rbu/rbu.c sqlite3.o \ diff --git a/manifest b/manifest index 4359ecc0b2..6cdd069a22 100644 --- a/manifest +++ b/manifest @@ -1,20 +1,20 @@ -C Merge\sthe\s"changebatch"\sfunctionality\sinto\sthis\sbranch. -D 2017-01-09T07:00:41.373 -F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e +C Merge\sall\srecent\senhancements\sfrom\strunk. +D 2017-03-24T15:09:47.005 +F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da -F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7 -F VERSION cddd8d88dc8202afa0ebc96da61fc4acbd1e96a5 +F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a +F README.md 2b15fae33852f2f53996774c21fb41e1d94181c4401a0e43ac93e11f2cc901b9 +F VERSION 3605fa447e4623f5ff4a6adc97b1fde9a257b8f2 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903 F autoconf/Makefile.am 1a47d071e3d5435f8f7ebff7eb6703848bbd65d4 -F autoconf/Makefile.msc b6d27f735911fd09a589d3c966936c47a5850c17 +F autoconf/Makefile.msc 92e3d7fd64dc5c23a23d33824ee4709aaea00a6331c4bb41a1aadcf6600a49f7 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7 F autoconf/README.txt 4f04b0819303aabaa35fff5f7b257fb0c1ef95f1 -F autoconf/configure.ac cacf2616abf6e4a569bde2ef365c143caeec40bc +F autoconf/configure.ac 2893b823ecc86cea13739f6c8109a41392254d1db08235c5615e0af5722c8578 F autoconf/tea/Makefile.in b438a7020446c8a8156e8d97c8914a04833da6fd F autoconf/tea/README 3e9a3c060f29a44344ab50aec506f4db903fb873 F autoconf/tea/aclocal.m4 52c47aac44ce0ddb1f918b6993e8beb8eee88f43 @@ -30,13 +30,13 @@ F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977 F config.h.in 6376abec766e9a0785178b1823b5a587e9f1ccbc F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55 -F configure fcfa56d5d66668c9263dc007724f62acff06dc83 x +F configure b15d030a8ae2e672d76c97b6667dacb98934a467 x F configure.ac 605173e829ab64514ed89f9b53d0da1739d7b0a0 F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/lemon.html b5a3c07d33ecb8e019ce8f7660fe2dbbad9d7977 F doc/pager-invariants.txt 27fed9a70ddad2088750c4a2b493b63853da2710 F doc/vfs-shm.txt e101f27ea02a8387ce46a05be2b1a902a021d37a -F ext/README.txt 913a7bd3f4837ab14d7e063304181787658b14e1 +F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd w ext/README.txt F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91 F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef @@ -70,9 +70,9 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c e028eb13432f108d2e22cded019fc980700e4e00 +F ext/fts3/fts3.c f0d5de1bc2155ba7cd7c0c1a751779a3a8857fa34d5c12f3b233a23fa2e79ea2 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h 89d0bd4595a0de384dac78e94b803de12586e8dd +F ext/fts3/fts3Int.h eb2502000148e80913b965db3e59f29251266d0a F ext/fts3/fts3_aux.c 9edc3655fcb287f0467d0a4b886a01c6185fe9f1 F ext/fts3/fts3_expr.c dfd571a24412779ac01f25c01d888c6ef7b2d0ef F ext/fts3/fts3_hash.c 29b986e43f4e9dd40110eafa377dc0d63c422c60 @@ -86,27 +86,28 @@ F ext/fts3/fts3_tokenize_vtab.c a27593ab19657166f6fa5ec073b678cc29a75860 F ext/fts3/fts3_tokenizer.c a22bf311a71f3efa9d7012d8cc48fc9b0f3dace7 F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 -F ext/fts3/fts3_unicode.c a93f5edc0aff44ef8b06d7cb55b52026541ca145 -F ext/fts3/fts3_unicode2.c c3d01968d497bd7001e7dc774ba75b372738c057 -F ext/fts3/fts3_write.c c3863f23b6b4623c8b9d5cf31c12ce4469f78ca9 +F ext/fts3/fts3_unicode.c 525a3bd9a7564603c5c061b7de55403a565307758a94600e8a2f6b00d1c40d9d +F ext/fts3/fts3_unicode2.c cc04fc672bfd42b1e650398cb0bf71f64f9aae032cfe75bbcfe75b9cf966029c +F ext/fts3/fts3_write.c a51d48d646974ee2fb4b17fcd5da0416a5759a32dcacc2cce2ba00d5a767848e F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 +F ext/fts3/tool/fts3cov.sh c331d006359456cf6f8f953e37f2b9c7d568f3863f00bb5f7eb87fea4ac01b73 F ext/fts3/tool/fts3view.c 202801a2056995b763864d60c2dee744d46f1677 F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 -F ext/fts3/unicode/mkunicode.tcl 2debed3f582d77b3fdd0b8830880250021571fd8 +F ext/fts3/unicode/mkunicode.tcl ab0543a3b2399092ea2dd75df1bef333405b0d7f6b8c4951a0fbb60e780cb69f F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h 62f3e33ceeb9a428db139f9c012186b371da1cc7 -F ext/fts5/fts5Int.h b2eda36e0f224365c8e23dc8f559311834f1c13f +F ext/fts5/fts5Int.h c629b24d2b92b99596f3b8e82289fddca06df6e1 F ext/fts5/fts5_aux.c 67acf8d51723cf28ffc3828210ba662df4b8d267 F ext/fts5/fts5_buffer.c 4c1502d4c956cd092c89ce4480867f9d8bf325cd F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857 -F ext/fts5/fts5_expr.c dc2cee9f56b1818b85df59304b8104a5dfb8ab60 +F ext/fts5/fts5_expr.c c6ecc2280162a3714d15dce2a8f2299f748b627c F ext/fts5/fts5_hash.c 880998e596b60f078348d48732ca4ad9a90caad2 F ext/fts5/fts5_index.c f67032a9a529ba52a545e6e3ab970764199c05d4 F ext/fts5/fts5_main.c f85281445dcf8be32d18841c93a6f90fe27dbfe2 -F ext/fts5/fts5_storage.c de0ed8a06738bde433afe11e92295ceaffbc4e58 +F ext/fts5/fts5_storage.c 8f0e65cb33bde8f449e1c9b4be4600d18b4da6e9 F ext/fts5/fts5_tcl.c 4a901f00c8553740dba63511603f5527d741c26a F ext/fts5/fts5_test_mi.c 783b86697ebf773c18fc109992426c0173a055bc F ext/fts5/fts5_test_tok.c db08af63673c3a7d39f053b36fd6e065017706be @@ -160,10 +161,12 @@ F ext/fts5/test/fts5fault8.test 6785af34bd1760de74e2824ea9c161965af78f85 F ext/fts5/test/fts5fault9.test e10e395428a9ea0596ebe752ff7123d16ab78e08 F ext/fts5/test/fts5faultA.test fa5d59c0ff62b7125cd14eee38ded1c46e15a7ea F ext/fts5/test/fts5faultB.test 7f3bba790fa172073ac314f9b8ed197390b61eca +F ext/fts5/test/fts5faultD.test cc5d1225556e356615e719c612e845d41bff7d5a F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741 F ext/fts5/test/fts5fuzz1.test bece4695fc169b61ab236ada7931c6e4942cbef9 F ext/fts5/test/fts5hash.test 06f9309ccb4d5050a131594e9e47d0b21456837d F ext/fts5/test/fts5integrity.test f5e4f8d284385875068ad0f3e894ce43e9de835d +F ext/fts5/test/fts5lastrowid.test 4fac1aba696dd6c956e03b0cf91f6f1f3aaec494 F ext/fts5/test/fts5matchinfo.test f7dde99697bcb310ea8faa8eb2714d9f4dfc0e1b F ext/fts5/test/fts5merge.test 9f65f090d214ff865c56bef4f864aaa1182af6e3 F ext/fts5/test/fts5merge2.test a6da3c16d694235938d1939f503cfa53f0943d75 @@ -184,11 +187,11 @@ F ext/fts5/test/fts5rowid.test 16908a99d6efc9ba21081b4f2b86b3fc699839a6 F ext/fts5/test/fts5simple.test 5da9b15ed534eb0be9f279d8a2bb2e24d30e4e38 F ext/fts5/test/fts5simple2.test 00839031878f52391562594fdab0503e424ee071 F ext/fts5/test/fts5simple3.test 25faa8cb8ad470c6f01f670bcc1317c19a89f091 -F ext/fts5/test/fts5synonym.test 6475d189c2e20d60795808f83e36bf9318708d48 +F ext/fts5/test/fts5synonym.test f964f9a98580dec06c995d4638ecb36eee60ae4a F ext/fts5/test/fts5synonym2.test aa4c43bd3b691ff80f658cb064f5ab40690e834e F ext/fts5/test/fts5tok1.test beb894c6f3468f10a574302f69ebe4436b0287c7 F ext/fts5/test/fts5tok2.test dcacb32d4a2a3f0dd3215d4a3987f78ae4be21a2 -F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 +F ext/fts5/test/fts5tokenizer.test f64ab5fe48fa1317ff6315264bfe26e73cbdefde F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 F ext/fts5/test/fts5unicode2.test 529ac7e8648c943bc87bfed1e427128a2f3f9e33 F ext/fts5/test/fts5unicode3.test 35c3d02aa7acf7d43d8de3bfe32c15ba96e8928e @@ -202,18 +205,20 @@ F ext/fts5/tool/loadfts5.tcl 95b03429ee6b138645703c6ca192c3ac96eaf093 F ext/fts5/tool/mkfts5c.tcl d1c2a9ab8e0ec690a52316f33dd9b1d379942f45 F ext/fts5/tool/showfts5.tcl d54da0e067306663e2d5d523965ca487698e722c F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43 -F ext/icu/icu.c 03ff6f90f3004a7e5a86205b581b2b7035ebf6e1 +F ext/icu/icu.c 84900472a088a3a172c6c079f58a1d3a1952c332 F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 +F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601d90b2 F ext/misc/amatch.c 211108e201105e4bb0c076527b8cfd34330fc234 F ext/misc/carray.c 40c27641010a4dc67e3690bdb7c9d36ca58b3c2d F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704 F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83 F ext/misc/csv.c 531a46cbad789fca0aa9db69a0e6c8ac9e68767d +F ext/misc/dbdump.c 34174d537318027b159e3341105866c05a4149376ede59988c82280bed144e6d F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c -F ext/misc/json1.c e0797ef1bfa637429ceab9140ecba620809ee02a +F ext/misc/json1.c 52f93b91b8bbbfcd84dca34be383974137e83a1673c71d5f728b16b68bb57d9c F ext/misc/memvfs.c e5225bc22e79dde6b28380f3a068ddf600683a33 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c 92699c8cd7d517ff610e6037e56506f8904dae2e @@ -222,12 +227,14 @@ F ext/misc/remember.c 8440f8d0b452c5cdefb62b57135ccd1267aa729d F ext/misc/rot13.c 1ac6f95f99b575907b9b09c81a349114cf9be45a F ext/misc/scrub.c 1c5bfb8b0cd18b602fcb55755e84abf0023ac2fb F ext/misc/series.c e11e534ada797d5b816d7e7a93c022306563ca35 +F ext/misc/sha1.c 0b9e9b855354910d3ca467bf39099d570e73db56 +F ext/misc/shathree.c fa185d7aee0ad0aca5e091b4a2db7baff11796170e5793b5de99e511a13af448 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 F ext/misc/spellfix.c a4723b6aff748a417b5091b68a46443265c40f0d F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512 F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95 F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178 -F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e +F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 F ext/rbu/rbu.c b2c0b5e6ae1a89affc0edfc127ebfa5f637a0ce4 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842 @@ -247,21 +254,24 @@ F ext/rbu/rbuB.test c25bc325b8072a766e56bb76c001866b405925c2 F ext/rbu/rbuC.test efe47db508a0269b683cb2a1913a425ffd39a831 F ext/rbu/rbu_common.tcl a38e8e2d4a50fd6aaf151633714c1b1d2fae3ead F ext/rbu/rbucrash.test 8d2ed5d4b05fef6c00c2a6b5f7ead71fa172a695 +F ext/rbu/rbucrash2.test b2ecbdd7bb72c88bd217c65bd00dafa07f7f2d4d F ext/rbu/rbudiff.test 3e605cf624d00d04d0fb1316a3acec4fbe3b3ac5 F ext/rbu/rbudor.test 99b05cc0df613e962c2c8085cfb05686a09cf315 F ext/rbu/rbufault.test cc0be8d5d392d98b0c2d6a51be377ea989250a89 F ext/rbu/rbufault2.test 9a7f19edd6ea35c4c9f807d8a3db0a03a5670c06 F ext/rbu/rbufault3.test 54a399888ac4af44c68f9f58afbed23149428bca +F ext/rbu/rbufault4.test 34e70701cbec51571ffbd9fbf9d4e0f2ec495ca7 F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda -F ext/rbu/rbuprogress.test e3e25fb7622641b8f2df7c6b7a7eb6fddfc46a4b +F ext/rbu/rbuprogress.test 1849d4e0e50616edf5ce75ce7db86622e656b5cf +F ext/rbu/rburesume.test 8acb77f4a422ff55acfcfc9cc15a5cb210b1de83 F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 F ext/rbu/rbuvacuum.test 4a977447c15c2581ab668781d9ef4294382530e0 -F ext/rbu/rbuvacuum2.test 2569205b74ff40fbf3bda2fce33a58eb40eebdcc -F ext/rbu/sqlite3rbu.c e074c38798b90591f7f0cf0032d62f152ce5a95e +F ext/rbu/rbuvacuum2.test 2074ab14fe66e1c7e7210c62562650dcd215bbaa +F ext/rbu/sqlite3rbu.c 2a89efba9eeba8e6c89a498dc195e8efbdde2694 F ext/rbu/sqlite3rbu.h 6fb6294c34a9ca93b5894a33bca530c6f08decba F ext/rbu/test_rbu.c 5aa22616afac6f71ebd3d9bc9bf1006cfabcca88 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c 078fc27417b48aaebe988621bf96ef0e1645b201 +F ext/rtree/rtree.c 0acd285bfacc347579a5df9fe947212fb99e2775a40c43f027c3a16936c58e7e F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e F ext/rtree/rtree1.test 42dadfc7b44a436cd74a1bebc0b9b689e4eaf7ec F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba @@ -272,7 +282,7 @@ F ext/rtree/rtree6.test 773a90db2dce6a8353dd0d5b64bca69b29761196 F ext/rtree/rtree7.test 1fa710b9e6bf997a0c1a537b81be7bb6fded1971 F ext/rtree/rtree8.test db79c812f9e4a11f9b1f3f9934007884610a713a F ext/rtree/rtree9.test b5eb13849545dfd271a54ff16784cb00d8792aea -F ext/rtree/rtreeA.test ace05e729a36e342d40cf94e9efc7b4723d9dcdf +F ext/rtree/rtreeA.test ac8b503931f2f397cc3c3303354e1e085dcadb86 F ext/rtree/rtreeB.test c85f9ce78766c4e68b8b89fbf2979ee9cfa82b4e F ext/rtree/rtreeC.test c0a9c67f2efa98b6fae12acb8a28348d231a481d F ext/rtree/rtreeD.test bdfaaf26df8b4eea7364039aca9150bc1e1f8825 @@ -287,9 +297,9 @@ F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F ext/session/changebatch1.test 9ceaa8f7b2a505933e250fbe6cbc550e4ce1e59d F ext/session/changebatchfault.test be49c793219bf387ad692a60856b921f0854ad6d F ext/session/changeset.c 4ccbaa4531944c24584bf6a61ba3a39c62b6267a -F ext/session/session1.test e5125b216d1e8c91e0984b361b0b68529e7c5dfb +F ext/session/session1.test c8a50e0e8581dc1a00e832aa59bb61f180404d44 F ext/session/session2.test 284de45abae4cc1082bc52012ee81521d5ac58e0 -F ext/session/session3.test a7a9ce59b8d1e49e2cc23d81421ac485be0eea01 +F ext/session/session3.test ce9ce3dfa489473987f899e9f6a0f2db9bde3479 F ext/session/session4.test 457b02bdc349eb01151e54de014df77abd3c08c8 F ext/session/session5.test 716bc6fafd625ce60dfa62ae128971628c1a1169 F ext/session/session6.test 443789bc2fca12e4f7075cf692c60b8a2bea1a26 @@ -302,21 +312,24 @@ F ext/session/sessionD.test d4744c78334162851d2a2f285c7e603e31b49aa2 F ext/session/sessionE.test e60a238c47f0feb3bb707e7f35e22be09c7e8f26 F ext/session/sessionF.test c2f178d4dfd723a5fd94a730ea2ccb44c669e3ce F ext/session/sessionG.test 01ef705096a9d3984eebdcca79807a211dee1b60 -F ext/session/session_common.tcl 9b696a341cf1d3744823715ed92bb19749b6c3d4 +F ext/session/session_common.tcl 7776eda579773113b30c7abfd4545c445228cb73 +F ext/session/session_speed_test.c edc1f96fd5e0e4b16eb03e2a73041013d59e8723 +F ext/session/sessionat.test b25d61d663ebc795506bf74079dc4ba0092fad25 F ext/session/sessionfault.test da273f2712b6411e85e71465a1733b8501dbf6f7 F ext/session/sessionfault2.test 04aa0bc9aa70ea43d8de82c4f648db4de1e990b0 -F ext/session/sqlite3changebatch.c 96011bdf72ac12ebf707f4d0f0e719e333631605 +F ext/session/sessionwor.test 2f3744236dc8b170a695b7d8ddc8c743c7e79fdc +F ext/session/sqlite3changebatch.c d5553b79e012ee2cb06c0a96bdf9dfe19e66354390ea0036cc46c4953142d517 F ext/session/sqlite3changebatch.h e72016998c9a22d439ddfd547b69e1ebac810c24 -F ext/session/sqlite3session.c 143d2a1547e760cd2653d1c7bf30acfb668de04f -F ext/session/sqlite3session.h 30b9ba3c5906d711b4e5b2e532d7a1f419942a29 +F ext/session/sqlite3session.c 459b276e12065e34036f19cdc72528ac49f2d9977d0a142b9ef9b38122fe7ec0 +F ext/session/sqlite3session.h ec4463a01e6e1c942f618f8944ceb96f05f5fec06f8de9af6f269fc216df1c68 F ext/session/test_session.c badd5da3cb561564b093745f7d843430d1d76347 -F ext/userauth/sqlite3userauth.h 19cb6f0e31316d0ee4afdfb7a85ef9da3333a220 +F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 -F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e +F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 87ee1e20124ca5cae5a5aa08608a0d8f63063eb8 +F main.mk a8a58a3b44912e2ab0c6e52a94f8dd97282e4ce192b2f760f81adfa62ec51512 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -329,38 +342,38 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c 3b23977620ce9662ac54443f65b87ba996e36121 -F src/analyze.c 3c4a63ff7a55faefecf6eb1589932fdbc06b2415 +F src/analyze.c 0d0ccf7520a201d8747ea2f02c92c26e26f801bc161f714f27b9f7630dde0421 F src/attach.c 8c476f8bd5d2afe11d925f890d30e527e5b0ce43 F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c 5e5bf5079aae45b91e71eb087f3eddd28deafafe -F src/btree.h 9306cfe42eed20cf2d0e407172704f5193af26cc -F src/btreeInt.h bade42398d4dcadd2263c6d17886e812e2471e87 -F src/build.c 58bd67d6648d6120592605768a7008fd52fb2656 +F src/btree.c a6be01292ef0d290dbb768943542a112c5425702fd606099241e2bd7502d6e30 +F src/btree.h d04766565d07aa84f1324f6092a912a34e98f8790d607842ae73a4e45c3047e1 +F src/btreeInt.h b2cdfe83b930ae61c83ff9adc36f1b0aeda53758ab5a0755de5a0aec2be2f3a1 +F src/build.c 3761b2cc63542368fdd39686180fb02574311ec9953195661d4926e1256fdf7a F src/callback.c 2e76147783386374bf01b227f752c81ec872d730 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e -F src/ctime.c 9f2296a4e5d26ebf0e0d95a0af4628f1ea694e7a -F src/date.c dc3f1391d9297f8c748132813aaffcb117090d6e +F src/ctime.c 47d91a25ad8f199a71a5b1b7b169d6dd0d6e98c5719eca801568798743d1161c +F src/date.c ee676e7694dfadbdd2fde1a258a71be8360ba5ae F src/dbstat.c 19ee7a4e89979d4df8e44cfac7a8f905ec89b77d -F src/delete.c c8bc10d145c9666a34ae906250326fdaa8d58fa5 -F src/expr.c f06f41e5e5daca10fb090e70a2502dcc0dbc992b +F src/delete.c 0d9d5549d42e79ce4d82ff1db1e6c81e36d2f67c +F src/expr.c f12a581f342a6fd85d14c31e4fb84f16b3dd107f54d7728dddb62cebc79d7ce1 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 2e9aabe1aee76273aff8a84ee92c464e095400ae -F src/func.c c67273e1ec08abbdcc14c189892a3ff6eeece86b +F src/func.c 9d52522cc8ae7f5cdadfe14594262f1618bc1f86083c4cd6da861b4cf5af6174 F src/global.c 4a34512d82fc5aa13c802db06bcfff5e1d3de955 F src/hash.c 63d0ee752a3b92d4695b2b1f5259c4621b2cfebd F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4 F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 -F src/insert.c 7af46a3be2656f5e13791464625d93d6b07b8612 -F src/legacy.c 75d3023be8f0d2b99d60f905090341a03358c58e -F src/loadext.c 5d6642d141c07d366e43d359e94ec9de47add41d -F src/main.c e207b81542d13b9f13d61e78ca441f9781f055b0 -F src/malloc.c f3fad34cd570022abca558c573f1761fb09a8212 +F src/insert.c d4bb3a135948553d18cf992f76f7ed7b18aa0327f250607b5a6671e55d9947d5 +F src/legacy.c e88ed13c2d531decde75d42c2e35623fb9ce3cb0 +F src/loadext.c a68d8d1d14cf7488bb29dc5311cb1ce9a4404258 +F src/main.c 158326243c5ddc8b98a1e983fa488650cf76d760 +F src/malloc.c 89c98e3619d362dcffa5c1c639b364b65b474751 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 -F src/mem1.c 6919bcf12f221868ea066eec27e579fed95ce98b +F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 F src/mem3.c 8768ac94694f31ffaf8b4d0ea5dc08af7010a35a F src/mem5.c 9bf955937b07f8c32541c8a9991f33ce3173d944 @@ -370,44 +383,44 @@ F src/mutex.c 8e45800ee78e0cd1f1f3fe8e398853307f4a085c F src/mutex.h 779d588e3b7756ec3ecf7d78cde1d84aba414f85 F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4 F src/mutex_unix.c 27bb6cc49485ee46711a6580ab7b3f1402211d23 -F src/mutex_w32.c 5e6fe1c298fb5a8a15aaed4161d5759311431c17 +F src/mutex_w32.c a898fa969823b100c0f5fdc57e54c9a1e419ab4d F src/notify.c 9711a7575036f0d3040ba61bc6e217f13a9888e7 F src/os.c add02933b1dce7a39a005b00a2f5364b763e9a24 F src/os.h 8e976e59eb4ca1c0fca6d35ee803e38951cb0343 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c 30e2c43e4955db990e5b5a81e901f8aa74cc8820 -F src/os_win.c cf90abd4e50d9f56d2c20ce8e005aff55d7bd8e9 +F src/os_win.c 2a6c73eef01c51a048cc4ddccd57f981afbec18a F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a -F src/pager.c e5b8c9a975e0c208bc8886bf83b8780b6e578ac5 -F src/pager.h 0105583b49a9126b9d137f98ec57caefc64f51d4 -F src/parse.y bb387a078f2f7f78e5362479bfef4a05b9fd40b8 -F src/pcache.c 51070ec9b8251bbf9c6ea3d35fd96a458752929e +F src/pager.c 98a32b803fee30beaebe24ecbbf06bb2f8bc727957c0e410944b4597cbe8e81e +F src/pager.h 5e7b4e5afdcbdf558c211f27786672b3d2536003d0fb6c4888addb500c826e15 +F src/parse.y f48aeacb59e7f272b042872332af246a97847d234d17dfdacf719b8aa97f7236 +F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 F src/pcache1.c e3967219b2a92b9edcb9324a4ba75009090d3953 -F src/pragma.c 0e7a7c6f1c6fd8ff50c0fff65b8bb80174bc49c5 -F src/pragma.h 61aa5389118594bebb28120a6720401aee34ce1a +F src/pragma.c 2b244434e76c7075edbcfd9e4d634899af0944ff01183b126d4671f7407c2368 +F src/pragma.h c9c763958fec92b04125571472c9500b351c5f7f F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a -F src/printf.c ff10a9b9902cd2afe5f655f3013c6307d969b1fd +F src/printf.c 8757834f1b54dae512fb25eb1acc8e94a0d15dd2290b58f2563f65973265adb2 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 -F src/resolve.c bb070cf5f23611c44ab7e4788803684e385fc3fb +F src/resolve.c 3e518b962d932a997fae373366880fc028c75706 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac -F src/select.c 4437d9d5d56b6ffdedabf394c7fe3a07ff521ce9 -F src/shell.c 6095531aa900decdaa765e0f3993fba7153c92c1 -F src/sqlite.h.in 29bda4bee01248a5650567d7a22fac39bad1b542 +F src/select.c 2496d0cc6368dabe7ad2e4c7f5ed3ad9aa3b4d11cd90f33fa1d1ef72493f43aa +F src/shell.c ce39c4bbbaa02484cb45141034fd363c3bac80a6aa22b0bb3cecea1c6a4c0979 +F src/sqlite.h.in 723107d97f2345a7c103632169dc61366121c4ab65d75a7d83c6dc0e5bbe5ca4 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae -F src/sqliteInt.h 0081f4d95b49a1accc29e5c96cf35296d0f68098 -F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 +F src/sqliteInt.h 33e415da02eea725392ed9a9bf00cac7a19fed0c1f75678e6b23a3aa0ba178e9 +F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 -F src/table.c 5226df15ab9179b9ed558d89575ea0ce37b03fc9 -F src/tclsqlite.c 205c66b9b81d97978a155caa3ef5be9c4de2b174 +F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 +F src/tclsqlite.c 6c2151b6d8d98e183a04466d40df8889c0574d79 F src/test1.c 8a98191a1da8e100f77cdb5cc716df67d405028d F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5 F src/test3.c d03f5b5da9a2410b7a91c64b0d3306ed28ab6fee F src/test4.c 18ec393bb4d0ad1de729f0b94da7267270f3d8e6 F src/test5.c 328aae2c010c57a9829d255dc099d6899311672d -F src/test6.c 55aa2775c154415dcf4ed7cd1e19a193122b3a02 +F src/test6.c 004ad42f121f693b8cbe060d1a330678abc61620 F src/test7.c 5612e9aecf934d6df7bba6ce861fdf5ba5456010 F src/test8.c 4f4904721167b32f7a4fa8c7b32a07a673d6cc86 F src/test9.c 12e5ba554d2d1cbe0158f6ab3f7ffcd7a86ee4e5 @@ -415,10 +428,10 @@ F src/test_async.c 195ab49da082053fdb0f949c114b806a49ca770a F src/test_autoext.c 915d245e736652a219a907909bb6710f0d587871 F src/test_backup.c bf5da90c9926df0a4b941f2d92825a01bbe090a0 F src/test_bestindex.c d23f80d334c59662af69191854c76b8d3d0c8c96 -F src/test_blob.c 6a4c7920d1d9c6cc0f7aa50c89c4f80016aeda83 +F src/test_blob.c f65ac717da2618691cf9dad094e6da0219dcd208 F src/test_btree.c 8b2dc8b8848cf3a4db93f11578f075e82252a274 -F src/test_config.c f3b4459a0c65933783030fb4a05073571f8fa897 -F src/test_delete.c 8499d4d323f2ec8e28301deb3d6ddd8eef8b8139 +F src/test_config.c d3925a89099af6407929fe642adeb950b7e6150d738b4113b2a62ae3eb95a7cd +F src/test_delete.c af7eab5702f853fb1c62a5f7665e2234cf1ae17b F src/test_demovfs.c a0c3bdd45ed044115c2c9f7779e56eafff18741e F src/test_devsym.c 4e58dec2602d8e139ca08659f62a62450587cb58 F src/test_fs.c e16cbe68d3b107e00a907c20a9a02629870eb69b @@ -427,7 +440,7 @@ F src/test_hexio.c 1d4469ca61ab202a1fcec6543f584d2407205e8d F src/test_init.c 4413c211a94b62157ca4c145b3f27c497f03c664 F src/test_intarray.c 988fc61cb0ff539f4172c0d95f15287c92516f64 F src/test_intarray.h f3b7672f5d1056eac563c0d6ea8480a660b1475c -F src/test_journal.c d3b83f2bcb7792c709e57abddc456a2b1818643a +F src/test_journal.c 619f2aa10e0d7a5f87c0f06825bc61dfce1c6b9c7f3ad990fb13de6c3b8874a3 F src/test_loadext.c 337056bae59f80b9eb00ba82088b39d0f4fe6dfd F src/test_malloc.c c05f6c40bd6c8bfe5f1718212f81fd5687f91766 F src/test_multiplex.c e054459f7633f3ff8ce1245da724f9a8be189e4e @@ -448,37 +461,38 @@ F src/test_tclvar.c df9fe1213c2634687a9ca0b0bec0d2119d359ae3 F src/test_thread.c 911d15fb14e19c0c542bdc8aabf981c2f10a4858 F src/test_vfs.c f0186261a24de2671d080bcd8050732f0cb64f6e F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 -F src/test_windirent.c 600398db0198ca1c77ca183831bf456746b6f5c4 -F src/test_windirent.h 7edc57e2faa727026dbd5d010dd0e2e665d5aa01 +F src/test_windirent.c 17f91f5f2aa1bb7328abb49414c363b5d2a9d3ff +F src/test_windirent.h 5d67483a55442e31e1bde0f4a230e6e932ad5906 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c -F src/tokenize.c 5c2f516876fc27fbd7753913f032f49eb89e83b5 -F src/treeview.c 4e44ade3bfe59d82005039f72e09333ce2b4162c +F src/tokenize.c dc748c5afcb9e5beb3ef5651bc99a4622e30f6a1 +F src/treeview.c 84d0ac737e1231702679f0289180021e19c5cc186ec413e8dcb704a887c76ec8 F src/trigger.c c9f0810043b265724fdb1bdd466894f984dfc182 -F src/update.c 1da7c462110bffed442a42884cb0d528c1db46d8 +F src/update.c 456d4a4656f8a03c2abc88a51b19172197400e58 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c -F src/util.c a88b0466fddf445ce752226d4698ca3faada620a -F src/vacuum.c ebcf45799c7a82729a496ef5b65c5d3b2715ef8e -F src/vdbe.c 0254dc5bfb4d00b5ccc359332eeef26eb66dc5ee -F src/vdbe.h b0866e4191f096f1c987a84b042c3599bdf5423b -F src/vdbeInt.h 281cb70332dc8b593b8c7afe776f3a2ba7d4255e -F src/vdbeapi.c d6ebaa465f070eb1af8ba4e7b34583ece87bdd24 -F src/vdbeaux.c d1e423b8a659705cbad5a502a219fba22f306048 -F src/vdbeblob.c f4f98ea672b242f807c08c92c7faaa79e5091b65 +F src/util.c ca8440ede81e155d15cff7c101654f60b55a9ae6 +F src/vacuum.c 8aa1db0b7052269890ac03089ed030eefa88480897915f727703ca74d0676380 +F src/vdbe.c 9a6b75f8265756bf49c776fa9f561b0c13a123064734e8237586e4c4c11897b6 +F src/vdbe.h 59998ffd71d7caa8886bc78dafaf8caeccd4c13c +F src/vdbeInt.h 4e4b15b2e1330e1636e4e01974eab2b0b985092f +F src/vdbeapi.c 5b08d82592bcff4470601fe78aaabebd50837860 +F src/vdbeaux.c 0a2d70e76ffe87869d4280f23477994f747fb0fa21305f4f9afcbdce22e4bafd +F src/vdbeblob.c 359891617358deefc85bef7bcf787fa6b77facb9 F src/vdbemem.c 3b5a9a5b375458d3e12a50ae1aaa41eeec2175fd F src/vdbesort.c eda25cb2d1727efca6f7862fea32b8aa33c0face F src/vdbetrace.c 41963d5376f0349842b5fc4aaaaacd7d9cdc0834 -F src/vtab.c c4bbe0f870f52036553f8098aee0703997f0577a +F src/vtab.c 007513c2ef52472fcdea6a741683d50662e82790 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 2858e71b30d521c80949fa10ed6b116367865594 F src/wal.h 8659519a248ef0f33e575b20ab04211cebe1f430 -F src/walker.c 91a6df7435827e41cff6bb7df50ea00934ee78b0 -F src/where.c 6bbf9284f4f15a6fa48663d033870cc0d7f5ee66 -F src/whereInt.h 2bcc3d176e6091cb8f50a30b65c006e88a73614d -F src/wherecode.c e04ac8f24c3ac8621df6c3be3ac8c7d4fa893745 -F src/whereexpr.c 87ecdf24beba4498e4380b31c4131febb0a6ceaa +F src/walker.c b71a992b413b3a022572eccf29ef4b4890223791 +F src/where.c e815093e5ee039b6b4eb19b646d22deb1a3a523f +F src/whereInt.h 2d50c2b74a33be44cb68fdecee30b4d93552f1f4 +F src/wherecode.c 677e95413c472c0b413023b6b69a47f40fce1b04 +F src/whereexpr.c 130cdd1a43af71b19755270fb1224874cf55158c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd +F test/affinity3.test 6a101af2fc945ce2912f6fe54dd646018551710d F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test b35b4cd69fc913f90d39a575e171e1116c3a4bb7 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 @@ -518,6 +532,7 @@ F test/attachmalloc.test 3a4bfca9545bfe906a8d2e622de10fbac5b711b0 F test/auth.test c6ede04bee65637ff354b43fc1235aa560c0863e F test/auth2.test 9eb7fce9f34bf1f50d3f366fb3e606be5a2000a1 F test/auth3.test 0d48b901cf111c14b4b1b5205c7d28f1a278190f +F test/autoanalyze1.test b9cc3f32a990fa56669b668d237c6d53e983554ae80c0604992e18869a0b2dec F test/autoinc.test 6ae8fb69c9f656962464ae4e6667045d0dfc3b46 F test/autoindex1.test 14b63a9f1e405fe6d5bfc8c8d00249c2ebaf13ea F test/autoindex2.test 12ef578928102baaa0dc23ad397601a2f4ecb0df @@ -573,7 +588,7 @@ F test/capi3d.test 485048dc5cd07bc68011e4917ad035ad6047ab82 F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef -F test/check.test 85a84bfc4be0e83f668747211c7cd45a6721d485 +F test/check.test 33a698e8c63613449d85d624a38ef669bf20331daabebe3891c9405dd6df463a F test/close.test 83947daf3b700631f90f4850ddaab455be4af73d F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91 @@ -587,7 +602,7 @@ F test/collate7.test 8ec29d98f3ee4ccebce6e16ce3863fb6b8c7b868 F test/collate8.test cd9b3d3f999b8520ffaa7cc1647061fc5bab1334 F test/collate9.test 3adcc799229545940df2f25308dd1ad65869145a F test/collateA.test b8218ab90d1fa5c59dcf156efabb1b2599c580d6 -F test/collateB.test 8ec2accd2d7166c1eff0d2a39bc90262c6f89632 +F test/collateB.test 1e68906951b846570f29f20102ed91d29e634854ee47454d725f2151ecac0b95 F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1 F test/colname.test 08948a4809d22817e0e5de89c7c0a8bd90cb551b F test/concfault.test 500f17c3fcfe7705114422bcc6ddd3c740001a43 @@ -609,7 +624,7 @@ F test/corrupt8.test 2399dfe40d2c0c63af86706e30f3e6302a8d0516 F test/corrupt9.test 730a3db08d4ab9aa43392ea30d9c2b4879cbff85 F test/corruptA.test 53e56dafd180addcdadb402244b8cb9771d2ba26 F test/corruptB.test 73a8d6c0b9833697ecf16b63e3c5c05c945b5dec -F test/corruptC.test 4ef10844eba5213bd262f4d96784d7fcda114afe +F test/corruptC.test 46ec43bd90a02fd7b37ad8a7a949c55aa5717f89 F test/corruptD.test b3c205fac7952b1de645ce44bb02335cd9e3e040 F test/corruptE.test 82ccf4f8f543fdbedd4aa42c709cb077f7374c62 F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 @@ -617,6 +632,7 @@ F test/corruptG.test adf79b669cbfd19e28c8191a610d083ae53a6d51 F test/corruptH.test 79801d97ec5c2f9f3c87739aa1ec2eb786f96454 F test/corruptI.test 075fe1d75aa1d84e2949be56b6264376c41502e4 F test/corruptJ.test 4d5ccc4bf959464229a836d60142831ef76a5aa4 +F test/corruptK.test 814a59ec699d8546b4e29005fba3d16e933ef2fe F test/cost.test 1eedbfd868f806f3fa08ff072b04cf270dcf61c8 F test/count.test cb2e0f934c6eb33670044520748d2ecccd46259c F test/coveridxscan.test b629e896b14df2f000a99b8d170d80589c46562c @@ -635,9 +651,9 @@ F test/cse.test 277350a26264495e86b1785f34d2d0c8600e021c F test/csv01.test e0ba3caaa57e4c667a0b45977689fb8082f14348 F test/ctime.test ff6c38e822459d6ca743c34901caf57740b08b54 F test/cursorhint.test 7bc346788390475e77a345da2b92270d04d35856 -F test/cursorhint2.test fa41f0d997e67db921d08c31e73111b32811201a -F test/date.test a6a5a48b90907bca9fbcc79a30be5a715c1ab2fc -F test/dbfuzz.c 8cc2bdb818b4483a052f9f80f96be74cbd9a6e1d +F test/cursorhint2.test 8457e93d97f665f23f97cdbc8477d16e3480331b +F test/date.test 9b73bbeb1b82d9c1f44dec5cf563bf7da58d2373 +F test/dbfuzz.c 73047c920d6210e5912c87cdffd9a1c281d4252e F test/dbstatus.test 73149851b3aff14fc6db478e58f9083a66422cf5 F test/dbstatus2.test e93ab03bfae6d62d4d935f20de928c19ca0ed0ab F test/default.test 0cb49b1c315a0d81c81d775e407f66906a2a604d @@ -677,6 +693,7 @@ F test/e_wal.test ae9a593207a77d711443ee69ffe081fda9243625 F test/e_walauto.test 248af31e73c98df23476a22bdb815524c9dc3ba8 F test/e_walckpt.test 28c371a6bb5e5fe7f31679c1df1763a19d19e8a0 F test/e_walhook.test 4c0613a0c76e7a9d5c4c211e1b4cbcc1143914df +F test/emptytable.test a38110becbdfa6325cd65cb588dca658cd885f62 F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea F test/enc2.test 83437a79ba1545a55fb549309175c683fb334473 F test/enc3.test 6807f7a7740a00361ca8d0ccd66bc60c8dc5f2b6 @@ -685,7 +702,7 @@ F test/eqp.test 3fe051af50921284189d1970eb653f9fcf5117d2 F test/errmsg.test f31592a594b44ee121371d25ddd5d63497bb3401 F test/eval.test a64c9105d6ff163df7cf09d6ac29cdad5922078c F test/exclusive.test 9a57bd66e39144b888ca75c309914fcdefb4e3f9 -F test/exclusive2.test 32798111aae78a5deec980eee383213f189df308 +F test/exclusive2.test 984090e8e9d1b331d2e8111daf6e5d61dda0bef7 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 F test/exists.test 79a75323c78f02bbe9c251ea502a092f9ef63dac F test/expr.test 66a2c9ac34f74f036faa4092f5402c7d3162fc93 @@ -805,6 +822,7 @@ F test/fts4growth.test e5390da74619cacc389711bac9349640b32c4f9a F test/fts4growth2.test 13ad4e76451af6e6906c95cdc725d01b00044269 F test/fts4incr.test 4e353a0bd886ea984e56fce9e77724fc923b8d0d F test/fts4langid.test 65a7332c9bc257919e259a304aa8a38c41655b9d +F test/fts4lastrowid.test fa5e157955a3121615ef3e16ff5196e96c9e1e64 F test/fts4merge.test d2b39f6b1bd4a9738a13540e2d044cba11c43d47 F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 F test/fts4merge3.test 8d9ccb4a3d41c4c617a149d6c4b13ad02de797d0 @@ -825,7 +843,7 @@ F test/fuzz2.test 76dc35b32b6d6f965259508508abce75a6c4d7e1 F test/fuzz3.test b47377143f0c80f91ed29d722861077ff34415d5 F test/fuzz_common.tcl a87dfbb88c2a6b08a38e9a070dabd129e617b45b F test/fuzz_malloc.test 328f70aaca63adf29b4c6f06505ed0cf57ca7c26 -F test/fuzzcheck.c a87e6067a8d19844bade916841cb76150ecf24a2 +F test/fuzzcheck.c 2152602232c96d9c790eff3013e1369ce59de3203fa0b75bc613531448454e61 F test/fuzzdata1.db 7ee3227bad0e7ccdeb08a9e6822916777073c664 F test/fuzzdata2.db f03a420d3b822cc82e4f894ca957618fbe9c4973 F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba @@ -838,7 +856,8 @@ F test/gcfault.test dd28c228a38976d6336a3fc42d7e5f1ad060cb8c F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 F test/hexlit.test 4a6a5f46e3c65c4bf1fa06f5dd5a9507a5627751 F test/hidden.test 23c1393a79e846d68fd902d72c85d5e5dcf98711 -F test/hook.test 3a01b876691f9151d3e44562354f7d663ff90fce +F test/hook.test dbc0b87756e1e20e7497b56889c9e9cd2f8cc2b5 +F test/hook2.test b9ff3b8c6519fb67f33192f1afe86e7782ee4ac8 F test/icu.test 73956798bace8982909c00476b216714a6d0559a F test/ieee754.test 806fc0ce7f305f57e3331eaceeddcfec9339e607 F test/imposter1.test c3f1db2d3db2c24611a6596a3fc0ffc14f1466c8 @@ -852,7 +871,7 @@ F test/incrblob2.test a5ce5ed1d0b01e2ed347245a21170372528af0a5 F test/incrblob3.test d8d036fde015d4a159cd3cbae9d29003b37227a4 F test/incrblob4.test 21a52a6843a56cdcce968c6a86b72a7066d0e6ba F test/incrblob_err.test 69f9247fed50278d48ea710d1a8f9cdb09e4c0b8 -F test/incrblobfault.test 280474078f6da9e732cd2a215d3d854969014b6e +F test/incrblobfault.test 74dd8ac108304cea0b4a0df6df63a1567e558758 F test/incrcorrupt.test 6c567fbf870aa9e91866fe52ce6f200cd548939a F test/incrvacuum.test d2a6ddf5e429720b5fe502766af747915ccf6c32 F test/incrvacuum2.test 7d26cfda66c7e55898d196de54ac4ec7d86a4e3d @@ -867,7 +886,7 @@ F test/index6.test b4fc812290067a578b98bb2667b676db89e202a7 F test/index7.test 7feababe16f2091b229c22aff2bcc1d4d6b9d2bb F test/index8.test bc2e3db70e8e62459aaa1bd7e4a9b39664f8f9d7 F test/indexedby.test 9c4cd331224e57f79fbf411ae245e6272d415985 -F test/indexexpr1.test 7d243fac508b4a99fb900ffe34eb488312cfce84 +F test/indexexpr1.test 038b3befa74e5a75126b6e9dd2ae5df61c1c7cf7 F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 F test/insert.test 38742b5e9601c8f8d76e9b7555f7270288c2d371 @@ -889,7 +908,7 @@ F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b F test/join.test f9d4a28dec81c6e9dc21b73518e024d73b5ebf57 -F test/join2.test f2171c265e57ee298a27e57e7051d22962f9f324 +F test/join2.test a48f723c5692e2cbb23a9297ac2720cb77d51a70 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 F test/join5.test bc98ea4b4e5003f5b1453701ebb8cd7d1c01a550 @@ -903,8 +922,9 @@ F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa F test/json101.test c0897616f32d95431f37fd291cb78742181980ac F test/json102.test bf3fe7a706d30936a76a0f7a0375e1e8e73aff5a F test/json103.test c5f6b85e69de05f6b3195f9f9d5ce9cd179099a0 +F test/json104.test 877d5845f6303899b7889ea5dd1bea99076e3100574d5c536082245c5805dcaa F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff -F test/kvtest.c 2c66ddefcd03c2caa337f6dd79e6c82368af83df +F test/kvtest.c b9a9822dda05a1aa481215a52e2fc93cd8b22ee5 F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 F test/like.test 0603f4fa0dad50987f70032c05800cbfa8985302 @@ -928,7 +948,7 @@ F test/make-where7.tcl 05c16b5d4f5d6512881dfec560cb793915932ef9 F test/malloc.test 21c213365f2cca95ab2d7dc078dc8525f96065f8 F test/malloc3.test e3b32c724b5a124b57cb0ed177f675249ad0c66a F test/malloc4.test 957337613002b7058a85116493a262f679f3a261 -F test/malloc5.test 02ed7c5313f0a68d95f2dfca8c8962132bd1f04b +F test/malloc5.test 192d0b5ee1d023a07d5164c34cf47d010da73da1 F test/malloc6.test 2f039d9821927eacae43e1831f815e157659a151 F test/malloc7.test 7c68a32942858bc715284856c5507446bba88c3a F test/malloc8.test 9b7a3f8cb9cf0b12fff566e80a980b1767bd961d @@ -946,6 +966,7 @@ F test/mallocI.test 6c23a71df077fa5d387be90e7e669c5b368ca38a F test/mallocJ.test b5d1839da331d96223e5f458856f8ffe1366f62e F test/mallocK.test 27cb5566a6e5f2d76f9d4aa2eca45524401fd61e F test/mallocL.test fb311ff80afddf3b1a75e52289081f4754d901dc +F test/mallocM.test 78bbe9d3da84a5c679123cdb40d7b2010b18fc46e13897e4f253c6ba6fbff134 F test/malloc_common.tcl aac62499b76be719fac31e7a3e54a7fd53272e7f F test/manydb.test 28385ae2087967aa05c38624cec7d96ec74feb3e F test/mem5.test c6460fba403c5703141348cd90de1c294188c68f @@ -999,8 +1020,8 @@ F test/orderby7.test 3d1383d52ade5b9eb3a173b3147fdd296f0202da F test/orderby8.test 23ef1a5d72bd3adcc2f65561c654295d1b8047bd F test/orderby9.test 87fb9548debcc2cd141c5299002dd94672fa76a3 F test/oserror.test b32dc34f2363ef18532e3a0a7358e3e7e321974f -F test/ossfuzz.c e469138f4be3e92df6173b79b3b216ab6e17b407 -F test/ossshell.c d9f1a6f43e7bab45d6be857a5800f5d4a1861db3 +F test/ossfuzz.c f5abed3177f719df3c3109901fcdd26b9fb7f581c8da50fc26f3a81ddfb2c2ae +F test/ossshell.c 296ab63067841bd1b1e97b46a0b2af48ee7f69d50d1a723008bee12dd7122622 F test/ovfl.test 199c482696defceacee8c8e0e0ef36da62726b2f F test/pager1.test 841868017e9dd3cb459b8d78862091a7d9cff21d F test/pager2.test 67b8f40ae98112bcdba1f2b2d03ea83266418c71 @@ -1014,13 +1035,13 @@ F test/pagesize.test 5769fc62d8c890a83a503f67d47508dfdc543305 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff -F test/permutations.test cd0b7bc04bf5e50d3a993d24c834d591f7d4f5fe +F test/permutations.test af720e7d139e7e5417341d0f0eef2b911c0b067852138dc2f5b6a451b5725118 F test/pragma.test 1e94755164a3a3264cd39836de4bebcb7809e5f8 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f F test/pragma3.test 14c12bc5352b1e100e0b6b44f371053a81ccf8ed F test/pragma4.test 6e85b6eab8e61ffc9c7db59d842276674e8e3264 F test/printf.test b3ff34e73d59124140eaf89f7672e21bc2ca5fcc -F test/printf2.test 0b61566dd1c0f0b802f59dffa228c5dc5aa6b054 +F test/printf2.test 9e6db85f81c63f2367c34a9d7db384088bd374ad F test/progress.test ebab27f670bd0d4eb9d20d49cef96e68141d92fb F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc F test/queryonly.test 5f653159e0f552f0552d43259890c1089391dcca @@ -1101,7 +1122,7 @@ F test/sharedA.test 0cdf1a76dfa00e6beee66af5b534b1e8df2720f5 F test/sharedB.test 16cc7178e20965d75278f410943109b77b2e645e F test/shared_err.test 2f2aee20db294b9924e81f6ccbe60f19e21e8506 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 -F test/shell1.test 52ac23a345772ab0d6d3241a21a633fdaa3ed581 +F test/shell1.test bc1ca4161e1f459c9cd412d3f21f4ee635cf2322 F test/shell2.test e242a9912f44f4c23c3d1d802a83e934e84c853b F test/shell3.test 9b95ba643eaa228376f06a898fb410ee9b6e57c1 F test/shell4.test 89ad573879a745974ff2df20ff97c5d6ffffbd5d @@ -1136,7 +1157,7 @@ F test/speed3.test 694affeb9100526007436334cf7d08f3d74b85ef F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b -F test/speedtest1.c 4e8ea6165046f02e1cfe0f4700256e91c981ec10 +F test/speedtest1.c 7b1ab42b097b484c18d99e1d1c71a6a0c9c87a7a F test/spellfix.test f9c1f431e2c096c8775fec032952320c0e4700db F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3 F test/spellfix3.test 0f9efaaa502a0e0a09848028518a6fb096c8ad33 @@ -1154,9 +1175,10 @@ F test/subtype1.test 7fe09496352f97053af1437150751be2d0a0cae8 F test/superlock.test ec94f0556b6488d97f71c79f9061ae08d9ab8f12 F test/symlink.test c9ebe7330d228249e447038276bfc8a7b22f4849 F test/sync.test 2f84bdbc2b2df1fcb0220575b4b9f8cea94b7529 +F test/sync2.test 29af00fe9e468a1e20315d1edced8f8a32ac156df983ae9b53802e7d472675d5 F test/syscall.test f59ba4e25f7ba4a4c031026cc2ef8b6e4b4c639c F test/sysfault.test c9f2b0d8d677558f74de750c75e12a5454719d04 -F test/tabfunc01.test 8b2ef53caa37854864c89e1e57e8a10efd4f5e43 +F test/tabfunc01.test 699251cb99651415218a891384510a685c7ab012 F test/table.test b708f3e5fa2542fa51dfab21fc07b36ea445cb2f F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 @@ -1168,7 +1190,7 @@ F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptable2.test cd396beb41117a5302fff61767c35fa4270a0d5e F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc -F test/tester.tcl 4ce5afd5e192db4cae178e1a983b060e0f08c5d6 +F test/tester.tcl 581f0185434daf7026ccede4c07e8d1479186ec5 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1204,7 +1226,7 @@ F test/tkt-5e10420e8d.test 904d1687b3c06d43e5b3555bbcf6802e7c0ffd84 F test/tkt-5ee23731f.test 9db6e1d7209dc0794948b260d6f82b2b1de83a9f F test/tkt-6bfb98dfc0.test 24780633627b5cfc0635a5500c2389ebfb563336 F test/tkt-752e1646fc.test ea78d88d14fe9866bdd991c634483334639e13bf -F test/tkt-78e04e52ea.test 813779f8888f3ca226df656c4eef078f9635f3c9 +F test/tkt-78e04e52ea.test 1b2e6bf4f1d9887b216b6da774e5f25915ec8118 F test/tkt-7a31705a7e6.test e75a2bba4eec801b92c8040eb22096ac6d35e844 F test/tkt-7bbfb7d442.test 7b2cd79c7a17ae6750e75ec1a7846712a69c9d18 F test/tkt-80ba201079.test 105a721e6aad0ae3c5946d7615d1e4d03f6145b8 @@ -1360,6 +1382,7 @@ F test/unique2.test 3674e9f2a3f1fbbfd4772ac74b7a97090d0f77d2 F test/unixexcl.test d936ba2b06794018e136418addd59a2354eeae97 F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8 F test/update.test 6c68446b8a0a33d522a7c72b320934596a2d7d32 +F test/update2.test 81772e5c9b93d1006d66d65e9a1327281bdec4e8 F test/uri.test 3481026f00ade6dfe8adb7acb6e1e47b04369568 F test/uri2.test 9d3ba7a53ee167572d53a298ee4a5d38ec4a8fb7 F test/userauth01.test e740a2697a7b40d7c5003a7d7edaee16acd349a9 @@ -1373,7 +1396,7 @@ F test/vacuummem.test 7b42abb3208bd82dd23a7536588396f295a314f2 F test/varint.test ab7b110089a08b9926ed7390e7e97bdefeb74102 F test/veryquick.test 57ab846bacf7b90cf4e9a672721ea5c5b669b661 F test/view.test 765802c7a66d37fabd5ac8e2f2dbe572b43eb9ab -F test/vtab1.test 7c4b81abd88361ada9cbe414c459efca26be6bda +F test/vtab1.test ed4a576231d8a36e70fd18e2b79b621b31e7f22a F test/vtab2.test f8cd1bb9aba7143eba97812d9617880a36d247ad F test/vtab3.test b45f47d20f225ccc9c28dc915d92740c2dee311e F test/vtab4.test 8e73ed268f3d596bc3590f45fc948fb40f28e9c3 @@ -1388,7 +1411,7 @@ F test/vtabC.test 4528f459a13136f982e75614d120aef165f17292 F test/vtabD.test 05b3f1d77117271671089e48719524b676842e96 F test/vtabE.test d5024aa42754962f6bb0afd261681686488e7afe F test/vtabF.test 1918844c7c902f6a16c8dacf1ec8f84886d6e78b -F test/vtabH.test 97f61b0253260831af6232163f7852e2653baed6 +F test/vtabH.test 5f9253eb9e41ba9fe94f4aa3e9230191893d7764 F test/vtabI.test 751b07636700dbdea328e4265b6077ccd6811a3f F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 @@ -1464,10 +1487,10 @@ F test/wordcount.c 06efb84b7c48a4973c2c24ea06c93d00bce24389 F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa F test/zerodamage.test e59a56443d6298ecf7435f618f0b27654f0c849e F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5 -F tool/GetTclKit.bat f94784e3bdc2f50c539266f5467cbf1f27612cb3 +F tool/GetTclKit.bat 6afa640edc7810725aec61c3076ac617c4aaf0b7 F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91 F tool/addopcodes.tcl f2e9aba2c2c718624fa14059198456d9e519b925 -F tool/build-all-msvc.bat 018c1b273458a90c8ba633c6f0c5654cfcb138bf x +F tool/build-all-msvc.bat c12328d06c45fec8baada5949e3d5af54bf8c887 x F tool/build-shell.sh 950f47c6174f1eea171319438b93ba67ff5bf367 F tool/cg_anno.tcl f95b0006c52cf7f0496b506343415b6ee3cdcdd3 x F tool/checkSpacing.c 810e51703529a204fc4e1eb060e9ab663e3c06d2 @@ -1479,22 +1502,24 @@ F tool/fuzzershell.c dbf6c26eef936ec78cb0707570de3a4308b2507e F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce +F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f F tool/lemon.c 5ccba178a8e8a4b21e1c9232944d23973da38ad7 -F tool/lempar.c 320d630b44da693407684c64d9fa91a163419dac +F tool/lempar.c db1bdb4821f2d8fbd76e577cf3ab18642c8d08d1 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca +F tool/max-limits.c cbb635fbb37ae4d05f240bfb5b5270bb63c54439 F tool/mkautoconfamal.sh e855df211ecbcc7131dee817110ff386cfb112f7 F tool/mkkeywordhash.c f5ecfd09b56b8badc644554ed38c86056152b893 -F tool/mkmsvcmin.tcl 2f12f7fa8858bbe61cf81820a2da96c79ed1ca8d +F tool/mkmsvcmin.tcl 95b37e202cbed873aa8ffdbb493b9db45927be2b F tool/mkopcodec.tcl d1b6362bd3aa80d5520d4d6f3765badf01f6c43c F tool/mkopcodeh.tcl a01d2c1d8a6205b03fc635adf3735b4c523befd3 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e -F tool/mkpragmatab.tcl ebb4bfcd2f8010e0a3934b6118db4b5f2f5edf5c +F tool/mkpragmatab.tcl 2ffe6d5fdc2d3381621d6c77978ba054466e757f F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl fef88397668ae83166735c41af99d79f56afaabb F tool/mksqlite3c.tcl 06b2e6a0f21cc0a5d70fbbd136b3e0a96470645e -F tool/mksqlite3h.tcl c006c4e5da57c649b24b689511dcd270dd7b0249 +F tool/mksqlite3h.tcl b9836752c3d08f9fab2dfc0017ca9fd5d90ac863 F tool/mksqlite3internalh.tcl eb994013e833359137eb53a55acdad0b5ae1049b F tool/mkvsix.tcl b9e0777a213c23156b6542842c238479e496ebf5 F tool/offsets.c fe4262fdfa378e8f5499a42136d17bf3b98f6091 @@ -1505,14 +1530,14 @@ F tool/replace.tcl 60f91e8dd06ab81f74d213ecbd9c9945f32ac048 F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5 F tool/run-speed-test.sh f95d19fd669b68c4c38b6b475242841d47c66076 -F tool/showdb.c c695a5d5c8110640e0d9fadf5e254da90c79c36e +F tool/showdb.c e6bc9dba233bf1b57ca0a525a2bba762db4e223de84990739db3f09c46151b1e F tool/showjournal.c 5bad7ae8784a43d2b270d953060423b8bd480818 F tool/showlocks.c 9920bcc64f58378ff1118caead34147201f48c68 F tool/showstat4.c b14159aa062f661b394ba37b6b7b94bfb8012ab9 F tool/showwal.c ec79959834f7b21f1e0a2aa52bb7c056d2203977 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe F tool/spaceanal.tcl ab7d9bf68062907282a64b3e12ccbfad47193c5a -F tool/speed-check.sh e6ca0695b047af64201ebe0ef452e423f55d78b1 +F tool/speed-check.sh 9630ba0468b609c52f48309243d4eb6e9c34deda F tool/speedtest.tcl 06c76698485ccf597b9e7dbb1ac70706eb873355 F tool/speedtest16.c ecb6542862151c3e6509bbc00509b234562ae81e F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff @@ -1552,7 +1577,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c8ca3e0a8d82dbb077551c2d952cb2043f78333b 391344d88a284f92c59b8d96a315b69292641de0 -R fb6607db49678e5588472158e22f722f -U dan -Z 69d374dd90bc72d9adab6847f15400aa +P 50fb1eb368f14e4ddce2da875601c227f9f937fd c5441d2df2526723f72610cc14dd243223663979e67ecdd76fe06fcd366f2b29 +R cb23130cd73bf8b87b96abf4a0e40c76 +U drh +Z e5649ef618566f41fb419552c48628f4 diff --git a/manifest.uuid b/manifest.uuid index aec8401384..e33520b19d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -50fb1eb368f14e4ddce2da875601c227f9f937fd \ No newline at end of file +fd5676fe7f55b3d4fa15ca119af7c064c6f9d053affdfd5e748785e300affbeb \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index c480a0c507..495cc954ac 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -290,6 +290,7 @@ struct Stat4Accum { Stat4Sample *aBest; /* Array of nCol best samples */ int iMin; /* Index in a[] of entry with minimum score */ int nSample; /* Current number of samples */ + int nMaxEqZero; /* Max leading 0 in anEq[] for any a[] entry */ int iGet; /* Index of current sample accessed by stat_get() */ Stat4Sample *a; /* Array of mxSample Stat4Sample objects */ sqlite3 *db; /* Database connection, for malloc() */ @@ -554,6 +555,13 @@ static void sampleInsert(Stat4Accum *p, Stat4Sample *pNew, int nEqZero){ assert( IsStat4 || nEqZero==0 ); #ifdef SQLITE_ENABLE_STAT4 + /* Stat4Accum.nMaxEqZero is set to the maximum number of leading 0 + ** values in the anEq[] array of any sample in Stat4Accum.a[]. In + ** other words, if nMaxEqZero is n, then it is guaranteed that there + ** are no samples with Stat4Sample.anEq[m]==0 for (m>=n). */ + if( nEqZero>p->nMaxEqZero ){ + p->nMaxEqZero = nEqZero; + } if( pNew->isPSample==0 ){ Stat4Sample *pUpgrade = 0; assert( pNew->anEq[pNew->iCol]>0 ); @@ -651,12 +659,22 @@ static void samplePushPrevious(Stat4Accum *p, int iChng){ } } - /* Update the anEq[] fields of any samples already collected. */ + /* Check that no sample contains an anEq[] entry with an index of + ** p->nMaxEqZero or greater set to zero. */ for(i=p->nSample-1; i>=0; i--){ int j; - for(j=iChng; jnCol; j++){ - if( p->a[i].anEq[j]==0 ) p->a[i].anEq[j] = p->current.anEq[j]; + for(j=p->nMaxEqZero; jnCol; j++) assert( p->a[i].anEq[j]>0 ); + } + + /* Update the anEq[] fields of any samples already collected. */ + if( iChngnMaxEqZero ){ + for(i=p->nSample-1; i>=0; i--){ + int j; + for(j=iChng; jnCol; j++){ + if( p->a[i].anEq[j]==0 ) p->a[i].anEq[j] = p->current.anEq[j]; + } } + p->nMaxEqZero = iChng; } #endif @@ -797,6 +815,12 @@ static const FuncDef statPushFuncdef = { ** The content to returned is determined by the parameter J ** which is one of the STAT_GET_xxxx values defined above. ** +** The stat_get(P,J) function is not available to generic SQL. It is +** inserted as part of a manually constructed bytecode program. (See +** the callStatGet() routine below.) It is guaranteed that the P +** parameter will always be a poiner to a Stat4Accum object, never a +** NULL. +** ** If neither STAT3 nor STAT4 are enabled, then J is always ** STAT_GET_STAT1 and is hence omitted and this routine becomes ** a one-parameter function, stat_get(P), that always returns the @@ -1180,7 +1204,7 @@ static void analyzeOneTable( regKey = sqlite3GetTempRange(pParse, pPk->nKeyCol); for(j=0; jnKeyCol; j++){ k = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[j]); - assert( k>=0 && knCol ); + assert( k>=0 && knColumn ); sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, regKey+j); VdbeComment((v, "%s", pTab->aCol[pPk->aiColumn[j]].zName)); } @@ -1364,27 +1388,14 @@ void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ if( i==1 ) continue; /* Do not analyze the TEMP database */ analyzeDatabase(pParse, i); } - }else if( pName2->n==0 ){ - /* Form 2: Analyze the database or table named */ - iDb = sqlite3FindDb(db, pName1); - if( iDb>=0 ){ - analyzeDatabase(pParse, iDb); - }else{ - z = sqlite3NameFromToken(db, pName1); - if( z ){ - if( (pIdx = sqlite3FindIndex(db, z, 0))!=0 ){ - analyzeTable(pParse, pIdx->pTable, pIdx); - }else if( (pTab = sqlite3LocateTable(pParse, 0, z, 0))!=0 ){ - analyzeTable(pParse, pTab, 0); - } - sqlite3DbFree(db, z); - } - } + }else if( pName2->n==0 && (iDb = sqlite3FindDb(db, pName1))>=0 ){ + /* Analyze the schema named as the argument */ + analyzeDatabase(pParse, iDb); }else{ - /* Form 3: Analyze the fully qualified table name */ + /* Form 3: Analyze the table or index named as an argument */ iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pTableName); if( iDb>=0 ){ - zDb = db->aDb[iDb].zDbSName; + zDb = pName2->n ? db->aDb[iDb].zDbSName : 0; z = sqlite3NameFromToken(db, pTableName); if( z ){ if( (pIdx = sqlite3FindIndex(db, z, zDb))!=0 ){ @@ -1394,10 +1405,11 @@ void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ } sqlite3DbFree(db, z); } - } + } + } + if( db->nSqlExec==0 && (v = sqlite3GetVdbe(pParse))!=0 ){ + sqlite3VdbeAddOp0(v, OP_Expire); } - v = sqlite3GetVdbe(pParse); - if( v ) sqlite3VdbeAddOp0(v, OP_Expire); } /* @@ -1526,7 +1538,11 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ #endif pIndex->bUnordered = 0; decodeIntArray((char*)z, nCol, aiRowEst, pIndex->aiRowLogEst, pIndex); - if( pIndex->pPartIdxWhere==0 ) pTable->nRowLogEst = pIndex->aiRowLogEst[0]; + pIndex->hasStat1 = 1; + if( pIndex->pPartIdxWhere==0 ){ + pTable->nRowLogEst = pIndex->aiRowLogEst[0]; + pTable->tabFlags |= TF_HasStat1; + } }else{ Index fakeIdx; fakeIdx.szIdxRow = pTable->szTabRow; @@ -1535,6 +1551,7 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ #endif decodeIntArray((char*)z, 1, 0, &pTable->nRowLogEst, &fakeIdx); pTable->szTabRow = fakeIdx.szIdxRow; + pTable->tabFlags |= TF_HasStat1; } return 0; @@ -1615,7 +1632,7 @@ static void initAvgEq(Index *pIdx){ } } - if( nDist100>nSum100 ){ + if( nDist100>nSum100 && sumEqaDb[iDb].pSchema; assert( iDb>=0 && iDbnDb ); assert( db->aDb[iDb].pBt!=0 ); /* Clear any prior statistics */ assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ + for(i=sqliteHashFirst(&pSchema->tblHash); i; i=sqliteHashNext(i)){ + Table *pTab = sqliteHashData(i); + pTab->tabFlags &= ~TF_HasStat1; + } + for(i=sqliteHashFirst(&pSchema->idxHash); i; i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); - pIdx->aiRowLogEst[0] = 0; + pIdx->hasStat1 = 0; #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 sqlite3DeleteIndexSamples(db, pIdx); pIdx->aSample = 0; @@ -1860,9 +1882,9 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ /* Set appropriate defaults on all indexes not in the sqlite_stat1 table */ assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ + for(i=sqliteHashFirst(&pSchema->idxHash); i; i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); - if( pIdx->aiRowLogEst[0]==0 ) sqlite3DefaultRowEst(pIdx); + if( !pIdx->hasStat1 ) sqlite3DefaultRowEst(pIdx); } /* Load the statistics from the sqlite_stat4 table. */ @@ -1872,7 +1894,7 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ rc = loadStat4(db, sInfo.zDatabase); db->lookaside.bDisable--; } - for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ + for(i=sqliteHashFirst(&pSchema->idxHash); i; i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); sqlite3_free(pIdx->aiRowEst); pIdx->aiRowEst = 0; diff --git a/src/btree.c b/src/btree.c index b05d911865..e73a4eb37f 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1520,17 +1520,18 @@ static void ptrmapPutOvflPtr(MemPage *pPage, u8 *pCell, int *pRC){ /* -** Defragment the page given. All Cells are moved to the -** end of the page and all free space is collected into one -** big FreeBlk that occurs in between the header and cell -** pointer array and the cell content area. +** Defragment the page given. This routine reorganizes cells within the +** page so that there are no free-blocks on the free-block list. +** +** Parameter nMaxFrag is the maximum amount of fragmented space that may be +** present in the page after this routine returns. ** ** EVIDENCE-OF: R-44582-60138 SQLite may from time to time reorganize a ** b-tree page so that there are no freeblocks or fragment bytes, all ** unused bytes are contained in the unallocated space region, and all ** cells are packed tightly at the end of the page. */ -static int defragmentPage(MemPage *pPage){ +static int defragmentPage(MemPage *pPage, int nMaxFrag){ int i; /* Loop counter */ int pc; /* Address of the i-th cell */ int hdr; /* Offset to the page header */ @@ -1545,7 +1546,6 @@ static int defragmentPage(MemPage *pPage){ int iCellFirst; /* First allowable cell index */ int iCellLast; /* Last possible cell index */ - assert( sqlite3PagerIswriteable(pPage->pDbPage) ); assert( pPage->pBt!=0 ); assert( pPage->pBt->usableSize <= SQLITE_MAX_PAGE_SIZE ); @@ -1557,9 +1557,56 @@ static int defragmentPage(MemPage *pPage){ cellOffset = pPage->cellOffset; nCell = pPage->nCell; assert( nCell==get2byte(&data[hdr+3]) ); - usableSize = pPage->pBt->usableSize; - cbrk = usableSize; iCellFirst = cellOffset + 2*nCell; + usableSize = pPage->pBt->usableSize; + + /* This block handles pages with two or fewer free blocks and nMaxFrag + ** or fewer fragmented bytes. In this case it is faster to move the + ** two (or one) blocks of cells using memmove() and add the required + ** offsets to each pointer in the cell-pointer array than it is to + ** reconstruct the entire page. */ + if( (int)data[hdr+7]<=nMaxFrag ){ + int iFree = get2byte(&data[hdr+1]); + if( iFree ){ + int iFree2 = get2byte(&data[iFree]); + + /* pageFindSlot() has already verified that free blocks are sorted + ** in order of offset within the page, and that no block extends + ** past the end of the page. Provided the two free slots do not + ** overlap, this guarantees that the memmove() calls below will not + ** overwrite the usableSize byte buffer, even if the database page + ** is corrupt. */ + assert( iFree2==0 || iFree2>iFree ); + assert( iFree+get2byte(&data[iFree+2]) <= usableSize ); + assert( iFree2==0 || iFree2+get2byte(&data[iFree2+2]) <= usableSize ); + + if( 0==iFree2 || (data[iFree2]==0 && data[iFree2+1]==0) ){ + u8 *pEnd = &data[cellOffset + nCell*2]; + u8 *pAddr; + int sz2 = 0; + int sz = get2byte(&data[iFree+2]); + int top = get2byte(&data[hdr+5]); + if( iFree2 ){ + if( iFree+sz>iFree2 ) return SQLITE_CORRUPT_BKPT; + sz2 = get2byte(&data[iFree2+2]); + assert( iFree+sz+sz2+iFree2-(iFree+sz) <= usableSize ); + memmove(&data[iFree+sz+sz2], &data[iFree+sz], iFree2-(iFree+sz)); + sz += sz2; + } + cbrk = top+sz; + assert( cbrk+(iFree-top) <= usableSize ); + memmove(&data[cbrk], &data[top], iFree-top); + for(pAddr=&data[cellOffset]; pAddrnFree ){ + return SQLITE_CORRUPT_BKPT; + } assert( cbrk>=iCellFirst ); put2byte(&data[hdr+5], cbrk); data[hdr+1] = 0; data[hdr+2] = 0; - data[hdr+7] = 0; memset(&data[iCellFirst], 0, cbrk-iCellFirst); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); - if( cbrk-iCellFirst!=pPage->nFree ){ - return SQLITE_CORRUPT_BKPT; - } return SQLITE_OK; } @@ -1740,10 +1789,10 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ testcase( gap+2+nByte==top ); if( gap+2+nByte>top ){ assert( pPage->nCell>0 || CORRUPT_DB ); - rc = defragmentPage(pPage); + rc = defragmentPage(pPage, MIN(4, pPage->nFree - (2+nByte))); if( rc ) return rc; top = get2byteNotZero(&data[hdr+5]); - assert( gap+nByte<=top ); + assert( gap+2+nByte<=top ); } @@ -3015,6 +3064,31 @@ int sqlite3BtreeGetAutoVacuum(Btree *p){ #endif } +/* +** If the user has not set the safety-level for this database connection +** using "PRAGMA synchronous", and if the safety-level is not already +** set to the value passed to this function as the second parameter, +** set it so. +*/ +#if SQLITE_DEFAULT_SYNCHRONOUS!=SQLITE_DEFAULT_WAL_SYNCHRONOUS +static void setDefaultSyncFlag(BtShared *pBt, u8 safety_level){ + sqlite3 *db; + Db *pDb; + if( (db=pBt->db)!=0 && (pDb=db->aDb)!=0 ){ + while( pDb->pBt==0 || pDb->pBt->pBt!=pBt ){ pDb++; } + if( pDb->bSyncSet==0 + && pDb->safety_level!=safety_level + && pDb!=&db->aDb[1] + ){ + pDb->safety_level = safety_level; + sqlite3PagerSetFlags(pBt->pPager, + pDb->safety_level | (db->flags & PAGER_FLAGS_MASK)); + } + } +} +#else +# define setDefaultSyncFlag(pBt,safety_level) +#endif /* ** Get a reference to pPage1 of the database file. This will @@ -3088,26 +3162,15 @@ static int lockBtree(BtShared *pBt){ if( rc!=SQLITE_OK ){ goto page1_init_failed; }else{ -#if SQLITE_DEFAULT_SYNCHRONOUS!=SQLITE_DEFAULT_WAL_SYNCHRONOUS - sqlite3 *db; - Db *pDb; - if( (db=pBt->db)!=0 && (pDb=db->aDb)!=0 ){ - while( pDb->pBt==0 || pDb->pBt->pBt!=pBt ){ pDb++; } - if( pDb->bSyncSet==0 - && pDb->safety_level==SQLITE_DEFAULT_SYNCHRONOUS+1 - ){ - pDb->safety_level = SQLITE_DEFAULT_WAL_SYNCHRONOUS+1; - sqlite3PagerSetFlags(pBt->pPager, - pDb->safety_level | (db->flags & PAGER_FLAGS_MASK)); - } - } -#endif + setDefaultSyncFlag(pBt, SQLITE_DEFAULT_WAL_SYNCHRONOUS+1); if( isOpen==0 ){ releasePage(pPage1); return SQLITE_OK; } } rc = SQLITE_NOTADB; + }else{ + setDefaultSyncFlag(pBt, SQLITE_DEFAULT_SYNCHRONOUS+1); } #endif @@ -3574,12 +3637,14 @@ static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){ if( eType==PTRMAP_OVERFLOW1 ){ CellInfo info; pPage->xParseCell(pPage, pCell, &info); - if( info.nLocalaData+pPage->maskPage - && iFrom==get4byte(pCell+info.nSize-4) - ){ - put4byte(pCell+info.nSize-4, iTo); - break; + if( info.nLocal pPage->aData+pPage->pBt->usableSize ){ + return SQLITE_CORRUPT_BKPT; + } + if( iFrom==get4byte(pCell+info.nSize-4) ){ + put4byte(pCell+info.nSize-4, iTo); + break; + } } }else{ if( get4byte(pCell)==iFrom ){ @@ -4432,7 +4497,12 @@ int sqlite3BtreeSavepoint(Btree *p, int op, int iSavepoint){ assert( iSavepoint>=0 || (iSavepoint==-1 && op==SAVEPOINT_ROLLBACK) ); sqlite3BtreeEnter(p); btreePtrmapEnd(pBt, op, iSavepoint); - rc = sqlite3PagerSavepoint(pBt->pPager, op, iSavepoint); + if( op==SAVEPOINT_ROLLBACK ){ + rc = saveAllCursors(pBt, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3PagerSavepoint(pBt->pPager, op, iSavepoint); + } if( rc==SQLITE_OK ){ if( iSavepoint<0 && (pBt->btsFlags & BTS_INITIALLY_EMPTY)!=0 ){ pBt->nPage = 0; @@ -4818,7 +4888,6 @@ static int copyPayload( ** ** 0: The operation is a read. Populate the overflow cache. ** 1: The operation is a write. Populate the overflow cache. -** 2: The operation is a read. Do not populate the overflow cache. ** ** A total of "amt" bytes are read or written beginning at "offset". ** Data is read to or from the buffer pBuf. @@ -4826,13 +4895,13 @@ static int copyPayload( ** The content being read or written might appear on the main page ** or be scattered out on multiple overflow pages. ** -** If the current cursor entry uses one or more overflow pages and the -** eOp argument is not 2, this function may allocate space for and lazily -** populates the overflow page-list cache array (BtCursor.aOverflow). +** If the current cursor entry uses one or more overflow pages +** this function may allocate space for and lazily populate +** the overflow page-list cache array (BtCursor.aOverflow). ** Subsequent calls use this cache to make seeking to the supplied offset ** more efficient. ** -** Once an overflow page-list cache has been allocated, it may be +** Once an overflow page-list cache has been allocated, it must be ** invalidated if some other cursor writes to the same table, or if ** the cursor is moved to a different row. Additionally, in auto-vacuum ** mode, the following events may invalidate an overflow page-list cache. @@ -4854,21 +4923,17 @@ static int accessPayload( MemPage *pPage = pCur->apPage[pCur->iPage]; /* Btree page of current entry */ BtShared *pBt = pCur->pBt; /* Btree this cursor belongs to */ #ifdef SQLITE_DIRECT_OVERFLOW_READ - unsigned char * const pBufStart = pBuf; - int bEnd; /* True if reading to end of data */ + unsigned char * const pBufStart = pBuf; /* Start of original out buffer */ #endif assert( pPage ); + assert( eOp==0 || eOp==1 ); assert( pCur->eState==CURSOR_VALID ); assert( pCur->aiIdx[pCur->iPage]nCell ); assert( cursorHoldsMutex(pCur) ); - assert( eOp!=2 || offset==0 ); /* Always start from beginning for eOp==2 */ getCellInfo(pCur); aPayload = pCur->info.pPayload; -#ifdef SQLITE_DIRECT_OVERFLOW_READ - bEnd = offset+amt==pCur->info.nPayload; -#endif assert( offset+amt <= pCur->info.nPayload ); assert( aPayload > pPage->aData ); @@ -4887,7 +4952,7 @@ static int accessPayload( if( a+offset>pCur->info.nLocal ){ a = pCur->info.nLocal - offset; } - rc = copyPayload(&aPayload[offset], pBuf, a, (eOp & 0x01), pPage->pDbPage); + rc = copyPayload(&aPayload[offset], pBuf, a, eOp, pPage->pDbPage); offset = 0; pBuf += a; amt -= a; @@ -4903,53 +4968,46 @@ static int accessPayload( nextPage = get4byte(&aPayload[pCur->info.nLocal]); /* If the BtCursor.aOverflow[] has not been allocated, allocate it now. - ** Except, do not allocate aOverflow[] for eOp==2. ** ** The aOverflow[] array is sized at one entry for each overflow page ** in the overflow chain. The page number of the first overflow page is ** stored in aOverflow[0], etc. A value of 0 in the aOverflow[] array ** means "not yet known" (the cache is lazily populated). */ - if( eOp!=2 && (pCur->curFlags & BTCF_ValidOvfl)==0 ){ + if( (pCur->curFlags & BTCF_ValidOvfl)==0 ){ int nOvfl = (pCur->info.nPayload-pCur->info.nLocal+ovflSize-1)/ovflSize; if( nOvfl>pCur->nOvflAlloc ){ Pgno *aNew = (Pgno*)sqlite3Realloc( pCur->aOverflow, nOvfl*2*sizeof(Pgno) ); if( aNew==0 ){ - rc = SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM_BKPT; }else{ pCur->nOvflAlloc = nOvfl*2; pCur->aOverflow = aNew; } } - if( rc==SQLITE_OK ){ - memset(pCur->aOverflow, 0, nOvfl*sizeof(Pgno)); - pCur->curFlags |= BTCF_ValidOvfl; + memset(pCur->aOverflow, 0, nOvfl*sizeof(Pgno)); + pCur->curFlags |= BTCF_ValidOvfl; + }else{ + /* If the overflow page-list cache has been allocated and the + ** entry for the first required overflow page is valid, skip + ** directly to it. + */ + if( pCur->aOverflow[offset/ovflSize] ){ + iIdx = (offset/ovflSize); + nextPage = pCur->aOverflow[iIdx]; + offset = (offset%ovflSize); } } - /* If the overflow page-list cache has been allocated and the - ** entry for the first required overflow page is valid, skip - ** directly to it. - */ - if( (pCur->curFlags & BTCF_ValidOvfl)!=0 - && pCur->aOverflow[offset/ovflSize] - ){ - iIdx = (offset/ovflSize); - nextPage = pCur->aOverflow[iIdx]; - offset = (offset%ovflSize); - } - - for( ; rc==SQLITE_OK && amt>0 && nextPage; iIdx++){ - + assert( rc==SQLITE_OK && amt>0 ); + while( nextPage ){ /* If required, populate the overflow page-list cache. */ - if( (pCur->curFlags & BTCF_ValidOvfl)!=0 ){ - assert( pCur->aOverflow[iIdx]==0 - || pCur->aOverflow[iIdx]==nextPage - || CORRUPT_DB ); - pCur->aOverflow[iIdx] = nextPage; - } + assert( pCur->aOverflow[iIdx]==0 + || pCur->aOverflow[iIdx]==nextPage + || CORRUPT_DB ); + pCur->aOverflow[iIdx] = nextPage; if( offset>=ovflSize ){ /* The only reason to read this page is to obtain the page @@ -4957,11 +5015,7 @@ static int accessPayload( ** data is not required. So first try to lookup the overflow ** page-list cache, if any, then fall back to the getOverflowPage() ** function. - ** - ** Note that the aOverflow[] array must be allocated because eOp!=2 - ** here. If eOp==2, then offset==0 and this branch is never taken. */ - assert( eOp!=2 ); assert( pCur->curFlags & BTCF_ValidOvfl ); assert( pCur->pBtree->db==pBt->db ); if( pCur->aOverflow[iIdx+1] ){ @@ -4975,7 +5029,7 @@ static int accessPayload( ** range of data that is being read (eOp==0) or written (eOp!=0). */ #ifdef SQLITE_DIRECT_OVERFLOW_READ - sqlite3_file *fd; + sqlite3_file *fd; /* File from which to do direct overflow read */ #endif int a = amt; if( a + offset > ovflSize ){ @@ -4987,27 +5041,25 @@ static int accessPayload( ** ** 1) this is a read operation, and ** 2) data is required from the start of this overflow page, and - ** 3) the database is file-backed, and - ** 4) there is no open write-transaction, and - ** 5) the database is not a WAL database, - ** 6) all data from the page is being read. - ** 7) at least 4 bytes have already been read into the output buffer + ** 3) there is no open write-transaction, and + ** 4) the database is file-backed, and + ** 5) the page is not in the WAL file + ** 6) at least 4 bytes have already been read into the output buffer ** ** then data can be read directly from the database file into the ** output buffer, bypassing the page-cache altogether. This speeds ** up loading large records that span many overflow pages. */ - if( (eOp&0x01)==0 /* (1) */ + if( eOp==0 /* (1) */ && offset==0 /* (2) */ - && (bEnd || a==ovflSize) /* (6) */ - && pBt->inTransaction==TRANS_READ /* (4) */ - && (fd = sqlite3PagerFile(pBt->pPager))->pMethods /* (3) */ - && 0==sqlite3PagerUseWal(pBt->pPager) /* (5) */ - && &pBuf[-4]>=pBufStart /* (7) */ + && pBt->inTransaction==TRANS_READ /* (3) */ + && (fd = sqlite3PagerFile(pBt->pPager))->pMethods /* (4) */ + && 0==sqlite3PagerUseWal(pBt->pPager, nextPage) /* (5) */ + && &pBuf[-4]>=pBufStart /* (6) */ ){ u8 aSave[4]; u8 *aWrite = &pBuf[-4]; - assert( aWrite>=pBufStart ); /* hence (7) */ + assert( aWrite>=pBufStart ); /* due to (6) */ memcpy(aSave, aWrite, 4); rc = sqlite3OsRead(fd, aWrite, a+4, (i64)pBt->pageSize*(nextPage-1)); nextPage = get4byte(aWrite); @@ -5018,24 +5070,27 @@ static int accessPayload( { DbPage *pDbPage; rc = sqlite3PagerGet(pBt->pPager, nextPage, &pDbPage, - ((eOp&0x01)==0 ? PAGER_GET_READONLY : 0) + (eOp==0 ? PAGER_GET_READONLY : 0) ); if( rc==SQLITE_OK ){ aPayload = sqlite3PagerGetData(pDbPage); nextPage = get4byte(aPayload); - rc = copyPayload(&aPayload[offset+4], pBuf, a, (eOp&0x01), pDbPage); + rc = copyPayload(&aPayload[offset+4], pBuf, a, eOp, pDbPage); sqlite3PagerUnref(pDbPage); offset = 0; } } amt -= a; + if( amt==0 ) return rc; pBuf += a; } + if( rc ) break; + iIdx++; } } if( rc==SQLITE_OK && amt>0 ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_BKPT; /* Overflow chain ends prematurely */ } return rc; } @@ -5064,21 +5119,34 @@ int sqlite3BtreePayload(BtCursor *pCur, u32 offset, u32 amt, void *pBuf){ assert( pCur->aiIdx[pCur->iPage]apPage[pCur->iPage]->nCell ); return accessPayload(pCur, offset, amt, (unsigned char*)pBuf, 0); } + +/* +** This variant of sqlite3BtreePayload() works even if the cursor has not +** in the CURSOR_VALID state. It is only used by the sqlite3_blob_read() +** interface. +*/ #ifndef SQLITE_OMIT_INCRBLOB -int sqlite3BtreePayloadChecked(BtCursor *pCur, u32 offset, u32 amt, void *pBuf){ +static SQLITE_NOINLINE int accessPayloadChecked( + BtCursor *pCur, + u32 offset, + u32 amt, + void *pBuf +){ int rc; if ( pCur->eState==CURSOR_INVALID ){ return SQLITE_ABORT; } assert( cursorOwnsBtShared(pCur) ); - rc = restoreCursorPosition(pCur); - if( rc==SQLITE_OK ){ - assert( pCur->eState==CURSOR_VALID ); - assert( pCur->iPage>=0 && pCur->apPage[pCur->iPage] ); - assert( pCur->aiIdx[pCur->iPage]apPage[pCur->iPage]->nCell ); - rc = accessPayload(pCur, offset, amt, pBuf, 0); + rc = btreeRestoreCursorPosition(pCur); + return rc ? rc : accessPayload(pCur, offset, amt, pBuf, 0); +} +int sqlite3BtreePayloadChecked(BtCursor *pCur, u32 offset, u32 amt, void *pBuf){ + if( pCur->eState==CURSOR_VALID ){ + assert( cursorOwnsBtShared(pCur) ); + return accessPayload(pCur, offset, amt, pBuf, 0); + }else{ + return accessPayloadChecked(pCur, offset, amt, pBuf); } - return rc; } #endif /* SQLITE_OMIT_INCRBLOB */ @@ -5167,7 +5235,7 @@ static int moveToChild(BtCursor *pCur, u32 newPgno){ pCur, pCur->curPagerFlags); } -#if SQLITE_DEBUG +#ifdef SQLITE_DEBUG /* ** Page pParent is an internal (non-leaf) tree page. This function ** asserts that page number iChild is the left-child if the iIdx'th @@ -5484,9 +5552,26 @@ int sqlite3BtreeMovetoUnpacked( *pRes = 0; return SQLITE_OK; } - if( (pCur->curFlags & BTCF_AtLast)!=0 && pCur->info.nKeyinfo.nKeycurFlags & BTCF_AtLast)!=0 ){ + *pRes = -1; + return SQLITE_OK; + } + /* If the requested key is one more than the previous key, then + ** try to get there using sqlite3BtreeNext() rather than a full + ** binary search. This is an optimization only. The correct answer + ** is still obtained without this ase, only a little more slowely */ + if( pCur->info.nKey+1==intKey && !pCur->skipNext ){ + *pRes = 0; + rc = sqlite3BtreeNext(pCur, pRes); + if( rc ) return rc; + if( *pRes==0 ){ + getCellInfo(pCur); + if( pCur->info.nKey==intKey ){ + return SQLITE_OK; + } + } + } } } @@ -5622,7 +5707,8 @@ int sqlite3BtreeMovetoUnpacked( goto moveto_finish; } pCur->aiIdx[pCur->iPage] = (u16)idx; - rc = accessPayload(pCur, 0, nCell, (unsigned char*)pCellKey, 2); + rc = accessPayload(pCur, 0, nCell, (unsigned char*)pCellKey, 0); + pCur->curFlags &= ~BTCF_ValidOvfl; if( rc ){ sqlite3_free(pCellKey); goto moveto_finish; @@ -5692,6 +5778,30 @@ int sqlite3BtreeEof(BtCursor *pCur){ return (CURSOR_VALID!=pCur->eState); } +/* +** Return an estimate for the number of rows in the table that pCur is +** pointing to. Return a negative number if no estimate is currently +** available. +*/ +i64 sqlite3BtreeRowCountEst(BtCursor *pCur){ + i64 n; + u8 i; + + assert( cursorOwnsBtShared(pCur) ); + assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); + + /* Currently this interface is only called by the OP_IfSmaller + ** opcode, and it that case the cursor will always be valid and + ** will always point to a leaf node. */ + if( NEVER(pCur->eState!=CURSOR_VALID) ) return -1; + if( NEVER(pCur->apPage[pCur->iPage]->leaf==0) ) return -1; + + for(n=1, i=0; i<=pCur->iPage; i++){ + n *= pCur->apPage[i]->nCell; + } + return n; +} + /* ** Advance the cursor to the next entry in the database. If ** successful then set *pRes=0. If the cursor @@ -6546,7 +6656,7 @@ static int fillInCell( ** Use a call to btreeParseCellPtr() to verify that the values above ** were computed correctly. */ -#if SQLITE_DEBUG +#ifdef SQLITE_DEBUG { CellInfo info; pPage->xParseCell(pPage, pCell, &info); @@ -7673,7 +7783,6 @@ static int balance_nonroot( for(i=0; inFree; - if( szNew[i]<0 ){ rc = SQLITE_CORRUPT_BKPT; goto balance_cleanup; } for(j=0; jnOverflow; j++){ szNew[i] += 2 + p->xCellSize(p, p->apOvfl[j]); } @@ -8071,7 +8180,7 @@ static int balance_nonroot( ** free space needs to be up front. */ assert( nNew==1 || CORRUPT_DB ); - rc = defragmentPage(apNew[0]); + rc = defragmentPage(apNew[0], -1); testcase( rc!=SQLITE_OK ); assert( apNew[0]->nFree == (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2) @@ -8352,7 +8461,7 @@ static int balance(BtCursor *pCur){ int sqlite3BtreeInsert( BtCursor *pCur, /* Insert data into the table of this cursor */ const BtreePayload *pX, /* Content of the row to be inserted */ - int appendBias, /* True if this is likely an append */ + int flags, /* True if this is likely an append */ int seekResult /* Result of prior MovetoUnpacked() call */ ){ int rc; @@ -8365,6 +8474,8 @@ int sqlite3BtreeInsert( unsigned char *oldCell; unsigned char *newCell = 0; + assert( (flags & (BTREE_SAVEPOSITION|BTREE_APPEND))==flags ); + if( pCur->eState==CURSOR_FAULT ){ assert( pCur->skipNext!=SQLITE_OK ); return pCur->skipNext; @@ -8405,6 +8516,11 @@ int sqlite3BtreeInsert( ** cursors open on the row being replaced */ invalidateIncrblobCursors(p, pX->nKey, 0); + /* If BTREE_SAVEPOSITION is set, the cursor must already be pointing + ** to a row with the same key as the new entry being inserted. */ + assert( (flags & BTREE_SAVEPOSITION)==0 || + ((pCur->curFlags&BTCF_ValidNKey)!=0 && pX->nKey==pCur->info.nKey) ); + /* If the cursor is currently on the last row and we are appending a ** new row onto the end, set the "loc" to avoid an unnecessary ** btreeMoveto() call */ @@ -8414,10 +8530,10 @@ int sqlite3BtreeInsert( && pCur->info.nKey==pX->nKey-1 ){ loc = -1; }else if( loc==0 ){ - rc = sqlite3BtreeMovetoUnpacked(pCur, 0, pX->nKey, appendBias, &loc); + rc = sqlite3BtreeMovetoUnpacked(pCur, 0, pX->nKey, flags!=0, &loc); if( rc ) return rc; } - }else if( loc==0 ){ + }else if( loc==0 && (flags & BTREE_SAVEPOSITION)==0 ){ if( pX->nMem ){ UnpackedRecord r; r.pKeyInfo = pCur->pKeyInfo; @@ -8428,9 +8544,9 @@ int sqlite3BtreeInsert( r.r1 = 0; r.r2 = 0; r.eqSeen = 0; - rc = sqlite3BtreeMovetoUnpacked(pCur, &r, 0, appendBias, &loc); + rc = sqlite3BtreeMovetoUnpacked(pCur, &r, 0, flags!=0, &loc); }else{ - rc = btreeMoveto(pCur, pX->pKey, pX->nKey, appendBias, &loc); + rc = btreeMoveto(pCur, pX->pKey, pX->nKey, flags!=0, &loc); } if( rc ) return rc; } @@ -8518,6 +8634,20 @@ int sqlite3BtreeInsert( ** from trying to save the current position of the cursor. */ pCur->apPage[pCur->iPage]->nOverflow = 0; pCur->eState = CURSOR_INVALID; + if( (flags & BTREE_SAVEPOSITION) && rc==SQLITE_OK ){ + rc = moveToRoot(pCur); + if( pCur->pKeyInfo ){ + assert( pCur->pKey==0 ); + pCur->pKey = sqlite3Malloc( pX->nKey ); + if( pCur->pKey==0 ){ + rc = SQLITE_NOMEM; + }else{ + memcpy(pCur->pKey, pX->pKey, pX->nKey); + } + } + pCur->eState = CURSOR_REQUIRESEEK; + pCur->nKey = pX->nKey; + } } assert( pCur->apPage[pCur->iPage]->nOverflow==0 ); diff --git a/src/btree.h b/src/btree.h index a2bd57f1ac..c26fea1f21 100644 --- a/src/btree.h +++ b/src/btree.h @@ -249,9 +249,10 @@ int sqlite3BtreeCursorHasMoved(BtCursor*); int sqlite3BtreeCursorRestore(BtCursor*, int*); int sqlite3BtreeDelete(BtCursor*, u8 flags); -/* Allowed flags for the 2nd argument to sqlite3BtreeDelete() */ +/* Allowed flags for sqlite3BtreeDelete() and sqlite3BtreeInsert() */ #define BTREE_SAVEPOSITION 0x02 /* Leave cursor pointing at NEXT or PREV */ #define BTREE_AUXDELETE 0x04 /* not the primary delete operation */ +#define BTREE_APPEND 0x08 /* Insert is likely an append */ /* An instance of the BtreePayload object describes the content of a single ** entry in either an index or table btree. @@ -282,7 +283,7 @@ struct BtreePayload { }; int sqlite3BtreeInsert(BtCursor*, const BtreePayload *pPayload, - int bias, int seekResult); + int flags, int seekResult); int sqlite3BtreeFirst(BtCursor*, int *pRes); int sqlite3BtreeLast(BtCursor*, int *pRes); int sqlite3BtreeNext(BtCursor*, int *pRes); @@ -295,6 +296,7 @@ u32 sqlite3BtreePayloadSize(BtCursor*); char *sqlite3BtreeIntegrityCheck(Btree*, int *aRoot, int nRoot, int, int*); struct Pager *sqlite3BtreePager(Btree*); +i64 sqlite3BtreeRowCountEst(BtCursor*); #ifndef SQLITE_OMIT_INCRBLOB int sqlite3BtreePayloadChecked(BtCursor*, u32 offset, u32 amt, void*); diff --git a/src/btreeInt.h b/src/btreeInt.h index 6b2d96e9ca..3822eaf185 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -703,11 +703,9 @@ struct IntegrityCk { */ #if SQLITE_BYTEORDER==4321 # define get2byteAligned(x) (*(u16*)(x)) -#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ - && GCC_VERSION>=4008000 +#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4008000 # define get2byteAligned(x) __builtin_bswap16(*(u16*)(x)) -#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ - && defined(_MSC_VER) && _MSC_VER>=1300 +#elif SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 # define get2byteAligned(x) _byteswap_ushort(*(u16*)(x)) #else # define get2byteAligned(x) ((x)[0]<<8 | (x)[1]) diff --git a/src/build.c b/src/build.c index ea282ee8ea..790e7a6651 100644 --- a/src/build.c +++ b/src/build.c @@ -1117,6 +1117,7 @@ void sqlite3AddNotNull(Parse *pParse, int onError){ p = pParse->pNewTable; if( p==0 || NEVER(p->nCol<1) ) return; p->aCol[p->nCol-1].notNull = (u8)onError; + p->tabFlags |= TF_HasNotNull; } /* @@ -3456,6 +3457,9 @@ void sqlite3DefaultRowEst(Index *pIdx){ int nCopy = MIN(ArraySize(aVal), pIdx->nKeyCol); int i; + /* Indexes with default row estimates should not have stat1 data */ + assert( !pIdx->hasStat1 ); + /* Set the first entry (number of rows in the index) to the estimated ** number of rows in the table, or half the number of rows in the table ** for a partial index. But do not let the estimate drop below 10. */ diff --git a/src/ctime.c b/src/ctime.c index 969bbc73d2..20e94edba5 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -57,7 +57,7 @@ static const char * const azCompileOpt[] = { #if SQLITE_COVERAGE_TEST "COVERAGE_TEST", #endif -#if SQLITE_DEBUG +#ifdef SQLITE_DEBUG "DEBUG", #endif #if SQLITE_DEFAULT_LOCKING_MODE @@ -66,6 +66,12 @@ static const char * const azCompileOpt[] = { #if defined(SQLITE_DEFAULT_MMAP_SIZE) && !defined(SQLITE_DEFAULT_MMAP_SIZE_xc) "DEFAULT_MMAP_SIZE=" CTIMEOPT_VAL(SQLITE_DEFAULT_MMAP_SIZE), #endif +#if SQLITE_DEFAULT_SYNCHRONOUS + "DEFAULT_SYNCHRONOUS=" CTIMEOPT_VAL(SQLITE_DEFAULT_SYNCHRONOUS), +#endif +#if SQLITE_DEFAULT_WAL_SYNCHRONOUS + "DEFAULT_WAL_SYNCHRONOUS=" CTIMEOPT_VAL(SQLITE_DEFAULT_WAL_SYNCHRONOUS), +#endif #if SQLITE_DIRECT_OVERFLOW_READ "DIRECT_OVERFLOW_READ", #endif diff --git a/src/date.c b/src/date.c index a08248ce2f..b49d506bc4 100644 --- a/src/date.c +++ b/src/date.c @@ -743,18 +743,19 @@ static int parseModifier( ** or month or year. */ if( sqlite3_strnicmp(z, "start of ", 9)!=0 ) break; + if( !p->validJD && !p->validYMD && !p->validHMS ) break; z += 9; computeYMD(p); p->validHMS = 1; p->h = p->m = 0; p->s = 0.0; + p->rawS = 0; p->validTZ = 0; p->validJD = 0; if( sqlite3_stricmp(z,"month")==0 ){ p->D = 1; rc = 0; }else if( sqlite3_stricmp(z,"year")==0 ){ - computeYMD(p); p->M = 1; p->D = 1; rc = 0; diff --git a/src/delete.c b/src/delete.c index 9ec8ed59e3..0683f9b9dd 100644 --- a/src/delete.c +++ b/src/delete.c @@ -519,12 +519,8 @@ void sqlite3DeleteFrom( #endif { int count = (pParse->nested==0); /* True to count changes */ - int iIdxNoSeek = -1; - if( bComplex==0 && aiCurOnePass[1]!=iDataCur ){ - iIdxNoSeek = aiCurOnePass[1]; - } sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, - iKey, nKey, count, OE_Default, eOnePass, iIdxNoSeek); + iKey, nKey, count, OE_Default, eOnePass, aiCurOnePass[1]); } /* End of the loop over all rowids/primary-keys. */ @@ -604,15 +600,17 @@ delete_from_cleanup: ** ** If eMode is ONEPASS_MULTI, then this call is being made as part ** of a ONEPASS delete that affects multiple rows. In this case, if -** iIdxNoSeek is a valid cursor number (>=0), then its position should -** be preserved following the delete operation. Or, if iIdxNoSeek is not -** a valid cursor number, the position of iDataCur should be preserved -** instead. +** iIdxNoSeek is a valid cursor number (>=0) and is not the same as +** iDataCur, then its position should be preserved following the delete +** operation. Or, if iIdxNoSeek is not a valid cursor number, the +** position of iDataCur should be preserved instead. ** ** iIdxNoSeek: -** If iIdxNoSeek is a valid cursor number (>=0), then it identifies an -** index cursor (from within array of cursors starting at iIdxCur) that -** already points to the index entry to be deleted. +** If iIdxNoSeek is a valid cursor number (>=0) not equal to iDataCur, +** then it identifies an index cursor (from within array of cursors +** starting at iIdxCur) that already points to the index entry to be deleted. +** Except, this optimization is disabled if there are BEFORE triggers since +** the trigger body might have moved the cursor. */ void sqlite3GenerateRowDelete( Parse *pParse, /* Parsing context */ @@ -683,13 +681,18 @@ void sqlite3GenerateRowDelete( /* If any BEFORE triggers were coded, then seek the cursor to the ** row to be deleted again. It may be that the BEFORE triggers moved - ** the cursor or of already deleted the row that the cursor was + ** the cursor or already deleted the row that the cursor was ** pointing to. + ** + ** Also disable the iIdxNoSeek optimization since the BEFORE trigger + ** may have moved that cursor. */ if( addrStart=0 ); + iIdxNoSeek = -1; } /* Do FK processing. This call checks that any FK constraints that @@ -712,11 +715,13 @@ void sqlite3GenerateRowDelete( u8 p5 = 0; sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur,0,iIdxNoSeek); sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, (count?OPFLAG_NCHANGE:0)); - sqlite3VdbeAppendP4(v, (char*)pTab, P4_TABLE); + if( pParse->nested==0 ){ + sqlite3VdbeAppendP4(v, (char*)pTab, P4_TABLE); + } if( eMode!=ONEPASS_OFF ){ sqlite3VdbeChangeP5(v, OPFLAG_AUXDELETE); } - if( iIdxNoSeek>=0 ){ + if( iIdxNoSeek>=0 && iIdxNoSeek!=iDataCur ){ sqlite3VdbeAddOp1(v, OP_Delete, iIdxNoSeek); } if( eMode==ONEPASS_MULTI ) p5 |= OPFLAG_SAVEPOSITION; @@ -870,6 +875,10 @@ int sqlite3GenerateIndexKey( } if( regOut ){ sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol, regOut); + if( pIdx->pTable->pSelect ){ + const char *zAff = sqlite3IndexAffinityStr(pParse->db, pIdx); + sqlite3VdbeChangeP4(v, -1, zAff, P4_TRANSIENT); + } } sqlite3ReleaseTempRange(pParse, regBase, nCol); return regBase; diff --git a/src/expr.c b/src/expr.c index 154d078ff7..ce948be69e 100644 --- a/src/expr.c +++ b/src/expr.c @@ -231,7 +231,7 @@ static char comparisonAffinity(Expr *pExpr){ aff = sqlite3CompareAffinity(pExpr->pRight, aff); }else if( ExprHasProperty(pExpr, EP_xIsSelect) ){ aff = sqlite3CompareAffinity(pExpr->x.pSelect->pEList->a[0].pExpr, aff); - }else if( NEVER(aff==0) ){ + }else if( aff==0 ){ aff = SQLITE_AFF_BLOB; } return aff; @@ -967,8 +967,13 @@ void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr, u32 n){ /* Wildcard of the form "?nnn". Convert "nnn" to an integer and ** use it as the variable number */ i64 i; - int bOk = 0==sqlite3Atoi64(&z[1], &i, n-1, SQLITE_UTF8); - x = (ynVar)i; + int bOk; + if( n==2 ){ /*OPTIMIZATION-IF-TRUE*/ + i = z[1]-'0'; /* The common case of ?N for a single digit N */ + bOk = 1; + }else{ + bOk = 0==sqlite3Atoi64(&z[1], &i, n-1, SQLITE_UTF8); + } testcase( i==0 ); testcase( i==1 ); testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]-1 ); @@ -978,6 +983,7 @@ void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr, u32 n){ db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]); return; } + x = (ynVar)i; if( x>pParse->nVar ){ pParse->nVar = (int)x; doAdd = 1; @@ -1412,33 +1418,41 @@ IdList *sqlite3IdListDup(sqlite3 *db, IdList *p){ } return pNew; } -Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){ - Select *pNew, *pPrior; +Select *sqlite3SelectDup(sqlite3 *db, Select *pDup, int flags){ + Select *pRet = 0; + Select *pNext = 0; + Select **pp = &pRet; + Select *p; + assert( db!=0 ); - if( p==0 ) return 0; - pNew = sqlite3DbMallocRawNN(db, sizeof(*p) ); - if( pNew==0 ) return 0; - pNew->pEList = sqlite3ExprListDup(db, p->pEList, flags); - pNew->pSrc = sqlite3SrcListDup(db, p->pSrc, flags); - pNew->pWhere = sqlite3ExprDup(db, p->pWhere, flags); - pNew->pGroupBy = sqlite3ExprListDup(db, p->pGroupBy, flags); - pNew->pHaving = sqlite3ExprDup(db, p->pHaving, flags); - pNew->pOrderBy = sqlite3ExprListDup(db, p->pOrderBy, flags); - pNew->op = p->op; - pNew->pPrior = pPrior = sqlite3SelectDup(db, p->pPrior, flags); - if( pPrior ) pPrior->pNext = pNew; - pNew->pNext = 0; - pNew->pLimit = sqlite3ExprDup(db, p->pLimit, flags); - pNew->pOffset = sqlite3ExprDup(db, p->pOffset, flags); - pNew->iLimit = 0; - pNew->iOffset = 0; - pNew->selFlags = p->selFlags & ~SF_UsesEphemeral; - pNew->addrOpenEphm[0] = -1; - pNew->addrOpenEphm[1] = -1; - pNew->nSelectRow = p->nSelectRow; - pNew->pWith = withDup(db, p->pWith); - sqlite3SelectSetName(pNew, p->zSelName); - return pNew; + for(p=pDup; p; p=p->pPrior){ + Select *pNew = sqlite3DbMallocRawNN(db, sizeof(*p) ); + if( pNew==0 ) break; + pNew->pEList = sqlite3ExprListDup(db, p->pEList, flags); + pNew->pSrc = sqlite3SrcListDup(db, p->pSrc, flags); + pNew->pWhere = sqlite3ExprDup(db, p->pWhere, flags); + pNew->pGroupBy = sqlite3ExprListDup(db, p->pGroupBy, flags); + pNew->pHaving = sqlite3ExprDup(db, p->pHaving, flags); + pNew->pOrderBy = sqlite3ExprListDup(db, p->pOrderBy, flags); + pNew->op = p->op; + pNew->pNext = pNext; + pNew->pPrior = 0; + pNew->pLimit = sqlite3ExprDup(db, p->pLimit, flags); + pNew->pOffset = sqlite3ExprDup(db, p->pOffset, flags); + pNew->iLimit = 0; + pNew->iOffset = 0; + pNew->selFlags = p->selFlags & ~SF_UsesEphemeral; + pNew->addrOpenEphm[0] = -1; + pNew->addrOpenEphm[1] = -1; + pNew->nSelectRow = p->nSelectRow; + pNew->pWith = withDup(db, p->pWith); + sqlite3SelectSetName(pNew, p->zSelName); + *pp = pNew; + pp = &pNew->pPrior; + pNext = pNew; + } + + return pRet; } #else Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){ @@ -1846,6 +1860,7 @@ int sqlite3ExprContainsSubquery(Expr *p){ */ int sqlite3ExprIsInteger(Expr *p, int *pValue){ int rc = 0; + if( p==0 ) return 0; /* Can only happen following on OOM */ /* If an expression is an integer literal that fits in a signed 32-bit ** integer, then the EP_IntValue flag will have already been set */ @@ -2353,7 +2368,7 @@ static char *exprINAffinity(Parse *pParse, Expr *pExpr){ char *zRet; assert( pExpr->op==TK_IN ); - zRet = sqlite3DbMallocZero(pParse->db, nVal+1); + zRet = sqlite3DbMallocRaw(pParse->db, nVal+1); if( zRet ){ int i; for(i=0; iiTable); dest.zAffSdst = exprINAffinity(pParse, pExpr); - assert( (pExpr->iTable&0x0000FFFF)==pExpr->iTable ); pSelect->iLimit = 0; testcase( pSelect->selFlags & SF_Distinct ); testcase( pKeyInfo==0 ); /* Caused by OOM in sqlite3KeyInfoAlloc() */ @@ -3258,7 +3272,7 @@ void sqlite3ExprCodeGetColumnToReg( void sqlite3ExprCacheClear(Parse *pParse){ int i; -#if SQLITE_DEBUG +#ifdef SQLITE_DEBUG if( pParse->db->flags & SQLITE_VdbeAddopTrace ){ printf("CLEAR\n"); } @@ -4666,6 +4680,17 @@ int sqlite3ExprListCompare(ExprList *pA, ExprList *pB, int iTab){ return 0; } +/* +** Like sqlite3ExprCompare() except COLLATE operators at the top-level +** are ignored. +*/ +int sqlite3ExprCompareSkip(Expr *pA, Expr *pB, int iTab){ + return sqlite3ExprCompare( + sqlite3ExprSkipCollate(pA), + sqlite3ExprSkipCollate(pB), + iTab); +} + /* ** Return true if we can prove the pE2 will always be true if pE1 is ** true. Return false if we cannot complete the proof or if pE2 might diff --git a/src/func.c b/src/func.c index 885725bc6b..3d5a059a9f 100644 --- a/src/func.c +++ b/src/func.c @@ -204,15 +204,13 @@ static void instrFunc( if( typeHaystack==SQLITE_BLOB && typeNeedle==SQLITE_BLOB ){ zHaystack = sqlite3_value_blob(argv[0]); zNeedle = sqlite3_value_blob(argv[1]); - assert( zNeedle!=0 ); - assert( zHaystack!=0 || nHaystack==0 ); isText = 0; }else{ zHaystack = sqlite3_value_text(argv[0]); zNeedle = sqlite3_value_text(argv[1]); isText = 1; - if( zHaystack==0 || zNeedle==0 ) return; } + if( zNeedle==0 || (nHaystack && zHaystack==0) ) return; while( nNeedle<=nHaystack && memcmp(zHaystack, zNeedle, nNeedle)!=0 ){ N++; do{ diff --git a/src/insert.c b/src/insert.c index e261c1994e..a1f5442515 100644 --- a/src/insert.c +++ b/src/insert.c @@ -1527,6 +1527,9 @@ void sqlite3GenerateConstraintChecks( } sqlite3VdbeAddOp3(v, OP_MakeRecord, regIdx, pIdx->nColumn, aRegIdx[ix]); VdbeComment((v, "for %s", pIdx->zName)); +#ifdef SQLITE_ENABLE_NULL_TRIM + if( pIdx->idxType==2 ) sqlite3SetMakeRecordP5(v, pIdx->pTable); +#endif /* In an UPDATE operation, if this index is the PRIMARY KEY index ** of a WITHOUT ROWID table and there has been no change the @@ -1651,7 +1654,7 @@ void sqlite3GenerateConstraintChecks( } sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, regR, nPkField, 0, OE_Replace, - (pIdx==pPk ? ONEPASS_SINGLE : ONEPASS_OFF), -1); + (pIdx==pPk ? ONEPASS_SINGLE : ONEPASS_OFF), iThisCur); seenReplace = 1; break; } @@ -1668,6 +1671,28 @@ void sqlite3GenerateConstraintChecks( VdbeModuleComment((v, "END: GenCnstCks(%d)", seenReplace)); } +#ifdef SQLITE_ENABLE_NULL_TRIM +/* +** Change the P5 operand on the last opcode (which should be an OP_MakeRecord) +** to be the number of columns in table pTab that must not be NULL-trimmed. +** +** Or if no columns of pTab may be NULL-trimmed, leave P5 at zero. +*/ +void sqlite3SetMakeRecordP5(Vdbe *v, Table *pTab){ + u16 i; + + /* Records with omitted columns are only allowed for schema format + ** version 2 and later (SQLite version 3.1.4, 2005-02-20). */ + if( pTab->pSchema->file_format<2 ) return; + + for(i=pTab->nCol-1; i>0; i--){ + if( pTab->aCol[i].pDflt!=0 ) break; + if( pTab->aCol[i].colFlags & COLFLAG_PRIMKEY ) break; + } + sqlite3VdbeChangeP5(v, i+1); +} +#endif + /* ** This routine generates code to finish the INSERT or UPDATE operation ** that was started by a prior call to sqlite3GenerateConstraintChecks. @@ -1684,7 +1709,7 @@ void sqlite3CompleteInsertion( int iIdxCur, /* First index cursor */ int regNewData, /* Range of content */ int *aRegIdx, /* Register used by each index. 0 for unused indices */ - int isUpdate, /* True for UPDATE, False for INSERT */ + int update_flags, /* True for UPDATE, False for INSERT */ int appendBias, /* True if this is likely to be an append */ int useSeekResult /* True to set the USESEEKRESULT flag on OP_[Idx]Insert */ ){ @@ -1696,6 +1721,11 @@ void sqlite3CompleteInsertion( int i; /* Loop counter */ u8 bAffinityDone = 0; /* True if OP_Affinity has been run already */ + assert( update_flags==0 + || update_flags==OPFLAG_ISUPDATE + || update_flags==(OPFLAG_ISUPDATE|OPFLAG_SAVEPOSITION) + ); + v = sqlite3GetVdbe(pParse); assert( v!=0 ); assert( pTab->pSelect==0 ); /* This table is not a VIEW */ @@ -1706,21 +1736,30 @@ void sqlite3CompleteInsertion( sqlite3VdbeAddOp2(v, OP_IsNull, aRegIdx[i], sqlite3VdbeCurrentAddr(v)+2); VdbeCoverage(v); } - sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iIdxCur+i, aRegIdx[i], - aRegIdx[i]+1, - pIdx->uniqNotNull ? pIdx->nKeyCol: pIdx->nColumn); - pik_flags = 0; - if( useSeekResult ) pik_flags = OPFLAG_USESEEKRESULT; + pik_flags = (useSeekResult ? OPFLAG_USESEEKRESULT : 0); if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) ){ assert( pParse->nested==0 ); pik_flags |= OPFLAG_NCHANGE; + pik_flags |= (update_flags & OPFLAG_SAVEPOSITION); +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + if( update_flags==0 ){ + sqlite3VdbeAddOp4(v, OP_InsertInt, + iIdxCur+i, aRegIdx[i], 0, (char*)pTab, P4_TABLE + ); + sqlite3VdbeChangeP5(v, OPFLAG_ISNOOP); + } +#endif } + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iIdxCur+i, aRegIdx[i], + aRegIdx[i]+1, + pIdx->uniqNotNull ? pIdx->nKeyCol: pIdx->nColumn); sqlite3VdbeChangeP5(v, pik_flags); } if( !HasRowid(pTab) ) return; regData = regNewData + 1; regRec = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp3(v, OP_MakeRecord, regData, pTab->nCol, regRec); + sqlite3SetMakeRecordP5(v, pTab); if( !bAffinityDone ){ sqlite3TableAffinity(v, pTab, 0); sqlite3ExprCacheAffinityChange(pParse, regData, pTab->nCol); @@ -1729,7 +1768,7 @@ void sqlite3CompleteInsertion( pik_flags = 0; }else{ pik_flags = OPFLAG_NCHANGE; - pik_flags |= (isUpdate?OPFLAG_ISUPDATE:OPFLAG_LASTROWID); + pik_flags |= (update_flags?update_flags:OPFLAG_LASTROWID); } if( appendBias ){ pik_flags |= OPFLAG_APPEND; @@ -1940,7 +1979,7 @@ static int xferOptimization( return 0; /* tab1 must not have triggers */ } #ifndef SQLITE_OMIT_VIRTUALTABLE - if( pDest->tabFlags & TF_Virtual ){ + if( IsVirtual(pDest) ){ return 0; /* tab1 must not be a virtual table */ } #endif @@ -2002,7 +2041,7 @@ static int xferOptimization( return 0; /* source and destination must both be WITHOUT ROWID or not */ } #ifndef SQLITE_OMIT_VIRTUALTABLE - if( pSrc->tabFlags & TF_Virtual ){ + if( IsVirtual(pSrc) ){ return 0; /* tab2 must not be a virtual table */ } #endif @@ -2138,7 +2177,7 @@ static int xferOptimization( addr1 = sqlite3VdbeAddOp2(v, OP_Rowid, iSrc, regRowid); assert( (pDest->tabFlags & TF_Autoincrement)==0 ); } - sqlite3VdbeAddOp2(v, OP_RowData, iSrc, regData); + sqlite3VdbeAddOp3(v, OP_RowData, iSrc, regData, 1); if( db->flags & SQLITE_Vacuum ){ sqlite3VdbeAddOp3(v, OP_Last, iDest, 0, -1); insFlags = OPFLAG_NCHANGE|OPFLAG_LASTROWID| @@ -2170,7 +2209,7 @@ static int xferOptimization( sqlite3VdbeChangeP5(v, OPFLAG_BULKCSR); VdbeComment((v, "%s", pDestIdx->zName)); addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iSrc, 0); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_RowData, iSrc, regData); + sqlite3VdbeAddOp3(v, OP_RowData, iSrc, regData, 1); if( db->flags & SQLITE_Vacuum ){ /* This INSERT command is part of a VACUUM operation, which guarantees ** that the destination table is empty. If all indexed columns use @@ -2188,8 +2227,6 @@ static int xferOptimization( ** sorted order. */ for(i=0; inColumn; i++){ const char *zColl = pSrcIdx->azColl[i]; - assert( sqlite3_stricmp(sqlite3StrBINARY, zColl)!=0 - || sqlite3StrBINARY==zColl ); if( sqlite3_stricmp(sqlite3StrBINARY, zColl) ) break; } if( i==pSrcIdx->nColumn ){ diff --git a/src/legacy.c b/src/legacy.c index bd34512d31..df8e2780fb 100644 --- a/src/legacy.c +++ b/src/legacy.c @@ -73,7 +73,7 @@ int sqlite3_exec( (SQLITE_DONE==rc && !callbackIsInit && db->flags&SQLITE_NullCallback)) ){ if( !callbackIsInit ){ - azCols = sqlite3DbMallocZero(db, 2*nCol*sizeof(const char*) + 1); + azCols = sqlite3DbMallocRaw(db, (2*nCol+1)*sizeof(const char*)); if( azCols==0 ){ goto exec_out; } @@ -94,6 +94,7 @@ int sqlite3_exec( goto exec_out; } } + azVals[i] = 0; } if( xCallback(pArg, nCol, azVals, azCols) ){ /* EVIDENCE-OF: R-38229-40159 If the callback function to diff --git a/src/loadext.c b/src/loadext.c index 6011fd2bf8..3296be60dd 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -18,7 +18,6 @@ #endif #include "sqlite3ext.h" #include "sqliteInt.h" -#include #ifndef SQLITE_OMIT_LOAD_EXTENSION /* diff --git a/src/main.c b/src/main.c index 9aad8fdd4c..4ac5327e4f 100644 --- a/src/main.c +++ b/src/main.c @@ -921,6 +921,21 @@ sqlite_int64 sqlite3_last_insert_rowid(sqlite3 *db){ return db->lastRowid; } +/* +** Set the value returned by the sqlite3_last_insert_rowid() API function. +*/ +void sqlite3_set_last_insert_rowid(sqlite3 *db, sqlite3_int64 iRowid){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return; + } +#endif + sqlite3_mutex_enter(db->mutex); + db->lastRowid = iRowid; + sqlite3_mutex_leave(db->mutex); +} + /* ** Return the number of changes in the most recent call to sqlite3_exec(). */ diff --git a/src/malloc.c b/src/malloc.c index 84191c78a1..6d49107790 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -217,11 +217,26 @@ static void sqlite3MallocAlarm(int nByte){ ** Do a memory allocation with statistics and alarms. Assume the ** lock is already held. */ -static int mallocWithAlarm(int n, void **pp){ - int nFull; +static void mallocWithAlarm(int n, void **pp){ void *p; + int nFull; assert( sqlite3_mutex_held(mem0.mutex) ); + assert( n>0 ); + + /* In Firefox (circa 2017-02-08), xRoundup() is remapped to an internal + ** implementation of malloc_good_size(), which must be called in debug + ** mode and specifically when the DMD "Dark Matter Detector" is enabled + ** or else a crash results. Hence, do not attempt to optimize out the + ** following xRoundup() call. */ nFull = sqlite3GlobalConfig.m.xRoundup(n); + +#ifdef SQLITE_MAX_MEMORY + if( sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED)+nFull>SQLITE_MAX_MEMORY ){ + *pp = 0; + return; + } +#endif + sqlite3StatusHighwater(SQLITE_STATUS_MALLOC_SIZE, n); if( mem0.alarmThreshold>0 ){ sqlite3_int64 nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); @@ -245,7 +260,6 @@ static int mallocWithAlarm(int n, void **pp){ sqlite3StatusUp(SQLITE_STATUS_MALLOC_COUNT, 1); } *pp = p; - return nFull; } /* @@ -411,7 +425,7 @@ int sqlite3MallocSize(void *p){ int sqlite3DbMallocSize(sqlite3 *db, void *p){ assert( p!=0 ); if( db==0 || !isLookaside(db,p) ){ -#if SQLITE_DEBUG +#ifdef SQLITE_DEBUG if( db==0 ){ assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); @@ -472,7 +486,7 @@ void sqlite3DbFree(sqlite3 *db, void *p){ } if( isLookaside(db, p) ){ LookasideSlot *pBuf = (LookasideSlot*)p; -#if SQLITE_DEBUG +#ifdef SQLITE_DEBUG /* Trash all content in the buffer being freed */ memset(p, 0xaa, db->lookaside.sz); #endif diff --git a/src/mem1.c b/src/mem1.c index b960ccfd47..512ab3747f 100644 --- a/src/mem1.c +++ b/src/mem1.c @@ -57,7 +57,9 @@ */ #include #include +#ifdef SQLITE_MIGHT_BE_SINGLE_CORE #include +#endif /* SQLITE_MIGHT_BE_SINGLE_CORE */ static malloc_zone_t* _sqliteZone_; #define SQLITE_MALLOC(x) malloc_zone_malloc(_sqliteZone_, (x)) #define SQLITE_FREE(x) malloc_zone_free(_sqliteZone_, (x)); @@ -125,7 +127,9 @@ static malloc_zone_t* _sqliteZone_; */ static void *sqlite3MemMalloc(int nByte){ #ifdef SQLITE_MALLOCSIZE - void *p = SQLITE_MALLOC( nByte ); + void *p; + testcase( ROUND8(nByte)==nByte ); + p = SQLITE_MALLOC( nByte ); if( p==0 ){ testcase( sqlite3GlobalConfig.xLog!=0 ); sqlite3_log(SQLITE_NOMEM, "failed to allocate %u bytes of memory", nByte); @@ -134,7 +138,7 @@ static void *sqlite3MemMalloc(int nByte){ #else sqlite3_int64 *p; assert( nByte>0 ); - nByte = ROUND8(nByte); + testcase( ROUND8(nByte)!=nByte ); p = SQLITE_MALLOC( nByte+8 ); if( p ){ p[0] = nByte; @@ -248,19 +252,10 @@ static int sqlite3MemInit(void *NotUsed){ }else{ /* only 1 core, use our own zone to contention over global locks, ** e.g. we have our own dedicated locks */ - bool success; - malloc_zone_t* newzone = malloc_create_zone(4096, 0); - malloc_set_zone_name(newzone, "Sqlite_Heap"); - do{ - success = OSAtomicCompareAndSwapPtrBarrier(NULL, newzone, - (void * volatile *)&_sqliteZone_); - }while(!_sqliteZone_); - if( !success ){ - /* somebody registered a zone first */ - malloc_destroy_zone(newzone); - } + _sqliteZone_ = malloc_create_zone(4096, 0); + malloc_set_zone_name(_sqliteZone_, "Sqlite_Heap"); } -#endif +#endif /* defined(__APPLE__) && !defined(SQLITE_WITHOUT_ZONEMALLOC) */ UNUSED_PARAMETER(NotUsed); return SQLITE_OK; } diff --git a/src/mutex_w32.c b/src/mutex_w32.c index 9570bdc0bf..9da93cf319 100644 --- a/src/mutex_w32.c +++ b/src/mutex_w32.c @@ -87,8 +87,7 @@ void sqlite3MemoryBarrier(void){ SQLITE_MEMORY_BARRIER; #elif defined(__GNUC__) __sync_synchronize(); -#elif !defined(SQLITE_DISABLE_INTRINSIC) && \ - defined(_MSC_VER) && _MSC_VER>=1300 +#elif MSVC_VERSION>=1300 _ReadWriteBarrier(); #elif defined(MemoryBarrier) MemoryBarrier(); @@ -299,8 +298,8 @@ static void winMutexEnter(sqlite3_mutex *p){ p->owner = tid; p->nRef++; if( p->trace ){ - OSTRACE(("ENTER-MUTEX tid=%lu, mutex=%p (%d), nRef=%d\n", - tid, p, p->trace, p->nRef)); + OSTRACE(("ENTER-MUTEX tid=%lu, mutex(%d)=%p (%d), nRef=%d\n", + tid, p->id, p, p->trace, p->nRef)); } #endif } @@ -342,8 +341,8 @@ static int winMutexTry(sqlite3_mutex *p){ #endif #ifdef SQLITE_DEBUG if( p->trace ){ - OSTRACE(("TRY-MUTEX tid=%lu, mutex=%p (%d), owner=%lu, nRef=%d, rc=%s\n", - tid, p, p->trace, p->owner, p->nRef, sqlite3ErrName(rc))); + OSTRACE(("TRY-MUTEX tid=%lu, mutex(%d)=%p (%d), owner=%lu, nRef=%d, rc=%s\n", + tid, p->id, p, p->trace, p->owner, p->nRef, sqlite3ErrName(rc))); } #endif return rc; @@ -371,8 +370,8 @@ static void winMutexLeave(sqlite3_mutex *p){ LeaveCriticalSection(&p->mutex); #ifdef SQLITE_DEBUG if( p->trace ){ - OSTRACE(("LEAVE-MUTEX tid=%lu, mutex=%p (%d), nRef=%d\n", - tid, p, p->trace, p->nRef)); + OSTRACE(("LEAVE-MUTEX tid=%lu, mutex(%d)=%p (%d), nRef=%d\n", + tid, p->id, p, p->trace, p->nRef)); } #endif } diff --git a/src/os_win.c b/src/os_win.c index 2cb5f7b0c8..a87d7d0925 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -352,7 +352,34 @@ struct winVfsAppData { ****************************************************************************** */ #ifndef SQLITE_WIN32_HEAP_CREATE -# define SQLITE_WIN32_HEAP_CREATE (TRUE) +# define SQLITE_WIN32_HEAP_CREATE (TRUE) +#endif + +/* + * This is the maximum possible initial size of the Win32-specific heap, in + * bytes. + */ +#ifndef SQLITE_WIN32_HEAP_MAX_INIT_SIZE +# define SQLITE_WIN32_HEAP_MAX_INIT_SIZE (4294967295U) +#endif + +/* + * This is the extra space for the initial size of the Win32-specific heap, + * in bytes. This value may be zero. + */ +#ifndef SQLITE_WIN32_HEAP_INIT_EXTRA +# define SQLITE_WIN32_HEAP_INIT_EXTRA (4194304) +#endif + +/* + * Calculate the maximum legal cache size, in pages, based on the maximum + * possible initial heap size and the default page size, setting aside the + * needed extra space. + */ +#ifndef SQLITE_WIN32_MAX_CACHE_SIZE +# define SQLITE_WIN32_MAX_CACHE_SIZE (((SQLITE_WIN32_HEAP_MAX_INIT_SIZE) - \ + (SQLITE_WIN32_HEAP_INIT_EXTRA)) / \ + (SQLITE_DEFAULT_PAGE_SIZE)) #endif /* @@ -361,25 +388,36 @@ struct winVfsAppData { */ #ifndef SQLITE_WIN32_CACHE_SIZE # if SQLITE_DEFAULT_CACHE_SIZE>=0 -# define SQLITE_WIN32_CACHE_SIZE (SQLITE_DEFAULT_CACHE_SIZE) +# define SQLITE_WIN32_CACHE_SIZE (SQLITE_DEFAULT_CACHE_SIZE) # else -# define SQLITE_WIN32_CACHE_SIZE (-(SQLITE_DEFAULT_CACHE_SIZE)) +# define SQLITE_WIN32_CACHE_SIZE (-(SQLITE_DEFAULT_CACHE_SIZE)) # endif #endif +/* + * Make sure that the calculated cache size, in pages, cannot cause the + * initial size of the Win32-specific heap to exceed the maximum amount + * of memory that can be specified in the call to HeapCreate. + */ +#if SQLITE_WIN32_CACHE_SIZE>SQLITE_WIN32_MAX_CACHE_SIZE +# undef SQLITE_WIN32_CACHE_SIZE +# define SQLITE_WIN32_CACHE_SIZE (2000) +#endif + /* * The initial size of the Win32-specific heap. This value may be zero. */ #ifndef SQLITE_WIN32_HEAP_INIT_SIZE -# define SQLITE_WIN32_HEAP_INIT_SIZE ((SQLITE_WIN32_CACHE_SIZE) * \ - (SQLITE_DEFAULT_PAGE_SIZE) + 4194304) +# define SQLITE_WIN32_HEAP_INIT_SIZE ((SQLITE_WIN32_CACHE_SIZE) * \ + (SQLITE_DEFAULT_PAGE_SIZE) + \ + (SQLITE_WIN32_HEAP_INIT_EXTRA)) #endif /* * The maximum size of the Win32-specific heap. This value may be zero. */ #ifndef SQLITE_WIN32_HEAP_MAX_SIZE -# define SQLITE_WIN32_HEAP_MAX_SIZE (0) +# define SQLITE_WIN32_HEAP_MAX_SIZE (0) #endif /* @@ -387,7 +425,7 @@ struct winVfsAppData { * zero for the default behavior. */ #ifndef SQLITE_WIN32_HEAP_FLAGS -# define SQLITE_WIN32_HEAP_FLAGS (0) +# define SQLITE_WIN32_HEAP_FLAGS (0) #endif diff --git a/src/pager.c b/src/pager.c index a7e83242dd..19f54b1cff 100644 --- a/src/pager.c +++ b/src/pager.c @@ -817,14 +817,20 @@ static const unsigned char aJournalMagic[] = { #define isOpen(pFd) ((pFd)->pMethods!=0) /* -** Return true if this pager uses a write-ahead log instead of the usual -** rollback journal. Otherwise false. +** Return true if this pager uses a write-ahead log to read page pgno. +** Return false if the pager reads pgno directly from the database. */ -#ifndef SQLITE_OMIT_WAL -int sqlite3PagerUseWal(Pager *pPager){ - return (pPager->pWal!=0); +#if !defined(SQLITE_OMIT_WAL) && defined(SQLITE_DIRECT_OVERFLOW_READ) +int sqlite3PagerUseWal(Pager *pPager, Pgno pgno){ + u32 iRead = 0; + int rc; + if( pPager->pWal==0 ) return 0; + rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iRead); + return rc || iRead; } -# define pagerUseWal(x) sqlite3PagerUseWal(x) +#endif +#ifndef SQLITE_OMIT_WAL +# define pagerUseWal(x) ((x)->pWal!=0) #else # define pagerUseWal(x) 0 # define pagerRollbackWal(x) 0 diff --git a/src/pager.h b/src/pager.h index 60a844c713..8bf2caa6f5 100644 --- a/src/pager.h +++ b/src/pager.h @@ -179,14 +179,16 @@ int sqlite3PagerSharedLock(Pager *pPager); int sqlite3PagerWalCallback(Pager *pPager); int sqlite3PagerOpenWal(Pager *pPager, int *pisOpen); int sqlite3PagerCloseWal(Pager *pPager, sqlite3*); - int sqlite3PagerUseWal(Pager *pPager); +# ifdef SQLITE_DIRECT_OVERFLOW_READ + int sqlite3PagerUseWal(Pager *pPager, Pgno); +# endif # ifdef SQLITE_ENABLE_SNAPSHOT int sqlite3PagerSnapshotGet(Pager *pPager, sqlite3_snapshot **ppSnapshot); int sqlite3PagerSnapshotOpen(Pager *pPager, sqlite3_snapshot *pSnapshot); int sqlite3PagerSnapshotRecover(Pager *pPager); # endif #else -# define sqlite3PagerUseWal(x) 0 +# define sqlite3PagerUseWal(x,y) 0 #endif #ifdef SQLITE_ENABLE_ZIPVFS diff --git a/src/parse.y b/src/parse.y index fa2aef17a7..914a8af44a 100644 --- a/src/parse.y +++ b/src/parse.y @@ -65,6 +65,19 @@ */ #define YYPARSEFREENEVERNULL 1 +/* +** In the amalgamation, the parse.c file generated by lemon and the +** tokenize.c file are concatenated. In that case, sqlite3RunParser() +** has access to the the size of the yyParser object and so the parser +** engine can be allocated from stack. In that case, only the +** sqlite3ParserInit() and sqlite3ParserFinalize() routines are invoked +** and the sqlite3ParserAlloc() and sqlite3ParserFree() routines can be +** omitted. +*/ +#ifdef SQLITE_AMALGAMATION +# define sqlite3Parser_ENGINEALWAYSONSTACK 1 +#endif + /* ** Alternative datatype for the argument to the malloc() routine passed ** into sqlite3ParserAlloc(). The default is size_t. @@ -902,7 +915,7 @@ term(A) ::= INTEGER(X). { A.pExpr = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &X, 1); A.zStart = X.z; A.zEnd = X.z + X.n; - if( A.pExpr ) A.pExpr->flags |= EP_Leaf; + if( A.pExpr ) A.pExpr->flags |= EP_Leaf|EP_Resolved; } expr(A) ::= VARIABLE(X). { if( !(X.z[0]=='#' && sqlite3Isdigit(X.z[1])) ){ @@ -1003,7 +1016,7 @@ expr(A) ::= expr(A) STAR|SLASH|REM(OP) expr(Y). {spanBinaryExpr(pParse,@OP,&A,&Y);} expr(A) ::= expr(A) CONCAT(OP) expr(Y). {spanBinaryExpr(pParse,@OP,&A,&Y);} %type likeop {Token} -likeop(A) ::= LIKE_KW|MATCH(X). {A=X;/*A-overwrites-X*/} +likeop(A) ::= LIKE_KW|MATCH(A). likeop(A) ::= NOT LIKE_KW|MATCH(X). {A=X; A.n|=0x80000000; /*A-overwrite-X*/} expr(A) ::= expr(A) likeop(OP) expr(Y). [LIKE_KW] { ExprList *pList; diff --git a/src/pcache.c b/src/pcache.c index 0fc44c5499..dc7d00f306 100644 --- a/src/pcache.c +++ b/src/pcache.c @@ -104,7 +104,7 @@ struct PCache { ** ** assert( sqlite3PcachePageSanity(pPg) ); */ -#if SQLITE_DEBUG +#ifdef SQLITE_DEBUG int sqlite3PcachePageSanity(PgHdr *pPg){ PCache *pCache; assert( pPg!=0 ); diff --git a/src/pragma.c b/src/pragma.c index 5251d0cd26..6c27fdc5c9 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -279,7 +279,7 @@ const char *sqlite3JournalModename(int eMode){ ** Locate a pragma in the aPragmaName[] array. */ static const PragmaName *pragmaLocate(const char *zName){ - int upr, lwr, mid, rc; + int upr, lwr, mid = 0, rc; lwr = 0; upr = ArraySize(aPragmaName)-1; while( lwr<=upr ){ @@ -295,6 +295,22 @@ static const PragmaName *pragmaLocate(const char *zName){ return lwr>upr ? 0 : &aPragmaName[mid]; } +/* +** Helper subroutine for PRAGMA integrity_check: +** +** Generate code to output a single-column result row with the result +** held in register regResult. Decrement the result count and halt if +** the maximum number of result rows have been issued. +*/ +static int integrityCheckResultRow(Vdbe *v, int regResult){ + int addr; + sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, 1); + addr = sqlite3VdbeAddOp3(v, OP_IfPos, 1, sqlite3VdbeCurrentAddr(v)+2, 1); + VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); + return addr; +} + /* ** Process a pragma statement. ** @@ -999,7 +1015,7 @@ void sqlite3Pragma( if( !db->autoCommit ){ sqlite3ErrorMsg(pParse, "Safety level may not be changed inside a transaction"); - }else{ + }else if( iDb!=1 ){ int iLevel = (getSafetyLevel(zRight,0,1)+1) & PAGER_SYNCHRONOUS_MASK; if( iLevel==0 ) iLevel = 1; pDb->safety_level = iLevel; @@ -1098,29 +1114,33 @@ void sqlite3Pragma( } break; +#ifdef SQLITE_DEBUG case PragTyp_STATS: { Index *pIdx; HashElem *i; - pParse->nMem = 4; + pParse->nMem = 5; sqlite3CodeVerifySchema(pParse, iDb); for(i=sqliteHashFirst(&pDb->pSchema->tblHash); i; i=sqliteHashNext(i)){ Table *pTab = sqliteHashData(i); - sqlite3VdbeMultiLoad(v, 1, "ssii", + sqlite3VdbeMultiLoad(v, 1, "ssiii", pTab->zName, 0, pTab->szTabRow, - pTab->nRowLogEst); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); + pTab->nRowLogEst, + pTab->tabFlags); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 5); for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - sqlite3VdbeMultiLoad(v, 2, "sii", + sqlite3VdbeMultiLoad(v, 2, "siii", pIdx->zName, pIdx->szIdxRow, - pIdx->aiRowLogEst[0]); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); + pIdx->aiRowLogEst[0], + pIdx->hasStat1); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 5); } } } break; +#endif case PragTyp_INDEX_INFO: if( zRight ){ Index *pIdx; @@ -1377,9 +1397,17 @@ void sqlite3Pragma( #endif #ifndef SQLITE_OMIT_INTEGRITY_CHECK - /* Pragma "quick_check" is reduced version of + /* PRAGMA integrity_check + ** PRAGMA integrity_check(N) + ** PRAGMA quick_check + ** PRAGMA quick_check(N) + ** + ** Verify the integrity of the database. + ** + ** The "quick_check" is reduced version of ** integrity_check designed to detect most database corruption - ** without most of the overhead of a full integrity-check. + ** without the overhead of cross-checking indexes. Quick_check + ** is linear time wherease integrity_check is O(NlogN). */ case PragTyp_INTEGRITY_CHECK: { int i, j, addr, mxErr; @@ -1410,7 +1438,7 @@ void sqlite3Pragma( mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; } } - sqlite3VdbeAddOp2(v, OP_Integer, mxErr, 1); /* reg[1] holds errors left */ + sqlite3VdbeAddOp2(v, OP_Integer, mxErr-1, 1); /* reg[1] holds errors left */ /* Do an integrity check on each database file */ for(i=0; inDb; i++){ @@ -1425,10 +1453,6 @@ void sqlite3Pragma( if( iDb>=0 && i!=iDb ) continue; sqlite3CodeVerifySchema(pParse, i); - addr = sqlite3VdbeAddOp1(v, OP_IfPos, 1); /* Halt if out of errors */ - VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); - sqlite3VdbeJumpHere(v, addr); /* Do an integrity check of the B-Tree ** @@ -1468,12 +1492,12 @@ void sqlite3Pragma( P4_DYNAMIC); sqlite3VdbeAddOp3(v, OP_Move, 2, 4, 1); sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 2); - sqlite3VdbeAddOp2(v, OP_ResultRow, 2, 1); + integrityCheckResultRow(v, 2); sqlite3VdbeJumpHere(v, addr); /* Make sure all the indices are constructed correctly. */ - for(x=sqliteHashFirst(pTbls); x && !isQuick; x=sqliteHashNext(x)){ + for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ Table *pTab = sqliteHashData(x); Index *pIdx, *pPk; Index *pPrior = 0; @@ -1481,12 +1505,14 @@ void sqlite3Pragma( int iDataCur, iIdxCur; int r1 = -1; - if( pTab->pIndex==0 ) continue; + if( pTab->tnum<1 ) continue; /* Skip VIEWs or VIRTUAL TABLEs */ + if( pTab->pCheck==0 + && (pTab->tabFlags & TF_HasNotNull)==0 + && (pTab->pIndex==0 || isQuick) + ){ + continue; /* No additional checks needed for this table */ + } pPk = HasRowid(pTab) ? 0 : sqlite3PrimaryKeyIndex(pTab); - addr = sqlite3VdbeAddOp1(v, OP_IfPos, 1); /* Stop if out of errors */ - VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); - sqlite3VdbeJumpHere(v, addr); sqlite3ExprCacheClear(pParse); sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenRead, 0, 1, 0, &iDataCur, &iIdxCur); @@ -1501,24 +1527,42 @@ void sqlite3Pragma( /* Verify that all NOT NULL columns really are NOT NULL */ for(j=0; jnCol; j++){ char *zErr; - int jmp2, jmp3; + int jmp2; if( j==pTab->iPKey ) continue; if( pTab->aCol[j].notNull==0 ) continue; sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, j, 3); sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); jmp2 = sqlite3VdbeAddOp1(v, OP_NotNull, 3); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); /* Decrement error limit */ zErr = sqlite3MPrintf(db, "NULL value in %s.%s", pTab->zName, pTab->aCol[j].zName); sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); - sqlite3VdbeAddOp2(v, OP_ResultRow, 3, 1); - jmp3 = sqlite3VdbeAddOp1(v, OP_IfPos, 1); VdbeCoverage(v); - sqlite3VdbeAddOp0(v, OP_Halt); + integrityCheckResultRow(v, 3); sqlite3VdbeJumpHere(v, jmp2); - sqlite3VdbeJumpHere(v, jmp3); + } + /* Verify CHECK constraints */ + if( pTab->pCheck && (db->flags & SQLITE_IgnoreChecks)==0 ){ + int addrCkFault = sqlite3VdbeMakeLabel(v); + int addrCkOk = sqlite3VdbeMakeLabel(v); + ExprList *pCheck = pTab->pCheck; + char *zErr; + int k; + pParse->iSelfTab = iDataCur; + sqlite3ExprCachePush(pParse); + for(k=pCheck->nExpr-1; k>0; k--){ + sqlite3ExprIfFalse(pParse, pCheck->a[k].pExpr, addrCkFault, 0); + } + sqlite3ExprIfTrue(pParse, pCheck->a[0].pExpr, addrCkOk, + SQLITE_JUMPIFNULL); + sqlite3VdbeResolveLabel(v, addrCkFault); + zErr = sqlite3MPrintf(db, "CHECK constraint failed in %s", + pTab->zName); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); + integrityCheckResultRow(v, 3); + sqlite3VdbeResolveLabel(v, addrCkOk); + sqlite3ExprCachePop(pParse); } /* Validate index entries for the current row */ - for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ + for(j=0, pIdx=pTab->pIndex; pIdx && !isQuick; pIdx=pIdx->pNext, j++){ int jmp2, jmp3, jmp4, jmp5; int ckUniq = sqlite3VdbeMakeLabel(v); if( pPk==pIdx ) continue; @@ -1529,16 +1573,13 @@ void sqlite3Pragma( /* Verify that an index entry exists for the current table row */ jmp2 = sqlite3VdbeAddOp4Int(v, OP_Found, iIdxCur+j, ckUniq, r1, pIdx->nColumn); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); /* Decrement error limit */ sqlite3VdbeLoadString(v, 3, "row "); sqlite3VdbeAddOp3(v, OP_Concat, 7, 3, 3); sqlite3VdbeLoadString(v, 4, " missing from index "); sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); jmp5 = sqlite3VdbeLoadString(v, 4, pIdx->zName); sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); - sqlite3VdbeAddOp2(v, OP_ResultRow, 3, 1); - jmp4 = sqlite3VdbeAddOp1(v, OP_IfPos, 1); VdbeCoverage(v); - sqlite3VdbeAddOp0(v, OP_Halt); + jmp4 = integrityCheckResultRow(v, 3); sqlite3VdbeJumpHere(v, jmp2); /* For UNIQUE indexes, verify that only one entry exists with the ** current key. The entry is unique if (1) any column is NULL @@ -1559,7 +1600,6 @@ void sqlite3Pragma( sqlite3VdbeJumpHere(v, jmp6); sqlite3VdbeAddOp4Int(v, OP_IdxGT, iIdxCur+j, uniqOk, r1, pIdx->nKeyCol); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); /* Decrement error limit */ sqlite3VdbeLoadString(v, 3, "non-unique entry in index "); sqlite3VdbeGoto(v, jmp5); sqlite3VdbeResolveLabel(v, uniqOk); @@ -1570,19 +1610,18 @@ void sqlite3Pragma( sqlite3VdbeAddOp2(v, OP_Next, iDataCur, loopTop); VdbeCoverage(v); sqlite3VdbeJumpHere(v, loopTop-1); #ifndef SQLITE_OMIT_BTREECOUNT - sqlite3VdbeLoadString(v, 2, "wrong # of entries in index "); - for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ - if( pPk==pIdx ) continue; - addr = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp2(v, OP_IfPos, 1, addr+2); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); - sqlite3VdbeAddOp2(v, OP_Count, iIdxCur+j, 3); - sqlite3VdbeAddOp3(v, OP_Eq, 8+j, addr+8, 3); VdbeCoverage(v); - sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); - sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); - sqlite3VdbeLoadString(v, 3, pIdx->zName); - sqlite3VdbeAddOp3(v, OP_Concat, 3, 2, 7); - sqlite3VdbeAddOp2(v, OP_ResultRow, 7, 1); + if( !isQuick ){ + sqlite3VdbeLoadString(v, 2, "wrong # of entries in index "); + for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ + if( pPk==pIdx ) continue; + sqlite3VdbeAddOp2(v, OP_Count, iIdxCur+j, 3); + addr = sqlite3VdbeAddOp3(v, OP_Eq, 8+j, 0, 3); VdbeCoverage(v); + sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); + sqlite3VdbeLoadString(v, 3, pIdx->zName); + sqlite3VdbeAddOp3(v, OP_Concat, 3, 2, 7); + integrityCheckResultRow(v, 7); + sqlite3VdbeJumpHere(v, addr); + } } #endif /* SQLITE_OMIT_BTREECOUNT */ } @@ -1591,7 +1630,7 @@ void sqlite3Pragma( static const int iLn = VDBE_OFFSET_LINENO(2); static const VdbeOpList endCode[] = { { OP_AddImm, 1, 0, 0}, /* 0 */ - { OP_If, 1, 4, 0}, /* 1 */ + { OP_IfNotZero, 1, 4, 0}, /* 1 */ { OP_String8, 0, 3, 0}, /* 2 */ { OP_ResultRow, 3, 1, 0}, /* 3 */ }; @@ -1599,7 +1638,7 @@ void sqlite3Pragma( aOp = sqlite3VdbeAddOpList(v, ArraySize(endCode), endCode, iLn); if( aOp ){ - aOp[0].p2 = -mxErr; + aOp[0].p2 = 1-mxErr; aOp[2].p4type = P4_STATIC; aOp[2].p4.z = "ok"; } @@ -1824,6 +1863,118 @@ void sqlite3Pragma( break; } + /* + ** PRAGMA optimize + ** PRAGMA optimize(MASK) + ** PRAGMA schema.optimize + ** PRAGMA schema.optimize(MASK) + ** + ** Attempt to optimize the database. All schemas are optimized in the first + ** two forms, and only the specified schema is optimized in the latter two. + ** + ** The details of optimizations performed by this pragma are expected + ** to change and improve over time. Applications should anticipate that + ** this pragma will perform new optimizations in future releases. + ** + ** The optional argument is a bitmask of optimizations to perform: + ** + ** 0x0001 Debugging mode. Do not actually perform any optimizations + ** but instead return one line of text for each optimization + ** that would have been done. Off by default. + ** + ** 0x0002 Run ANALYZE on tables that might benefit. On by default. + ** See below for additional information. + ** + ** 0x0004 (Not yet implemented) Record usage and performance + ** information from the current session in the + ** database file so that it will be available to "optimize" + ** pragmas run by future database connections. + ** + ** 0x0008 (Not yet implemented) Create indexes that might have + ** been helpful to recent queries + ** + ** The default MASK is and always shall be 0xfffe. 0xfffe means perform all ** of the optimizations listed above except Debug Mode, including new + ** optimizations that have not yet been invented. If new optimizations are + ** ever added that should be off by default, those off-by-default + ** optimizations will have bitmasks of 0x10000 or larger. + ** + ** DETERMINATION OF WHEN TO RUN ANALYZE + ** + ** In the current implementation, a table is analyzed if only if all of + ** the following are true: + ** + ** (1) MASK bit 0x02 is set. + ** + ** (2) The query planner used sqlite_stat1-style statistics for one or + ** more indexes of the table at some point during the lifetime of + ** the current connection. + ** + ** (3) One or more indexes of the table are currently unanalyzed OR + ** the number of rows in the table has increased by 25 times or more + ** since the last time ANALYZE was run. + ** + ** The rules for when tables are analyzed are likely to change in + ** future releases. + */ + case PragTyp_OPTIMIZE: { + int iDbLast; /* Loop termination point for the schema loop */ + int iTabCur; /* Cursor for a table whose size needs checking */ + HashElem *k; /* Loop over tables of a schema */ + Schema *pSchema; /* The current schema */ + Table *pTab; /* A table in the schema */ + Index *pIdx; /* An index of the table */ + LogEst szThreshold; /* Size threshold above which reanalysis is needd */ + char *zSubSql; /* SQL statement for the OP_SqlExec opcode */ + u32 opMask; /* Mask of operations to perform */ + + if( zRight ){ + opMask = (u32)sqlite3Atoi(zRight); + if( (opMask & 0x02)==0 ) break; + }else{ + opMask = 0xfffe; + } + iTabCur = pParse->nTab++; + for(iDbLast = zDb?iDb:db->nDb-1; iDb<=iDbLast; iDb++){ + if( iDb==1 ) continue; + sqlite3CodeVerifySchema(pParse, iDb); + pSchema = db->aDb[iDb].pSchema; + for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ + pTab = (Table*)sqliteHashData(k); + + /* If table pTab has not been used in a way that would benefit from + ** having analysis statistics during the current session, then skip it. + ** This also has the effect of skipping virtual tables and views */ + if( (pTab->tabFlags & TF_StatsUsed)==0 ) continue; + + /* Reanalyze if the table is 25 times larger than the last analysis */ + szThreshold = pTab->nRowLogEst + 46; assert( sqlite3LogEst(25)==46 ); + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( !pIdx->hasStat1 ){ + szThreshold = 0; /* Always analyze if any index lacks statistics */ + break; + } + } + if( szThreshold ){ + sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); + sqlite3VdbeAddOp3(v, OP_IfSmaller, iTabCur, + sqlite3VdbeCurrentAddr(v)+2+(opMask&1), szThreshold); + VdbeCoverage(v); + } + zSubSql = sqlite3MPrintf(db, "ANALYZE \"%w\".\"%w\"", + db->aDb[iDb].zDbSName, pTab->zName); + if( opMask & 0x01 ){ + int r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp4(v, OP_String8, 0, r1, 0, zSubSql, P4_DYNAMIC); + sqlite3VdbeAddOp2(v, OP_ResultRow, r1, 1); + }else{ + sqlite3VdbeAddOp4(v, OP_SqlExec, 0, 0, 0, zSubSql, P4_DYNAMIC); + } + } + } + sqlite3VdbeAddOp0(v, OP_Expire); + break; + } + /* ** PRAGMA busy_timeout ** PRAGMA busy_timeout = N diff --git a/src/pragma.h b/src/pragma.h index 5d8d0aa35b..9b1c723b3e 100644 --- a/src/pragma.h +++ b/src/pragma.h @@ -30,11 +30,11 @@ #define PragTyp_LOCKING_MODE 22 #define PragTyp_PAGE_COUNT 23 #define PragTyp_MMAP_SIZE 24 -#define PragTyp_PAGE_SIZE 25 -#define PragTyp_SECURE_DELETE 26 -#define PragTyp_SHRINK_MEMORY 27 -#define PragTyp_SOFT_HEAP_LIMIT 28 -#define PragTyp_STATS 29 +#define PragTyp_OPTIMIZE 25 +#define PragTyp_PAGE_SIZE 26 +#define PragTyp_SECURE_DELETE 27 +#define PragTyp_SHRINK_MEMORY 28 +#define PragTyp_SOFT_HEAP_LIMIT 29 #define PragTyp_SYNCHRONOUS 30 #define PragTyp_TABLE_INFO 31 #define PragTyp_TEMP_STORE 32 @@ -48,6 +48,7 @@ #define PragTyp_REKEY 40 #define PragTyp_LOCK_STATUS 41 #define PragTyp_PARSER_TRACE 42 +#define PragTyp_STATS 43 /* Property flags associated with various pragma. */ #define PragFlg_NeedSchema 0x01 /* Force schema load before running */ @@ -71,47 +72,48 @@ static const char *const pragCName[] = { /* 4 */ "notnull", /* 5 */ "dflt_value", /* 6 */ "pk", - /* 7 */ "table", /* Used by: stats */ - /* 8 */ "index", - /* 9 */ "width", - /* 10 */ "height", - /* 11 */ "seqno", /* Used by: index_info */ - /* 12 */ "cid", - /* 13 */ "name", - /* 14 */ "seqno", /* Used by: index_xinfo */ - /* 15 */ "cid", - /* 16 */ "name", - /* 17 */ "desc", - /* 18 */ "coll", - /* 19 */ "key", - /* 20 */ "seq", /* Used by: index_list */ - /* 21 */ "name", - /* 22 */ "unique", - /* 23 */ "origin", - /* 24 */ "partial", - /* 25 */ "seq", /* Used by: database_list */ - /* 26 */ "name", - /* 27 */ "file", - /* 28 */ "seq", /* Used by: collation_list */ - /* 29 */ "name", - /* 30 */ "id", /* Used by: foreign_key_list */ - /* 31 */ "seq", - /* 32 */ "table", - /* 33 */ "from", - /* 34 */ "to", - /* 35 */ "on_update", - /* 36 */ "on_delete", - /* 37 */ "match", - /* 38 */ "table", /* Used by: foreign_key_check */ - /* 39 */ "rowid", - /* 40 */ "parent", - /* 41 */ "fkid", - /* 42 */ "busy", /* Used by: wal_checkpoint */ - /* 43 */ "log", - /* 44 */ "checkpointed", - /* 45 */ "timeout", /* Used by: busy_timeout */ - /* 46 */ "database", /* Used by: lock_status */ - /* 47 */ "status", + /* 7 */ "tbl", /* Used by: stats */ + /* 8 */ "idx", + /* 9 */ "wdth", + /* 10 */ "hght", + /* 11 */ "flgs", + /* 12 */ "seqno", /* Used by: index_info */ + /* 13 */ "cid", + /* 14 */ "name", + /* 15 */ "seqno", /* Used by: index_xinfo */ + /* 16 */ "cid", + /* 17 */ "name", + /* 18 */ "desc", + /* 19 */ "coll", + /* 20 */ "key", + /* 21 */ "seq", /* Used by: index_list */ + /* 22 */ "name", + /* 23 */ "unique", + /* 24 */ "origin", + /* 25 */ "partial", + /* 26 */ "seq", /* Used by: database_list */ + /* 27 */ "name", + /* 28 */ "file", + /* 29 */ "seq", /* Used by: collation_list */ + /* 30 */ "name", + /* 31 */ "id", /* Used by: foreign_key_list */ + /* 32 */ "seq", + /* 33 */ "table", + /* 34 */ "from", + /* 35 */ "to", + /* 36 */ "on_update", + /* 37 */ "on_delete", + /* 38 */ "match", + /* 39 */ "table", /* Used by: foreign_key_check */ + /* 40 */ "rowid", + /* 41 */ "parent", + /* 42 */ "fkid", + /* 43 */ "busy", /* Used by: wal_checkpoint */ + /* 44 */ "log", + /* 45 */ "checkpointed", + /* 46 */ "timeout", /* Used by: busy_timeout */ + /* 47 */ "database", /* Used by: lock_status */ + /* 48 */ "status", }; /* Definitions of all built-in pragmas */ @@ -157,7 +159,7 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "busy_timeout", /* ePragTyp: */ PragTyp_BUSY_TIMEOUT, /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 45, 1, + /* ColNames: */ 46, 1, /* iArg: */ 0 }, #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) {/* zName: */ "cache_size", @@ -194,7 +196,7 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "collation_list", /* ePragTyp: */ PragTyp_COLLATION_LIST, /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 28, 2, + /* ColNames: */ 29, 2, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_COMPILEOPTION_DIAGS) @@ -229,7 +231,7 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "database_list", /* ePragTyp: */ PragTyp_DATABASE_LIST, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0, - /* ColNames: */ 25, 3, + /* ColNames: */ 26, 3, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && !defined(SQLITE_OMIT_DEPRECATED) @@ -266,14 +268,14 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "foreign_key_check", /* ePragTyp: */ PragTyp_FOREIGN_KEY_CHECK, /* ePragFlg: */ PragFlg_NeedSchema, - /* ColNames: */ 38, 4, + /* ColNames: */ 39, 4, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FOREIGN_KEY) {/* zName: */ "foreign_key_list", /* ePragTyp: */ PragTyp_FOREIGN_KEY_LIST, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 30, 8, + /* ColNames: */ 31, 8, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) @@ -336,17 +338,17 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "index_info", /* ePragTyp: */ PragTyp_INDEX_INFO, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 11, 3, + /* ColNames: */ 12, 3, /* iArg: */ 0 }, {/* zName: */ "index_list", /* ePragTyp: */ PragTyp_INDEX_LIST, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 20, 5, + /* ColNames: */ 21, 5, /* iArg: */ 0 }, {/* zName: */ "index_xinfo", /* ePragTyp: */ PragTyp_INDEX_INFO, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 14, 6, + /* ColNames: */ 15, 6, /* iArg: */ 1 }, #endif #if !defined(SQLITE_OMIT_INTEGRITY_CHECK) @@ -393,7 +395,7 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "lock_status", /* ePragTyp: */ PragTyp_LOCK_STATUS, /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 46, 2, + /* ColNames: */ 47, 2, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) @@ -412,6 +414,13 @@ static const PragmaName aPragmaName[] = { /* ePragFlg: */ 0, /* ColNames: */ 0, 0, /* iArg: */ 0 }, +#endif + {/* zName: */ "optimize", + /* ePragTyp: */ PragTyp_OPTIMIZE, + /* ePragFlg: */ PragFlg_Result1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) {/* zName: */ "page_count", /* ePragTyp: */ PragTyp_PAGE_COUNT, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq, @@ -510,11 +519,11 @@ static const PragmaName aPragmaName[] = { /* iArg: */ SQLITE_SqlTrace }, #endif #endif -#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) && defined(SQLITE_DEBUG) {/* zName: */ "stats", /* ePragTyp: */ PragTyp_STATS, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq, - /* ColNames: */ 7, 4, + /* ColNames: */ 7, 5, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) @@ -593,7 +602,7 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "wal_checkpoint", /* ePragTyp: */ PragTyp_WAL_CHECKPOINT, /* ePragFlg: */ PragFlg_NeedSchema, - /* ColNames: */ 42, 3, + /* ColNames: */ 43, 3, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) @@ -604,4 +613,4 @@ static const PragmaName aPragmaName[] = { /* iArg: */ SQLITE_WriteSchema|SQLITE_RecoveryMode }, #endif }; -/* Number of pragmas: 60 on by default, 73 total. */ +/* Number of pragmas: 60 on by default, 74 total. */ diff --git a/src/printf.c b/src/printf.c index ede86f1208..a14e658875 100644 --- a/src/printf.c +++ b/src/printf.c @@ -15,7 +15,7 @@ ** Conversion types fall into various categories as defined by the ** following enumeration. */ -#define etRADIX 0 /* Integer types. %d, %x, %o, and so forth */ +#define etRADIX 0 /* non-decimal integer types. %x %o */ #define etFLOAT 1 /* Floating point. %f */ #define etEXP 2 /* Exponentional notation. %e and %E */ #define etGENERIC 3 /* Floating or exponential, depending on exponent. %g */ @@ -33,8 +33,9 @@ #define etPOINTER 13 /* The %p conversion */ #define etSQLESCAPE3 14 /* %w -> Strings with '\"' doubled */ #define etORDINAL 15 /* %r -> 1st, 2nd, 3rd, 4th, etc. English only */ +#define etDECIMAL 16 /* %d or %u, but not %x, %o */ -#define etINVALID 16 /* Any unrecognized conversion type */ +#define etINVALID 17 /* Any unrecognized conversion type */ /* @@ -58,8 +59,8 @@ typedef struct et_info { /* Information about each format field */ /* ** Allowed values for et_info.flags */ -#define FLAG_SIGNED 1 /* True if the value to convert is signed */ -#define FLAG_STRING 4 /* Allow infinity precision */ +#define FLAG_SIGNED 1 /* True if the value to convert is signed */ +#define FLAG_STRING 4 /* Allow infinite precision */ /* @@ -69,7 +70,7 @@ typedef struct et_info { /* Information about each format field */ static const char aDigits[] = "0123456789ABCDEF0123456789abcdef"; static const char aPrefix[] = "-x0\000X0"; static const et_info fmtinfo[] = { - { 'd', 10, 1, etRADIX, 0, 0 }, + { 'd', 10, 1, etDECIMAL, 0, 0 }, { 's', 0, 4, etSTRING, 0, 0 }, { 'g', 0, 1, etGENERIC, 30, 0 }, { 'z', 0, 4, etDYNSTRING, 0, 0 }, @@ -78,7 +79,7 @@ static const et_info fmtinfo[] = { { 'w', 0, 4, etSQLESCAPE3, 0, 0 }, { 'c', 0, 0, etCHARX, 0, 0 }, { 'o', 8, 0, etRADIX, 0, 2 }, - { 'u', 10, 0, etRADIX, 0, 0 }, + { 'u', 10, 0, etDECIMAL, 0, 0 }, { 'x', 16, 0, etRADIX, 16, 1 }, { 'X', 16, 0, etRADIX, 0, 4 }, #ifndef SQLITE_OMIT_FLOATING_POINT @@ -87,7 +88,7 @@ static const et_info fmtinfo[] = { { 'E', 0, 1, etEXP, 14, 0 }, { 'G', 0, 1, etGENERIC, 14, 0 }, #endif - { 'i', 10, 1, etRADIX, 0, 0 }, + { 'i', 10, 1, etDECIMAL, 0, 0 }, { 'n', 0, 0, etSIZE, 0, 0 }, { '%', 0, 0, etPERCENT, 0, 0 }, { 'p', 16, 0, etPOINTER, 0, 1 }, @@ -179,14 +180,13 @@ void sqlite3VXPrintf( int idx; /* A general purpose loop counter */ int width; /* Width of the current field */ etByte flag_leftjustify; /* True if "-" flag is present */ - etByte flag_plussign; /* True if "+" flag is present */ - etByte flag_blanksign; /* True if " " flag is present */ + etByte flag_prefix; /* '+' or ' ' or 0 for prefix */ etByte flag_alternateform; /* True if "#" flag is present */ etByte flag_altform2; /* True if "!" flag is present */ etByte flag_zeropad; /* True if field width constant starts with zero */ - etByte flag_long; /* True if "l" flag is present */ - etByte flag_longlong; /* True if the "ll" flag is present */ + etByte flag_long; /* 1 for the "l" flag, 2 for "ll", 0 by default */ etByte done; /* Loop termination flag */ + etByte cThousand; /* Thousands separator for %d and %u */ etByte xtype = etINVALID; /* Conversion paradigm */ u8 bArgList; /* True for SQLITE_PRINTF_SQLFUNC */ char prefix; /* Prefix character. "+" or "-" or " " or '\0'. */ @@ -229,17 +229,18 @@ void sqlite3VXPrintf( break; } /* Find out what flags are present */ - flag_leftjustify = flag_plussign = flag_blanksign = + flag_leftjustify = flag_prefix = cThousand = flag_alternateform = flag_altform2 = flag_zeropad = 0; done = 0; do{ switch( c ){ case '-': flag_leftjustify = 1; break; - case '+': flag_plussign = 1; break; - case ' ': flag_blanksign = 1; break; + case '+': flag_prefix = '+'; break; + case ' ': flag_prefix = ' '; break; case '#': flag_alternateform = 1; break; case '!': flag_altform2 = 1; break; case '0': flag_zeropad = 1; break; + case ',': cThousand = ','; break; default: done = 1; break; } }while( !done && (c=(*++fmt))!=0 ); @@ -309,13 +310,11 @@ void sqlite3VXPrintf( flag_long = 1; c = *++fmt; if( c=='l' ){ - flag_longlong = 1; + flag_long = 2; c = *++fmt; - }else{ - flag_longlong = 0; } }else{ - flag_long = flag_longlong = 0; + flag_long = 0; } /* Fetch the info entry for the field */ infop = &fmtinfo[0]; @@ -333,15 +332,11 @@ void sqlite3VXPrintf( ** ** flag_alternateform TRUE if a '#' is present. ** flag_altform2 TRUE if a '!' is present. - ** flag_plussign TRUE if a '+' is present. + ** flag_prefix '+' or ' ' or zero ** flag_leftjustify TRUE if a '-' is present or if the ** field width was negative. ** flag_zeropad TRUE if the width began with 0. - ** flag_long TRUE if the letter 'l' (ell) prefixed - ** the conversion character. - ** flag_longlong TRUE if the letter 'll' (ell ell) prefixed - ** the conversion character. - ** flag_blanksign TRUE if a ' ' is present. + ** flag_long 1 for "l", 2 for "ll" ** width The specified field width. This is ** always non-negative. Zero is the default. ** precision The specified precision. The default @@ -351,19 +346,24 @@ void sqlite3VXPrintf( */ switch( xtype ){ case etPOINTER: - flag_longlong = sizeof(char*)==sizeof(i64); - flag_long = sizeof(char*)==sizeof(long int); + flag_long = sizeof(char*)==sizeof(i64) ? 2 : + sizeof(char*)==sizeof(long int) ? 1 : 0; /* Fall through into the next case */ case etORDINAL: - case etRADIX: + case etRADIX: + cThousand = 0; + /* Fall through into the next case */ + case etDECIMAL: if( infop->flags & FLAG_SIGNED ){ i64 v; if( bArgList ){ v = getIntArg(pArgList); - }else if( flag_longlong ){ - v = va_arg(ap,i64); }else if( flag_long ){ - v = va_arg(ap,long int); + if( flag_long==2 ){ + v = va_arg(ap,i64) ; + }else{ + v = va_arg(ap,long int); + } }else{ v = va_arg(ap,int); } @@ -376,17 +376,17 @@ void sqlite3VXPrintf( prefix = '-'; }else{ longvalue = v; - if( flag_plussign ) prefix = '+'; - else if( flag_blanksign ) prefix = ' '; - else prefix = 0; + prefix = flag_prefix; } }else{ if( bArgList ){ longvalue = (u64)getIntArg(pArgList); - }else if( flag_longlong ){ - longvalue = va_arg(ap,u64); }else if( flag_long ){ - longvalue = va_arg(ap,unsigned long int); + if( flag_long==2 ){ + longvalue = va_arg(ap,u64); + }else{ + longvalue = va_arg(ap,unsigned long int); + } }else{ longvalue = va_arg(ap,unsigned int); } @@ -396,16 +396,17 @@ void sqlite3VXPrintf( if( flag_zeropad && precision0 ); } length = (int)(&zOut[nOut-1]-bufpt); - for(idx=precision-length; idx>0; idx--){ + while( precision>length ){ *(--bufpt) = '0'; /* Zero pad */ + length++; + } + if( cThousand ){ + int nn = (length - 1)/3; /* Number of "," to insert */ + int ix = (length - 1)%3 + 1; + bufpt -= nn; + for(idx=0; nn>0; idx++){ + bufpt[idx] = bufpt[idx+nn]; + ix--; + if( ix==0 ){ + bufpt[++idx] = cThousand; + nn--; + ix = 3; + } + } } if( prefix ) *(--bufpt) = prefix; /* Add sign */ if( flag_alternateform && infop->prefix ){ /* Add "0" or "0x" */ @@ -454,9 +470,7 @@ void sqlite3VXPrintf( realvalue = -realvalue; prefix = '-'; }else{ - if( flag_plussign ) prefix = '+'; - else if( flag_blanksign ) prefix = ' '; - else prefix = 0; + prefix = flag_prefix; } if( xtype==etGENERIC && precision>0 ) precision--; testcase( precision>0xfff ); diff --git a/src/resolve.c b/src/resolve.c index dac73e5fa9..6b3caa625a 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -15,8 +15,6 @@ ** table and column. */ #include "sqliteInt.h" -#include -#include /* ** Walk the expression tree pExpr and increase the aggregate function @@ -231,7 +229,8 @@ static int lookupName( } /* Start at the inner-most context and move outward until a match is found */ - while( pNC && cnt==0 ){ + assert( pNC && cnt==0 ); + do{ ExprList *pEList; SrcList *pSrcList = pNC->pSrcList; @@ -416,11 +415,11 @@ static int lookupName( /* Advance to the next name context. The loop will exit when either ** we have a match (cnt>0) or when we run out of name contexts. */ - if( cnt==0 ){ - pNC = pNC->pNext; - nSubquery++; - } - } + if( cnt ) break; + pNC = pNC->pNext; + nSubquery++; + }while( pNC ); + /* ** If X and Y are NULL (in other words if only the column name Z is @@ -610,33 +609,38 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ #endif /* defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) */ - /* A lone identifier is the name of a column. - */ - case TK_ID: { - return lookupName(pParse, 0, 0, pExpr->u.zToken, pNC, pExpr); - } - - /* A table name and column name: ID.ID + /* A column name: ID + ** Or table name and column name: ID.ID ** Or a database, table and column: ID.ID.ID + ** + ** The TK_ID and TK_OUT cases are combined so that there will only + ** be one call to lookupName(). Then the compiler will in-line + ** lookupName() for a size reduction and performance increase. */ + case TK_ID: case TK_DOT: { const char *zColumn; const char *zTable; const char *zDb; Expr *pRight; - /* if( pSrcList==0 ) break; */ - notValid(pParse, pNC, "the \".\" operator", NC_IdxExpr); - pRight = pExpr->pRight; - if( pRight->op==TK_ID ){ + if( pExpr->op==TK_ID ){ zDb = 0; - zTable = pExpr->pLeft->u.zToken; - zColumn = pRight->u.zToken; + zTable = 0; + zColumn = pExpr->u.zToken; }else{ - assert( pRight->op==TK_DOT ); - zDb = pExpr->pLeft->u.zToken; - zTable = pRight->pLeft->u.zToken; - zColumn = pRight->pRight->u.zToken; + notValid(pParse, pNC, "the \".\" operator", NC_IdxExpr); + pRight = pExpr->pRight; + if( pRight->op==TK_ID ){ + zDb = 0; + zTable = pExpr->pLeft->u.zToken; + zColumn = pRight->u.zToken; + }else{ + assert( pRight->op==TK_DOT ); + zDb = pExpr->pLeft->u.zToken; + zTable = pRight->pLeft->u.zToken; + zColumn = pRight->pRight->u.zToken; + } } return lookupName(pParse, zDb, zTable, zColumn, pNC, pExpr); } diff --git a/src/select.c b/src/select.c index 75ae082e96..bb055c7894 100644 --- a/src/select.c +++ b/src/select.c @@ -657,6 +657,7 @@ static void codeDistinct( sqlite3VdbeAddOp4Int(v, OP_Found, iTab, addrRepeat, iMem, N); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_MakeRecord, iMem, N, r1); sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iTab, r1, iMem, N); + sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); sqlite3ReleaseTempReg(pParse, r1); } @@ -3748,7 +3749,9 @@ static int flattenSubquery( }else{ pParent->pWhere = sqlite3ExprAnd(db, pWhere, pParent->pWhere); } - substSelect(pParse, pParent, iParent, pSub->pEList, 0); + if( db->mallocFailed==0 ){ + substSelect(pParse, pParent, iParent, pSub->pEList, 0); + } /* The flattened query is distinct if either the inner or the ** outer query is distinct. @@ -4185,7 +4188,15 @@ static int withExpand( pCte->zCteErr = "circular reference: %s"; pSavedWith = pParse->pWith; pParse->pWith = pWith; - sqlite3WalkSelect(pWalker, bMayRecursive ? pSel->pPrior : pSel); + if( bMayRecursive ){ + Select *pPrior = pSel->pPrior; + assert( pPrior->pWith==0 ); + pPrior->pWith = pSel->pWith; + sqlite3WalkSelect(pWalker, pPrior); + pPrior->pWith = 0; + }else{ + sqlite3WalkSelect(pWalker, pSel); + } pParse->pWith = pWith; for(pLeft=pSel; pLeft->pPrior; pLeft=pLeft->pPrior); @@ -4229,10 +4240,12 @@ static int withExpand( */ static void selectPopWith(Walker *pWalker, Select *p){ Parse *pParse = pWalker->pParse; - With *pWith = findRightmost(p)->pWith; - if( pWith!=0 ){ - assert( pParse->pWith==pWith ); - pParse->pWith = pWith->pOuter; + if( pParse->pWith && p->pPrior==0 ){ + With *pWith = findRightmost(p)->pWith; + if( pWith!=0 ){ + assert( pParse->pWith==pWith ); + pParse->pWith = pWith->pOuter; + } } } #else @@ -4282,8 +4295,8 @@ static int selectExpander(Walker *pWalker, Select *p){ } pTabList = p->pSrc; pEList = p->pEList; - if( pWalker->xSelectCallback2==selectPopWith ){ - sqlite3WithPush(pParse, findRightmost(p)->pWith, 0); + if( p->pWith ){ + sqlite3WithPush(pParse, p->pWith, 0); } /* Make sure cursor numbers have been assigned to all entries in @@ -4570,9 +4583,7 @@ static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){ sqlite3WalkSelect(&w, pSelect); } w.xSelectCallback = selectExpander; - if( (pSelect->selFlags & SF_MultiValue)==0 ){ - w.xSelectCallback2 = selectPopWith; - } + w.xSelectCallback2 = selectPopWith; sqlite3WalkSelect(&w, pSelect); } @@ -5651,7 +5662,7 @@ int sqlite3Select( ** of output. */ resetAccumulator(pParse, &sAggInfo); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pMinMax,0,flag,0); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pMinMax, 0,flag,0); if( pWInfo==0 ){ sqlite3ExprListDelete(db, pDel); goto select_end; diff --git a/src/shell.c b/src/shell.c index b0928ce792..e71c34e6df 100644 --- a/src/shell.c +++ b/src/shell.c @@ -455,28 +455,6 @@ static int isNumber(const char *z, int *realnum){ return *z==0; } -/* -** A global char* and an SQL function to access its current value -** from within an SQL statement. This program used to use the -** sqlite_exec_printf() API to substitue a string into an SQL statement. -** The correct way to do this with sqlite3 is to use the bind API, but -** since the shell is built around the callback paradigm it would be a lot -** of work. Instead just use this hack, which is quite harmless. -*/ -static const char *zShellStatic = 0; -static void shellstaticFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - assert( 0==argc ); - assert( zShellStatic ); - UNUSED_PARAMETER(argc); - UNUSED_PARAMETER(argv); - sqlite3_result_text(context, zShellStatic, -1, SQLITE_STATIC); -} - - /* ** Compute a string length that is limited to what can be stored in ** lower 30 bits of a 32-bit signed integer. @@ -577,6 +555,779 @@ static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ } return zResult; } +/* +** A variable length string to which one can append text. +*/ +typedef struct ShellText ShellText; +struct ShellText { + char *z; + int n; + int nAlloc; +}; + +/* +** Initialize and destroy a ShellText object +*/ +static void initText(ShellText *p){ + memset(p, 0, sizeof(*p)); +} +static void freeText(ShellText *p){ + free(p->z); + initText(p); +} + +/* zIn is either a pointer to a NULL-terminated string in memory obtained +** from malloc(), or a NULL pointer. The string pointed to by zAppend is +** added to zIn, and the result returned in memory obtained from malloc(). +** zIn, if it was not NULL, is freed. +** +** If the third argument, quote, is not '\0', then it is used as a +** quote character for zAppend. +*/ +static void appendText(ShellText *p, char const *zAppend, char quote){ + int len; + int i; + int nAppend = strlen30(zAppend); + + len = nAppend+p->n+1; + if( quote ){ + len += 2; + for(i=0; in+len>=p->nAlloc ){ + p->nAlloc = p->nAlloc*2 + len + 20; + p->z = realloc(p->z, p->nAlloc); + if( p->z==0 ){ + memset(p, 0, sizeof(*p)); + return; + } + } + + if( quote ){ + char *zCsr = p->z+p->n; + *zCsr++ = quote; + for(i=0; in = (int)(zCsr - p->z); + *zCsr = '\0'; + }else{ + memcpy(p->z+p->n, zAppend, nAppend); + p->n += nAppend; + p->z[p->n] = '\0'; + } +} + +/* +** Attempt to determine if identifier zName needs to be quoted, either +** because it contains non-alphanumeric characters, or because it is an +** SQLite keyword. Be conservative in this estimate: When in doubt assume +** that quoting is required. +** +** Return '"' if quoting is required. Return 0 if no quoting is required. +*/ +static char quoteChar(const char *zName){ + /* All SQLite keywords, in alphabetical order */ + static const char *azKeywords[] = { + "ABORT", "ACTION", "ADD", "AFTER", "ALL", "ALTER", "ANALYZE", "AND", "AS", + "ASC", "ATTACH", "AUTOINCREMENT", "BEFORE", "BEGIN", "BETWEEN", "BY", + "CASCADE", "CASE", "CAST", "CHECK", "COLLATE", "COLUMN", "COMMIT", + "CONFLICT", "CONSTRAINT", "CREATE", "CROSS", "CURRENT_DATE", + "CURRENT_TIME", "CURRENT_TIMESTAMP", "DATABASE", "DEFAULT", "DEFERRABLE", + "DEFERRED", "DELETE", "DESC", "DETACH", "DISTINCT", "DROP", "EACH", + "ELSE", "END", "ESCAPE", "EXCEPT", "EXCLUSIVE", "EXISTS", "EXPLAIN", + "FAIL", "FOR", "FOREIGN", "FROM", "FULL", "GLOB", "GROUP", "HAVING", "IF", + "IGNORE", "IMMEDIATE", "IN", "INDEX", "INDEXED", "INITIALLY", "INNER", + "INSERT", "INSTEAD", "INTERSECT", "INTO", "IS", "ISNULL", "JOIN", "KEY", + "LEFT", "LIKE", "LIMIT", "MATCH", "NATURAL", "NO", "NOT", "NOTNULL", + "NULL", "OF", "OFFSET", "ON", "OR", "ORDER", "OUTER", "PLAN", "PRAGMA", + "PRIMARY", "QUERY", "RAISE", "RECURSIVE", "REFERENCES", "REGEXP", + "REINDEX", "RELEASE", "RENAME", "REPLACE", "RESTRICT", "RIGHT", + "ROLLBACK", "ROW", "SAVEPOINT", "SELECT", "SET", "TABLE", "TEMP", + "TEMPORARY", "THEN", "TO", "TRANSACTION", "TRIGGER", "UNION", "UNIQUE", + "UPDATE", "USING", "VACUUM", "VALUES", "VIEW", "VIRTUAL", "WHEN", "WHERE", + "WITH", "WITHOUT", + }; + int i, lwr, upr, mid, c; + if( !isalpha((unsigned char)zName[0]) && zName[0]!='_' ) return '"'; + for(i=0; zName[i]; i++){ + if( !isalnum((unsigned char)zName[i]) && zName[i]!='_' ) return '"'; + } + lwr = 0; + upr = sizeof(azKeywords)/sizeof(azKeywords[0]) - 1; + while( lwr<=upr ){ + mid = (lwr+upr)/2; + c = sqlite3_stricmp(azKeywords[mid], zName); + if( c==0 ) return '"'; + if( c<0 ){ + lwr = mid+1; + }else{ + upr = mid-1; + } + } + return 0; +} + +/****************************************************************************** +** SHA3 hash implementation copied from ../ext/misc/shathree.c +*/ +typedef sqlite3_uint64 u64; +/* +** Macros to determine whether the machine is big or little endian, +** and whether or not that determination is run-time or compile-time. +** +** For best performance, an attempt is made to guess at the byte-order +** using C-preprocessor macros. If that is unsuccessful, or if +** -DSHA3_BYTEORDER=0 is set, then byte-order is determined +** at run-time. +*/ +#ifndef SHA3_BYTEORDER +# if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ + defined(__arm__) +# define SHA3_BYTEORDER 1234 +# elif defined(sparc) || defined(__ppc__) +# define SHA3_BYTEORDER 4321 +# else +# define SHA3_BYTEORDER 0 +# endif +#endif + + +/* +** State structure for a SHA3 hash in progress +*/ +typedef struct SHA3Context SHA3Context; +struct SHA3Context { + union { + u64 s[25]; /* Keccak state. 5x5 lines of 64 bits each */ + unsigned char x[1600]; /* ... or 1600 bytes */ + } u; + unsigned nRate; /* Bytes of input accepted per Keccak iteration */ + unsigned nLoaded; /* Input bytes loaded into u.x[] so far this cycle */ + unsigned ixMask; /* Insert next input into u.x[nLoaded^ixMask]. */ +}; + +/* +** A single step of the Keccak mixing function for a 1600-bit state +*/ +static void KeccakF1600Step(SHA3Context *p){ + int i; + u64 B0, B1, B2, B3, B4; + u64 C0, C1, C2, C3, C4; + u64 D0, D1, D2, D3, D4; + static const u64 RC[] = { + 0x0000000000000001ULL, 0x0000000000008082ULL, + 0x800000000000808aULL, 0x8000000080008000ULL, + 0x000000000000808bULL, 0x0000000080000001ULL, + 0x8000000080008081ULL, 0x8000000000008009ULL, + 0x000000000000008aULL, 0x0000000000000088ULL, + 0x0000000080008009ULL, 0x000000008000000aULL, + 0x000000008000808bULL, 0x800000000000008bULL, + 0x8000000000008089ULL, 0x8000000000008003ULL, + 0x8000000000008002ULL, 0x8000000000000080ULL, + 0x000000000000800aULL, 0x800000008000000aULL, + 0x8000000080008081ULL, 0x8000000000008080ULL, + 0x0000000080000001ULL, 0x8000000080008008ULL + }; +# define A00 (p->u.s[0]) +# define A01 (p->u.s[1]) +# define A02 (p->u.s[2]) +# define A03 (p->u.s[3]) +# define A04 (p->u.s[4]) +# define A10 (p->u.s[5]) +# define A11 (p->u.s[6]) +# define A12 (p->u.s[7]) +# define A13 (p->u.s[8]) +# define A14 (p->u.s[9]) +# define A20 (p->u.s[10]) +# define A21 (p->u.s[11]) +# define A22 (p->u.s[12]) +# define A23 (p->u.s[13]) +# define A24 (p->u.s[14]) +# define A30 (p->u.s[15]) +# define A31 (p->u.s[16]) +# define A32 (p->u.s[17]) +# define A33 (p->u.s[18]) +# define A34 (p->u.s[19]) +# define A40 (p->u.s[20]) +# define A41 (p->u.s[21]) +# define A42 (p->u.s[22]) +# define A43 (p->u.s[23]) +# define A44 (p->u.s[24]) +# define ROL64(a,x) ((a<>(64-x))) + + for(i=0; i<24; i+=4){ + C0 = A00^A10^A20^A30^A40; + C1 = A01^A11^A21^A31^A41; + C2 = A02^A12^A22^A32^A42; + C3 = A03^A13^A23^A33^A43; + C4 = A04^A14^A24^A34^A44; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A11^D1), 44); + B2 = ROL64((A22^D2), 43); + B3 = ROL64((A33^D3), 21); + B4 = ROL64((A44^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i]; + A11 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A20^D0), 3); + B3 = ROL64((A31^D1), 45); + B4 = ROL64((A42^D2), 61); + B0 = ROL64((A03^D3), 28); + B1 = ROL64((A14^D4), 20); + A20 = B0 ^((~B1)& B2 ); + A31 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A40^D0), 18); + B0 = ROL64((A01^D1), 1); + B1 = ROL64((A12^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A34^D4), 8); + A40 = B0 ^((~B1)& B2 ); + A01 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A10^D0), 36); + B2 = ROL64((A21^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A43^D3), 56); + B0 = ROL64((A04^D4), 27); + A10 = B0 ^((~B1)& B2 ); + A21 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A30^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A02^D2), 62); + B1 = ROL64((A13^D3), 55); + B2 = ROL64((A24^D4), 39); + A30 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + C0 = A00^A20^A40^A10^A30; + C1 = A11^A31^A01^A21^A41; + C2 = A22^A42^A12^A32^A02; + C3 = A33^A03^A23^A43^A13; + C4 = A44^A14^A34^A04^A24; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A31^D1), 44); + B2 = ROL64((A12^D2), 43); + B3 = ROL64((A43^D3), 21); + B4 = ROL64((A24^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i+1]; + A31 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A40^D0), 3); + B3 = ROL64((A21^D1), 45); + B4 = ROL64((A02^D2), 61); + B0 = ROL64((A33^D3), 28); + B1 = ROL64((A14^D4), 20); + A40 = B0 ^((~B1)& B2 ); + A21 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A30^D0), 18); + B0 = ROL64((A11^D1), 1); + B1 = ROL64((A42^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A04^D4), 8); + A30 = B0 ^((~B1)& B2 ); + A11 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A20^D0), 36); + B2 = ROL64((A01^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A13^D3), 56); + B0 = ROL64((A44^D4), 27); + A20 = B0 ^((~B1)& B2 ); + A01 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A10^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A22^D2), 62); + B1 = ROL64((A03^D3), 55); + B2 = ROL64((A34^D4), 39); + A10 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + C0 = A00^A40^A30^A20^A10; + C1 = A31^A21^A11^A01^A41; + C2 = A12^A02^A42^A32^A22; + C3 = A43^A33^A23^A13^A03; + C4 = A24^A14^A04^A44^A34; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A21^D1), 44); + B2 = ROL64((A42^D2), 43); + B3 = ROL64((A13^D3), 21); + B4 = ROL64((A34^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i+2]; + A21 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A30^D0), 3); + B3 = ROL64((A01^D1), 45); + B4 = ROL64((A22^D2), 61); + B0 = ROL64((A43^D3), 28); + B1 = ROL64((A14^D4), 20); + A30 = B0 ^((~B1)& B2 ); + A01 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A10^D0), 18); + B0 = ROL64((A31^D1), 1); + B1 = ROL64((A02^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A44^D4), 8); + A10 = B0 ^((~B1)& B2 ); + A31 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A40^D0), 36); + B2 = ROL64((A11^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A03^D3), 56); + B0 = ROL64((A24^D4), 27); + A40 = B0 ^((~B1)& B2 ); + A11 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A20^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A12^D2), 62); + B1 = ROL64((A33^D3), 55); + B2 = ROL64((A04^D4), 39); + A20 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + C0 = A00^A30^A10^A40^A20; + C1 = A21^A01^A31^A11^A41; + C2 = A42^A22^A02^A32^A12; + C3 = A13^A43^A23^A03^A33; + C4 = A34^A14^A44^A24^A04; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A01^D1), 44); + B2 = ROL64((A02^D2), 43); + B3 = ROL64((A03^D3), 21); + B4 = ROL64((A04^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i+3]; + A01 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A10^D0), 3); + B3 = ROL64((A11^D1), 45); + B4 = ROL64((A12^D2), 61); + B0 = ROL64((A13^D3), 28); + B1 = ROL64((A14^D4), 20); + A10 = B0 ^((~B1)& B2 ); + A11 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A20^D0), 18); + B0 = ROL64((A21^D1), 1); + B1 = ROL64((A22^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A24^D4), 8); + A20 = B0 ^((~B1)& B2 ); + A21 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A30^D0), 36); + B2 = ROL64((A31^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A33^D3), 56); + B0 = ROL64((A34^D4), 27); + A30 = B0 ^((~B1)& B2 ); + A31 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A40^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A42^D2), 62); + B1 = ROL64((A43^D3), 55); + B2 = ROL64((A44^D4), 39); + A40 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + } +} + +/* +** Initialize a new hash. iSize determines the size of the hash +** in bits and should be one of 224, 256, 384, or 512. Or iSize +** can be zero to use the default hash size of 256 bits. +*/ +static void SHA3Init(SHA3Context *p, int iSize){ + memset(p, 0, sizeof(*p)); + if( iSize>=128 && iSize<=512 ){ + p->nRate = (1600 - ((iSize + 31)&~31)*2)/8; + }else{ + p->nRate = (1600 - 2*256)/8; + } +#if SHA3_BYTEORDER==1234 + /* Known to be little-endian at compile-time. No-op */ +#elif SHA3_BYTEORDER==4321 + p->ixMask = 7; /* Big-endian */ +#else + { + static unsigned int one = 1; + if( 1==*(unsigned char*)&one ){ + /* Little endian. No byte swapping. */ + p->ixMask = 0; + }else{ + /* Big endian. Byte swap. */ + p->ixMask = 7; + } + } +#endif +} + +/* +** Make consecutive calls to the SHA3Update function to add new content +** to the hash +*/ +static void SHA3Update( + SHA3Context *p, + const unsigned char *aData, + unsigned int nData +){ + unsigned int i = 0; +#if SHA3_BYTEORDER==1234 + if( (p->nLoaded % 8)==0 && ((aData - (const unsigned char*)0)&7)==0 ){ + for(; i+7u.s[p->nLoaded/8] ^= *(u64*)&aData[i]; + p->nLoaded += 8; + if( p->nLoaded>=p->nRate ){ + KeccakF1600Step(p); + p->nLoaded = 0; + } + } + } +#endif + for(; iu.x[p->nLoaded] ^= aData[i]; +#elif SHA3_BYTEORDER==4321 + p->u.x[p->nLoaded^0x07] ^= aData[i]; +#else + p->u.x[p->nLoaded^p->ixMask] ^= aData[i]; +#endif + p->nLoaded++; + if( p->nLoaded==p->nRate ){ + KeccakF1600Step(p); + p->nLoaded = 0; + } + } +} + +/* +** After all content has been added, invoke SHA3Final() to compute +** the final hash. The function returns a pointer to the binary +** hash value. +*/ +static unsigned char *SHA3Final(SHA3Context *p){ + unsigned int i; + if( p->nLoaded==p->nRate-1 ){ + const unsigned char c1 = 0x86; + SHA3Update(p, &c1, 1); + }else{ + const unsigned char c2 = 0x06; + const unsigned char c3 = 0x80; + SHA3Update(p, &c2, 1); + p->nLoaded = p->nRate - 1; + SHA3Update(p, &c3, 1); + } + for(i=0; inRate; i++){ + p->u.x[i+p->nRate] = p->u.x[i^p->ixMask]; + } + return &p->u.x[p->nRate]; +} + +/* +** Implementation of the sha3(X,SIZE) function. +** +** Return a BLOB which is the SIZE-bit SHA3 hash of X. The default +** size is 256. If X is a BLOB, it is hashed as is. +** For all other non-NULL types of input, X is converted into a UTF-8 string +** and the string is hashed without the trailing 0x00 terminator. The hash +** of a NULL value is NULL. +*/ +static void sha3Func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + SHA3Context cx; + int eType = sqlite3_value_type(argv[0]); + int nByte = sqlite3_value_bytes(argv[0]); + int iSize; + if( argc==1 ){ + iSize = 256; + }else{ + iSize = sqlite3_value_int(argv[1]); + if( iSize!=224 && iSize!=256 && iSize!=384 && iSize!=512 ){ + sqlite3_result_error(context, "SHA3 size should be one of: 224 256 " + "384 512", -1); + return; + } + } + if( eType==SQLITE_NULL ) return; + SHA3Init(&cx, iSize); + if( eType==SQLITE_BLOB ){ + SHA3Update(&cx, sqlite3_value_blob(argv[0]), nByte); + }else{ + SHA3Update(&cx, sqlite3_value_text(argv[0]), nByte); + } + sqlite3_result_blob(context, SHA3Final(&cx), iSize/8, SQLITE_TRANSIENT); +} + +/* Compute a string using sqlite3_vsnprintf() with a maximum length +** of 50 bytes and add it to the hash. +*/ +static void hash_step_vformat( + SHA3Context *p, /* Add content to this context */ + const char *zFormat, + ... +){ + va_list ap; + int n; + char zBuf[50]; + va_start(ap, zFormat); + sqlite3_vsnprintf(sizeof(zBuf),zBuf,zFormat,ap); + va_end(ap); + n = (int)strlen(zBuf); + SHA3Update(p, (unsigned char*)zBuf, n); +} + +/* +** Implementation of the sha3_query(SQL,SIZE) function. +** +** This function compiles and runs the SQL statement(s) given in the +** argument. The results are hashed using a SIZE-bit SHA3. The default +** size is 256. +** +** The format of the byte stream that is hashed is summarized as follows: +** +** S: +** R +** N +** I +** F +** B: +** T: +** +** is the original SQL text for each statement run and is +** the size of that text. The SQL text is UTF-8. A single R character +** occurs before the start of each row. N means a NULL value. +** I mean an 8-byte little-endian integer . F is a floating point +** number with an 8-byte little-endian IEEE floating point value . +** B means blobs of bytes. T means text rendered as +** bytes of UTF-8. The and values are expressed as an ASCII +** text integers. +** +** For each SQL statement in the X input, there is one S segment. Each +** S segment is followed by zero or more R segments, one for each row in the +** result set. After each R, there are one or more N, I, F, B, or T segments, +** one for each column in the result set. Segments are concatentated directly +** with no delimiters of any kind. +*/ +static void sha3QueryFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + const char *zSql = (const char*)sqlite3_value_text(argv[0]); + sqlite3_stmt *pStmt = 0; + int nCol; /* Number of columns in the result set */ + int i; /* Loop counter */ + int rc; + int n; + const char *z; + SHA3Context cx; + int iSize; + + if( argc==1 ){ + iSize = 256; + }else{ + iSize = sqlite3_value_int(argv[1]); + if( iSize!=224 && iSize!=256 && iSize!=384 && iSize!=512 ){ + sqlite3_result_error(context, "SHA3 size should be one of: 224 256 " + "384 512", -1); + return; + } + } + if( zSql==0 ) return; + SHA3Init(&cx, iSize); + while( zSql[0] ){ + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zSql); + if( rc ){ + char *zMsg = sqlite3_mprintf("error SQL statement [%s]: %s", + zSql, sqlite3_errmsg(db)); + sqlite3_finalize(pStmt); + sqlite3_result_error(context, zMsg, -1); + sqlite3_free(zMsg); + return; + } + if( !sqlite3_stmt_readonly(pStmt) ){ + char *zMsg = sqlite3_mprintf("non-query: [%s]", sqlite3_sql(pStmt)); + sqlite3_finalize(pStmt); + sqlite3_result_error(context, zMsg, -1); + sqlite3_free(zMsg); + return; + } + nCol = sqlite3_column_count(pStmt); + z = sqlite3_sql(pStmt); + if( z==0 ){ + sqlite3_finalize(pStmt); + continue; + } + n = (int)strlen(z); + hash_step_vformat(&cx,"S%d:",n); + SHA3Update(&cx,(unsigned char*)z,n); + + /* Compute a hash over the result of the query */ + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + SHA3Update(&cx,(const unsigned char*)"R",1); + for(i=0; i=1; j--){ + x[j] = u & 0xff; + u >>= 8; + } + x[0] = 'I'; + SHA3Update(&cx, x, 9); + break; + } + case SQLITE_FLOAT: { + sqlite3_uint64 u; + int j; + unsigned char x[9]; + double r = sqlite3_column_double(pStmt,i); + memcpy(&u, &r, 8); + for(j=8; j>=1; j--){ + x[j] = u & 0xff; + u >>= 8; + } + x[0] = 'F'; + SHA3Update(&cx,x,9); + break; + } + case SQLITE_TEXT: { + int n2 = sqlite3_column_bytes(pStmt, i); + const unsigned char *z2 = sqlite3_column_text(pStmt, i); + hash_step_vformat(&cx,"T%d:",n2); + SHA3Update(&cx, z2, n2); + break; + } + case SQLITE_BLOB: { + int n2 = sqlite3_column_bytes(pStmt, i); + const unsigned char *z2 = sqlite3_column_blob(pStmt, i); + hash_step_vformat(&cx,"B%d:",n2); + SHA3Update(&cx, z2, n2); + break; + } + } + } + } + sqlite3_finalize(pStmt); + } + sqlite3_result_blob(context, SHA3Final(&cx), iSize/8, SQLITE_TRANSIENT); +} +/* End of SHA3 hashing logic copy/pasted from ../ext/misc/shathree.c +********************************************************************************/ #if defined(SQLITE_ENABLE_SESSION) /* @@ -610,13 +1361,10 @@ struct SavedModeInfo { typedef struct ShellState ShellState; struct ShellState { sqlite3 *db; /* The database */ - int echoOn; /* True to echo input commands */ int autoExplain; /* Automatically turn on .explain mode */ int autoEQP; /* Run EXPLAIN QUERY PLAN prior to seach SQL stmt */ int statsOn; /* True to display memory stats before each finalize */ int scanstatsOn; /* True to display scan stats before each finalize */ - int countChanges; /* True to display change counts */ - int backslashOn; /* Resolve C-style \x escapes in SQL input text */ int outCount; /* Revert to stdout when reaching zero */ int cnt; /* Number of records displayed so far */ FILE *out; /* Write results here */ @@ -655,9 +1403,20 @@ struct ShellState { /* ** These are the allowed shellFlgs values */ -#define SHFLG_Scratch 0x00001 /* The --scratch option is used */ -#define SHFLG_Pagecache 0x00002 /* The --pagecache option is used */ -#define SHFLG_Lookaside 0x00004 /* Lookaside memory is used */ +#define SHFLG_Scratch 0x00000001 /* The --scratch option is used */ +#define SHFLG_Pagecache 0x00000002 /* The --pagecache option is used */ +#define SHFLG_Lookaside 0x00000004 /* Lookaside memory is used */ +#define SHFLG_Backslash 0x00000008 /* The --backslash option is used */ +#define SHFLG_PreserveRowid 0x00000010 /* .dump preserves rowid values */ +#define SHFLG_CountChanges 0x00000020 /* .changes setting */ +#define SHFLG_Echo 0x00000040 /* .echo or --echo setting */ + +/* +** Macros for testing and setting shellFlgs +*/ +#define ShellHasFlag(P,X) (((P)->shellFlgs & (X))!=0) +#define ShellSetFlag(P,X) ((P)->shellFlgs|=(X)) +#define ShellClearFlag(P,X) ((P)->shellFlgs&=(~(X))) /* ** These are the allowed modes. @@ -731,32 +1490,52 @@ static void output_hex_blob(FILE *out, const void *pBlob, int nBlob){ /* ** Output the given string as a quoted string using SQL quoting conventions. +** +** The "\n" and "\r" characters are converted to char(10) and char(13) +** to prevent them from being transformed by end-of-line translators. */ static void output_quoted_string(FILE *out, const char *z){ int i; - int nSingle = 0; + char c; setBinaryMode(out, 1); - for(i=0; z[i]; i++){ - if( z[i]=='\'' ) nSingle++; - } - if( nSingle==0 ){ + for(i=0; (c = z[i])!=0 && c!='\'' && c!='\n' && c!='\r'; i++){} + if( c==0 ){ utf8_printf(out,"'%s'",z); }else{ - raw_printf(out,"'"); + int inQuote = 0; + int bStarted = 0; while( *z ){ - for(i=0; z[i] && z[i]!='\''; i++){} - if( i==0 ){ - raw_printf(out,"''"); - z++; - }else if( z[i]=='\'' ){ - utf8_printf(out,"%.*s''",i,z); - z += i+1; - }else{ - utf8_printf(out,"%s",z); + for(i=0; (c = z[i])!=0 && c!='\n' && c!='\r' && c!='\''; i++){} + if( c=='\'' ) i++; + if( i ){ + if( !inQuote ){ + if( bStarted ) raw_printf(out, "||"); + raw_printf(out, "'"); + inQuote = 1; + } + utf8_printf(out, "%.*s", i, z); + z += i; + bStarted = 1; + } + if( c=='\'' ){ + raw_printf(out, "'"); + continue; + } + if( inQuote ){ + raw_printf(out, "'"); + inQuote = 0; + } + if( c==0 ){ break; } + for(i=0; (c = z[i])=='\r' || c=='\n'; i++){ + if( bStarted ) raw_printf(out, "||"); + raw_printf(out, "char(%d)", c); + bStarted = 1; + } + z += i; } - raw_printf(out,"'"); + if( inQuote ) raw_printf(out, "'"); } setTextMode(out, 1); } @@ -1248,9 +2027,13 @@ static int shell_callback( }else if( aiType && aiType[i]==SQLITE_TEXT ){ if( zSep[0] ) utf8_printf(p->out,"%s",zSep); output_quoted_string(p->out, azArg[i]); - }else if( aiType && (aiType[i]==SQLITE_INTEGER - || aiType[i]==SQLITE_FLOAT) ){ + }else if( aiType && aiType[i]==SQLITE_INTEGER ){ utf8_printf(p->out,"%s%s",zSep, azArg[i]); + }else if( aiType && aiType[i]==SQLITE_FLOAT ){ + char z[50]; + double r = sqlite3_column_double(p->pStmt, i); + sqlite3_snprintf(50,z,"%!.20g", r); + raw_printf(p->out, "%s%s", zSep, z); }else if( aiType && aiType[i]==SQLITE_BLOB && p->pStmt ){ const void *pBlob = sqlite3_column_blob(p->pStmt, i); int nBlob = sqlite3_column_bytes(p->pStmt, i); @@ -1295,6 +2078,70 @@ static int callback(void *pArg, int nArg, char **azArg, char **azCol){ return shell_callback(pArg, nArg, azArg, azCol, NULL); } +/* +** This is the callback routine from sqlite3_exec() that appends all +** output onto the end of a ShellText object. +*/ +static int captureOutputCallback(void *pArg, int nArg, char **azArg, char **az){ + ShellText *p = (ShellText*)pArg; + int i; + if( p->n ) appendText(p, "|", 0); + for(i=0; idb, + "SAVEPOINT selftest_init;\n" + "CREATE TABLE IF NOT EXISTS selftest(\n" + " tno INTEGER PRIMARY KEY,\n" /* Test number */ + " op TEXT,\n" /* Operator: memo run */ + " cmd TEXT,\n" /* Command text */ + " ans TEXT\n" /* Desired answer */ + ");" + "CREATE TEMP TABLE [_shell$self](op,cmd,ans);\n" + "INSERT INTO [_shell$self](rowid,op,cmd)\n" + " VALUES(coalesce((SELECT (max(tno)+100)/10 FROM selftest),10),\n" + " 'memo','Tests generated by --init');\n" + "INSERT INTO [_shell$self]\n" + " SELECT 'run',\n" + " 'SELECT hex(sha3_query(''SELECT type,name,tbl_name,sql " + "FROM sqlite_master ORDER BY 2'',224))',\n" + " hex(sha3_query('SELECT type,name,tbl_name,sql " + "FROM sqlite_master ORDER BY 2',224));\n" + "INSERT INTO [_shell$self]\n" + " SELECT 'run'," + " 'SELECT hex(sha3_query(''SELECT * FROM \"' ||" + " printf('%w',name) || '\" NOT INDEXED'',224))',\n" + " hex(sha3_query(printf('SELECT * FROM \"%w\" NOT INDEXED',name),224))\n" + " FROM (\n" + " SELECT name FROM sqlite_master\n" + " WHERE type='table'\n" + " AND name<>'selftest'\n" + " AND coalesce(rootpage,0)>0\n" + " )\n" + " ORDER BY name;\n" + "INSERT INTO [_shell$self]\n" + " VALUES('run','PRAGMA integrity_check','ok');\n" + "INSERT INTO selftest(tno,op,cmd,ans)" + " SELECT rowid*10,op,cmd,ans FROM [_shell$self];\n" + "DROP TABLE [_shell$self];" + ,0,0,&zErrMsg); + if( zErrMsg ){ + utf8_printf(stderr, "SELFTEST initialization failure: %s\n", zErrMsg); + sqlite3_free(zErrMsg); + } + sqlite3_exec(p->db, "RELEASE selftest_init",0,0,0); +} + + /* ** Set the destination table field of the ShellState structure to ** the name of the table given. Escape any quote characters in the @@ -1302,7 +2149,7 @@ static int callback(void *pArg, int nArg, char **azArg, char **azCol){ */ static void set_table_name(ShellState *p, const char *zName){ int i, n; - int needQuote; + int cQuote; char *z; if( p->zDestTable ){ @@ -1310,74 +2157,24 @@ static void set_table_name(ShellState *p, const char *zName){ p->zDestTable = 0; } if( zName==0 ) return; - needQuote = !isalpha((unsigned char)*zName) && *zName!='_'; - for(i=n=0; zName[i]; i++, n++){ - if( !isalnum((unsigned char)zName[i]) && zName[i]!='_' ){ - needQuote = 1; - if( zName[i]=='\'' ) n++; - } - } - if( needQuote ) n += 2; + cQuote = quoteChar(zName); + n = strlen30(zName); + if( cQuote ) n += 2; z = p->zDestTable = malloc( n+1 ); if( z==0 ){ raw_printf(stderr,"Error: out of memory\n"); exit(1); } n = 0; - if( needQuote ) z[n++] = '\''; + if( cQuote ) z[n++] = cQuote; for(i=0; zName[i]; i++){ z[n++] = zName[i]; - if( zName[i]=='\'' ) z[n++] = '\''; + if( zName[i]==cQuote ) z[n++] = cQuote; } - if( needQuote ) z[n++] = '\''; + if( cQuote ) z[n++] = cQuote; z[n] = 0; } -/* zIn is either a pointer to a NULL-terminated string in memory obtained -** from malloc(), or a NULL pointer. The string pointed to by zAppend is -** added to zIn, and the result returned in memory obtained from malloc(). -** zIn, if it was not NULL, is freed. -** -** If the third argument, quote, is not '\0', then it is used as a -** quote character for zAppend. -*/ -static char *appendText(char *zIn, char const *zAppend, char quote){ - int len; - int i; - int nAppend = strlen30(zAppend); - int nIn = (zIn?strlen30(zIn):0); - - len = nAppend+nIn+1; - if( quote ){ - len += 2; - for(i=0; i1 ){ + sqlite3_snprintf(sizeof(zLine), zLine, zFormat, iCur, iHiwtr); + }else{ + sqlite3_snprintf(sizeof(zLine), zLine, zFormat, iHiwtr); + } + raw_printf(p->out, "%-36s %s\n", zLabel, zLine); +} /* ** Display memory stats. @@ -1499,57 +2321,31 @@ static int display_stats( int iHiwtr; if( pArg && pArg->out ){ - - iHiwtr = iCur = -1; - sqlite3_status(SQLITE_STATUS_MEMORY_USED, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, - "Memory Used: %d (max %d) bytes\n", - iCur, iHiwtr); - iHiwtr = iCur = -1; - sqlite3_status(SQLITE_STATUS_MALLOC_COUNT, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, "Number of Outstanding Allocations: %d (max %d)\n", - iCur, iHiwtr); + displayStatLine(pArg, "Memory Used:", + "%lld (max %lld) bytes", SQLITE_STATUS_MEMORY_USED, bReset); + displayStatLine(pArg, "Number of Outstanding Allocations:", + "%lld (max %lld)", SQLITE_STATUS_MALLOC_COUNT, bReset); if( pArg->shellFlgs & SHFLG_Pagecache ){ - iHiwtr = iCur = -1; - sqlite3_status(SQLITE_STATUS_PAGECACHE_USED, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, - "Number of Pcache Pages Used: %d (max %d) pages\n", - iCur, iHiwtr); + displayStatLine(pArg, "Number of Pcache Pages Used:", + "%lld (max %lld) pages", SQLITE_STATUS_PAGECACHE_USED, bReset); } - iHiwtr = iCur = -1; - sqlite3_status(SQLITE_STATUS_PAGECACHE_OVERFLOW, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, - "Number of Pcache Overflow Bytes: %d (max %d) bytes\n", - iCur, iHiwtr); + displayStatLine(pArg, "Number of Pcache Overflow Bytes:", + "%lld (max %lld) bytes", SQLITE_STATUS_PAGECACHE_OVERFLOW, bReset); if( pArg->shellFlgs & SHFLG_Scratch ){ - iHiwtr = iCur = -1; - sqlite3_status(SQLITE_STATUS_SCRATCH_USED, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, - "Number of Scratch Allocations Used: %d (max %d)\n", - iCur, iHiwtr); + displayStatLine(pArg, "Number of Scratch Allocations Used:", + "%lld (max %lld)", SQLITE_STATUS_SCRATCH_USED, bReset); } - iHiwtr = iCur = -1; - sqlite3_status(SQLITE_STATUS_SCRATCH_OVERFLOW, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, - "Number of Scratch Overflow Bytes: %d (max %d) bytes\n", - iCur, iHiwtr); - iHiwtr = iCur = -1; - sqlite3_status(SQLITE_STATUS_MALLOC_SIZE, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, "Largest Allocation: %d bytes\n", - iHiwtr); - iHiwtr = iCur = -1; - sqlite3_status(SQLITE_STATUS_PAGECACHE_SIZE, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, "Largest Pcache Allocation: %d bytes\n", - iHiwtr); - iHiwtr = iCur = -1; - sqlite3_status(SQLITE_STATUS_SCRATCH_SIZE, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, "Largest Scratch Allocation: %d bytes\n", - iHiwtr); + displayStatLine(pArg, "Number of Scratch Overflow Bytes:", + "%lld (max %lld) bytes", SQLITE_STATUS_SCRATCH_OVERFLOW, bReset); + displayStatLine(pArg, "Largest Allocation:", + "%lld bytes", SQLITE_STATUS_MALLOC_SIZE, bReset); + displayStatLine(pArg, "Largest Pcache Allocation:", + "%lld bytes", SQLITE_STATUS_PAGECACHE_SIZE, bReset); + displayStatLine(pArg, "Largest Scratch Allocation:", + "%lld bytes", SQLITE_STATUS_SCRATCH_SIZE, bReset); #ifdef YYTRACKMAXSTACKDEPTH - iHiwtr = iCur = -1; - sqlite3_status(SQLITE_STATUS_PARSER_STACK, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, "Deepest Parser Stack: %d (max %d)\n", - iCur, iHiwtr); + displayStatLine(pArg, "Deepest Parser Stack:", + "%lld (max %lld)", SQLITE_STATUS_PARSER_STACK, bReset); #endif } @@ -1936,7 +2732,7 @@ static int shell_exec( } /* echo the sql statement if echo on */ - if( pArg && pArg->echoOn ){ + if( pArg && ShellHasFlag(pArg, SHFLG_Echo) ){ utf8_printf(pArg->out, "%s\n", zStmtSql ? zStmtSql : zSql); } @@ -2024,6 +2820,137 @@ static int shell_exec( return rc; } +/* +** Release memory previously allocated by tableColumnList(). +*/ +static void freeColumnList(char **azCol){ + int i; + for(i=1; azCol[i]; i++){ + sqlite3_free(azCol[i]); + } + /* azCol[0] is a static string */ + sqlite3_free(azCol); +} + +/* +** Return a list of pointers to strings which are the names of all +** columns in table zTab. The memory to hold the names is dynamically +** allocated and must be released by the caller using a subsequent call +** to freeColumnList(). +** +** The azCol[0] entry is usually NULL. However, if zTab contains a rowid +** value that needs to be preserved, then azCol[0] is filled in with the +** name of the rowid column. +** +** The first regular column in the table is azCol[1]. The list is terminated +** by an entry with azCol[i]==0. +*/ +static char **tableColumnList(ShellState *p, const char *zTab){ + char **azCol = 0; + sqlite3_stmt *pStmt; + char *zSql; + int nCol = 0; + int nAlloc = 0; + int nPK = 0; /* Number of PRIMARY KEY columns seen */ + int isIPK = 0; /* True if one PRIMARY KEY column of type INTEGER */ + int preserveRowid = ShellHasFlag(p, SHFLG_PreserveRowid); + int rc; + + zSql = sqlite3_mprintf("PRAGMA table_info=%Q", zTab); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc ) return 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + if( nCol>=nAlloc-2 ){ + nAlloc = nAlloc*2 + nCol + 10; + azCol = sqlite3_realloc(azCol, nAlloc*sizeof(azCol[0])); + if( azCol==0 ){ + raw_printf(stderr, "Error: out of memory\n"); + exit(1); + } + } + azCol[++nCol] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1)); + if( sqlite3_column_int(pStmt, 5) ){ + nPK++; + if( nPK==1 + && sqlite3_stricmp((const char*)sqlite3_column_text(pStmt,2), + "INTEGER")==0 + ){ + isIPK = 1; + }else{ + isIPK = 0; + } + } + } + sqlite3_finalize(pStmt); + azCol[0] = 0; + azCol[nCol+1] = 0; + + /* The decision of whether or not a rowid really needs to be preserved + ** is tricky. We never need to preserve a rowid for a WITHOUT ROWID table + ** or a table with an INTEGER PRIMARY KEY. We are unable to preserve + ** rowids on tables where the rowid is inaccessible because there are other + ** columns in the table named "rowid", "_rowid_", and "oid". + */ + if( preserveRowid && isIPK ){ + /* If a single PRIMARY KEY column with type INTEGER was seen, then it + ** might be an alise for the ROWID. But it might also be a WITHOUT ROWID + ** table or a INTEGER PRIMARY KEY DESC column, neither of which are + ** ROWID aliases. To distinguish these cases, check to see if + ** there is a "pk" entry in "PRAGMA index_list". There will be + ** no "pk" index if the PRIMARY KEY really is an alias for the ROWID. + */ + zSql = sqlite3_mprintf("SELECT 1 FROM pragma_index_list(%Q)" + " WHERE origin='pk'", zTab); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc ){ + freeColumnList(azCol); + return 0; + } + rc = sqlite3_step(pStmt); + sqlite3_finalize(pStmt); + preserveRowid = rc==SQLITE_ROW; + } + if( preserveRowid ){ + /* Only preserve the rowid if we can find a name to use for the + ** rowid */ + static char *azRowid[] = { "rowid", "_rowid_", "oid" }; + int i, j; + for(j=0; j<3; j++){ + for(i=1; i<=nCol; i++){ + if( sqlite3_stricmp(azRowid[j],azCol[i])==0 ) break; + } + if( i>nCol ){ + /* At this point, we know that azRowid[j] is not the name of any + ** ordinary column in the table. Verify that azRowid[j] is a valid + ** name for the rowid before adding it to azCol[0]. WITHOUT ROWID + ** tables will fail this last check */ + rc = sqlite3_table_column_metadata(p->db,0,zTab,azRowid[j],0,0,0,0,0); + if( rc==SQLITE_OK ) azCol[0] = azRowid[j]; + break; + } + } + } + return azCol; +} + +/* +** Toggle the reverse_unordered_selects setting. +*/ +static void toggleSelectOrder(sqlite3 *db){ + sqlite3_stmt *pStmt = 0; + int iSetting = 0; + char zStmt[100]; + sqlite3_prepare_v2(db, "PRAGMA reverse_unordered_selects", -1, &pStmt, 0); + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + iSetting = sqlite3_column_int(pStmt, 0); + } + sqlite3_finalize(pStmt); + sqlite3_snprintf(sizeof(zStmt), zStmt, + "PRAGMA reverse_unordered_selects(%d)", !iSetting); + sqlite3_exec(db, zStmt, 0, 0, 0); +} /* ** This is a different callback routine used for dumping the database. @@ -2031,22 +2958,21 @@ static int shell_exec( ** the table type ("index" or "table") and SQL to create the table. ** This routine should print text sufficient to recreate the table. */ -static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ +static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){ int rc; const char *zTable; const char *zType; const char *zSql; - const char *zPrepStmt = 0; ShellState *p = (ShellState *)pArg; - UNUSED_PARAMETER(azCol); + UNUSED_PARAMETER(azNotUsed); if( nArg!=3 ) return 1; zTable = azArg[0]; zType = azArg[1]; zSql = azArg[2]; if( strcmp(zTable, "sqlite_sequence")==0 ){ - zPrepStmt = "DELETE FROM sqlite_sequence;\n"; + raw_printf(p->out, "DELETE FROM sqlite_sequence;\n"); }else if( sqlite3_strglob("sqlite_stat?", zTable)==0 ){ raw_printf(p->out, "ANALYZE sqlite_master;\n"); }else if( strncmp(zTable, "sqlite_", 7)==0 ){ @@ -2069,58 +2995,70 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ } if( strcmp(zType, "table")==0 ){ - sqlite3_stmt *pTableInfo = 0; - char *zSelect = 0; - char *zTableInfo = 0; - char *zTmp = 0; - int nRow = 0; + ShellText sSelect; + ShellText sTable; + char **azCol; + int i; + char *savedDestTable; + int savedMode; - zTableInfo = appendText(zTableInfo, "PRAGMA table_info(", 0); - zTableInfo = appendText(zTableInfo, zTable, '"'); - zTableInfo = appendText(zTableInfo, ");", 0); - - rc = sqlite3_prepare_v2(p->db, zTableInfo, -1, &pTableInfo, 0); - free(zTableInfo); - if( rc!=SQLITE_OK || !pTableInfo ){ - return 1; + azCol = tableColumnList(p, zTable); + if( azCol==0 ){ + p->nErr++; + return 0; } - zSelect = appendText(zSelect, "SELECT 'INSERT INTO ' || ", 0); /* Always quote the table name, even if it appears to be pure ascii, ** in case it is a keyword. Ex: INSERT INTO "table" ... */ - zTmp = appendText(zTmp, zTable, '"'); - if( zTmp ){ - zSelect = appendText(zSelect, zTmp, '\''); - free(zTmp); - } - zSelect = appendText(zSelect, " || ' VALUES(' || ", 0); - rc = sqlite3_step(pTableInfo); - while( rc==SQLITE_ROW ){ - const char *zText = (const char *)sqlite3_column_text(pTableInfo, 1); - zSelect = appendText(zSelect, "quote(", 0); - zSelect = appendText(zSelect, zText, '"'); - rc = sqlite3_step(pTableInfo); - if( rc==SQLITE_ROW ){ - zSelect = appendText(zSelect, "), ", 0); - }else{ - zSelect = appendText(zSelect, ") ", 0); + initText(&sTable); + appendText(&sTable, zTable, quoteChar(zTable)); + /* If preserving the rowid, add a column list after the table name. + ** In other words: "INSERT INTO tab(rowid,a,b,c,...) VALUES(...)" + ** instead of the usual "INSERT INTO tab VALUES(...)". + */ + if( azCol[0] ){ + appendText(&sTable, "(", 0); + appendText(&sTable, azCol[0], 0); + for(i=1; azCol[i]; i++){ + appendText(&sTable, ",", 0); + appendText(&sTable, azCol[i], quoteChar(azCol[i])); } - nRow++; + appendText(&sTable, ")", 0); } - rc = sqlite3_finalize(pTableInfo); - if( rc!=SQLITE_OK || nRow==0 ){ - free(zSelect); - return 1; - } - zSelect = appendText(zSelect, "|| ')' FROM ", 0); - zSelect = appendText(zSelect, zTable, '"'); - rc = run_table_dump_query(p, zSelect, zPrepStmt); - if( rc==SQLITE_CORRUPT ){ - zSelect = appendText(zSelect, " ORDER BY rowid DESC", 0); - run_table_dump_query(p, zSelect, 0); + /* Build an appropriate SELECT statement */ + initText(&sSelect); + appendText(&sSelect, "SELECT ", 0); + if( azCol[0] ){ + appendText(&sSelect, azCol[0], 0); + appendText(&sSelect, ",", 0); } - free(zSelect); + for(i=1; azCol[i]; i++){ + appendText(&sSelect, azCol[i], quoteChar(azCol[i])); + if( azCol[i+1] ){ + appendText(&sSelect, ",", 0); + } + } + freeColumnList(azCol); + appendText(&sSelect, " FROM ", 0); + appendText(&sSelect, zTable, quoteChar(zTable)); + + savedDestTable = p->zDestTable; + savedMode = p->mode; + p->zDestTable = sTable.z; + p->mode = p->cMode = MODE_Insert; + rc = shell_exec(p->db, sSelect.z, shell_callback, p, 0); + if( (rc&0xff)==SQLITE_CORRUPT ){ + raw_printf(p->out, "/****** CORRUPTION ERROR *******/\n"); + toggleSelectOrder(p->db); + shell_exec(p->db, sSelect.z, shell_callback, p, 0); + toggleSelectOrder(p->db); + } + p->zDestTable = savedDestTable; + p->mode = savedMode; + freeText(&sTable); + freeText(&sSelect); + if( rc ) p->nErr++; } return 0; } @@ -2212,7 +3150,7 @@ static char zHelp[] = " html HTML code\n" " insert SQL insert statements for TABLE\n" " line One value per line\n" - " list Values delimited by .separator strings\n" + " list Values delimited by \"|\"\n" " quote Escape answers as for SQL\n" " tabs Tab-separated values\n" " tcl TCL list elements\n" @@ -2230,11 +3168,13 @@ static char zHelp[] = ".scanstats on|off Turn sqlite3_stmt_scanstatus() metrics on or off\n" ".schema ?PATTERN? Show the CREATE statements matching PATTERN\n" " Add --indent for pretty-printing\n" + ".selftest ?--init? Run tests defined in the SELFTEST table\n" ".separator COL ?ROW? Change the column separator and optionally the row\n" " separator for both the output mode and .import\n" #if defined(SQLITE_ENABLE_SESSION) ".session CMD ... Create or control sessions\n" #endif + ".sha3sum ?OPTIONS...? Compute a SHA3 hash of database content\n" ".shell CMD ARGS... Run CMD ARGS... in a system shell\n" ".show Show the current values for various settings\n" ".stats ?on|off? Show stats or turn stats on or off\n" @@ -2425,10 +3365,6 @@ static void open_db(ShellState *p, int keepAlive){ sqlite3_initialize(); sqlite3_open(p->zDbFilename, &p->db); globalDb = p->db; - if( p->db && sqlite3_errcode(p->db)==SQLITE_OK ){ - sqlite3_create_function(p->db, "shellstatic", 0, SQLITE_UTF8, 0, - shellstaticFunc, 0, 0); - } if( p->db==0 || SQLITE_OK!=sqlite3_errcode(p->db) ){ utf8_printf(stderr,"Error: unable to open database \"%s\": %s\n", p->zDbFilename, sqlite3_errmsg(p->db)); @@ -2442,6 +3378,14 @@ static void open_db(ShellState *p, int keepAlive){ readfileFunc, 0, 0); sqlite3_create_function(p->db, "writefile", 2, SQLITE_UTF8, 0, writefileFunc, 0, 0); + sqlite3_create_function(p->db, "sha3", 1, SQLITE_UTF8, 0, + sha3Func, 0, 0); + sqlite3_create_function(p->db, "sha3", 2, SQLITE_UTF8, 0, + sha3Func, 0, 0); + sqlite3_create_function(p->db, "sha3_query", 1, SQLITE_UTF8, 0, + sha3QueryFunc, 0, 0); + sqlite3_create_function(p->db, "sha3_query", 2, SQLITE_UTF8, 0, + sha3QueryFunc, 0, 0); } } @@ -2566,7 +3510,7 @@ static sqlite3_int64 integerValue(const char *zArg){ ** Interpret zArg as either an integer or a boolean value. Return 1 or 0 ** for TRUE and FALSE. Return the integer value if appropriate. */ -static int booleanValue(char *zArg){ +static int booleanValue(const char *zArg){ int i; if( zArg[0]=='0' && zArg[1]=='x' ){ for(i=2; hexDigitValue(zArg[i])>=0; i++){} @@ -2585,6 +3529,17 @@ static int booleanValue(char *zArg){ return 0; } +/* +** Set or clear a shell flag according to a boolean value. +*/ +static void setOrClearFlag(ShellState *p, unsigned mFlag, const char *zArg){ + if( booleanValue(zArg) ){ + ShellSetFlag(p, mFlag); + }else{ + ShellClearFlag(p, mFlag); + } +} + /* ** Close an output file, assuming it is not stderr or stdout */ @@ -3658,7 +4613,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='c' && n>=3 && strncmp(azArg[0], "changes", n)==0 ){ if( nArg==2 ){ - p->countChanges = booleanValue(azArg[1]); + setOrClearFlag(p, SHFLG_CountChanges, azArg[1]); }else{ raw_printf(stderr, "Usage: .changes on|off\n"); rc = 1; @@ -3722,21 +4677,42 @@ static int do_meta_command(char *zLine, ShellState *p){ }else if( c=='d' && strncmp(azArg[0], "dump", n)==0 ){ + const char *zLike = 0; + int i; + ShellClearFlag(p, SHFLG_PreserveRowid); + for(i=1; iout, "PRAGMA foreign_keys=OFF;\n"); raw_printf(p->out, "BEGIN TRANSACTION;\n"); p->writableSchema = 0; + /* Set writable_schema=ON since doing so forces SQLite to initialize + ** as much of the schema as it can even if the sqlite_master table is + ** corrupt. */ sqlite3_exec(p->db, "SAVEPOINT dump; PRAGMA writable_schema=ON", 0, 0, 0); p->nErr = 0; - if( nArg==1 ){ + if( zLike==0 ){ run_schema_dump_query(p, "SELECT name, type, sql FROM sqlite_master " "WHERE sql NOT NULL AND type=='table' AND name!='sqlite_sequence'" @@ -3750,21 +4726,20 @@ static int do_meta_command(char *zLine, ShellState *p){ "WHERE sql NOT NULL AND type IN ('index','trigger','view')", 0 ); }else{ - int i; - for(i=1; iwritableSchema ){ raw_printf(p->out, "PRAGMA writable_schema=OFF;\n"); @@ -3777,7 +4752,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='e' && strncmp(azArg[0], "echo", n)==0 ){ if( nArg==2 ){ - p->echoOn = booleanValue(azArg[1]); + setOrClearFlag(p, SHFLG_Echo, azArg[1]); }else{ raw_printf(stderr, "Usage: .echo on|off\n"); rc = 1; @@ -4288,15 +5263,20 @@ static int do_meta_command(char *zLine, ShellState *p){ int c2 = zMode[0]; if( c2=='l' && n2>2 && strncmp(azArg[1],"lines",n2)==0 ){ p->mode = MODE_Line; + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); }else if( c2=='c' && strncmp(azArg[1],"columns",n2)==0 ){ p->mode = MODE_Column; + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); }else if( c2=='l' && n2>2 && strncmp(azArg[1],"list",n2)==0 ){ p->mode = MODE_List; + sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Column); + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); }else if( c2=='h' && strncmp(azArg[1],"html",n2)==0 ){ p->mode = MODE_Html; }else if( c2=='t' && strncmp(azArg[1],"tcl",n2)==0 ){ p->mode = MODE_Tcl; sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Space); + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); }else if( c2=='c' && strncmp(azArg[1],"csv",n2)==0 ){ p->mode = MODE_Csv; sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma); @@ -4339,6 +5319,7 @@ static int do_meta_command(char *zLine, ShellState *p){ session_close_all(p); sqlite3_close(p->db); p->db = 0; + p->zDbFilename = 0; sqlite3_free(p->zFreeOnClose); p->zFreeOnClose = 0; /* Check for command-line arguments */ @@ -4568,17 +5549,17 @@ static int do_meta_command(char *zLine, ShellState *p){ callback(&data, 1, new_argv, new_colv); rc = SQLITE_OK; }else{ - zShellStatic = azArg[1]; - rc = sqlite3_exec(p->db, + char *zSql; + zSql = sqlite3_mprintf( "SELECT sql FROM " " (SELECT sql sql, type type, tbl_name tbl_name, name name, rowid x" " FROM sqlite_master UNION ALL" " SELECT sql, type, tbl_name, name, rowid FROM sqlite_temp_master) " - "WHERE lower(tbl_name) LIKE shellstatic()" + "WHERE lower(tbl_name) LIKE %Q" " AND type!='meta' AND sql NOTNULL " - "ORDER BY rowid", - callback, &data, &zErrMsg); - zShellStatic = 0; + "ORDER BY rowid", azArg[1]); + rc = sqlite3_exec(p->db, zSql, callback, &data, &zErrMsg); + sqlite3_free(zSql); } }else if( nArg==1 ){ rc = sqlite3_exec(p->db, @@ -4609,7 +5590,7 @@ static int do_meta_command(char *zLine, ShellState *p){ #if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_SELECTTRACE) if( c=='s' && n==11 && strncmp(azArg[0], "selecttrace", n)==0 ){ - sqlite3SelectTrace = integerValue(azArg[1]); + sqlite3SelectTrace = (int)integerValue(azArg[1]); }else #endif @@ -4832,6 +5813,119 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #endif + if( c=='s' && n>=4 && strncmp(azArg[0],"selftest",n)==0 ){ + int bIsInit = 0; /* True to initialize the SELFTEST table */ + int bVerbose = 0; /* Verbose output */ + int bSelftestExists; /* True if SELFTEST already exists */ + char **azTest = 0; /* Content of the SELFTEST table */ + int nRow = 0; /* Number of rows in the SELFTEST table */ + int nCol = 4; /* Number of columns in the SELFTEST table */ + int i; /* Loop counter */ + int nTest = 0; /* Number of tests runs */ + int nErr = 0; /* Number of errors seen */ + ShellText str; /* Answer for a query */ + static char *azDefaultTest[] = { + 0, 0, 0, 0, + "0", "memo", "Missing SELFTEST table - default checks only", "", + "1", "run", "PRAGMA integrity_check", "ok" + }; + static const int nDefaultRow = 2; + + open_db(p,0); + for(i=1; idb,"main","selftest",0,0,0,0,0,0) + != SQLITE_OK ){ + bSelftestExists = 0; + }else{ + bSelftestExists = 1; + } + if( bIsInit ){ + createSelftestTable(p); + bSelftestExists = 1; + } + if( bSelftestExists ){ + rc = sqlite3_get_table(p->db, + "SELECT tno,op,cmd,ans FROM selftest ORDER BY tno", + &azTest, &nRow, &nCol, 0); + if( rc ){ + raw_printf(stderr, "Error querying the selftest table\n"); + rc = 1; + sqlite3_free_table(azTest); + goto meta_command_exit; + }else if( nRow==0 ){ + sqlite3_free_table(azTest); + azTest = azDefaultTest; + nRow = nDefaultRow; + } + }else{ + azTest = azDefaultTest; + nRow = nDefaultRow; + } + initText(&str); + appendText(&str, "x", 0); + for(i=1; i<=nRow; i++){ + int tno = atoi(azTest[i*nCol]); + const char *zOp = azTest[i*nCol+1]; + const char *zSql = azTest[i*nCol+2]; + const char *zAns = azTest[i*nCol+3]; + + if( bVerbose>0 ){ + char *zQuote = sqlite3_mprintf("%q", zSql); + printf("%d: %s %s\n", tno, zOp, zSql); + sqlite3_free(zQuote); + } + if( strcmp(zOp,"memo")==0 ){ + utf8_printf(p->out, "%s\n", zSql); + }else + if( strcmp(zOp,"run")==0 ){ + char *zErrMsg = 0; + str.n = 0; + str.z[0] = 0; + rc = sqlite3_exec(p->db, zSql, captureOutputCallback, &str, &zErrMsg); + nTest++; + if( bVerbose ){ + utf8_printf(p->out, "Result: %s\n", str.z); + } + if( rc || zErrMsg ){ + nErr++; + rc = 1; + utf8_printf(p->out, "%d: error-code-%d: %s\n", tno, rc, zErrMsg); + sqlite3_free(zErrMsg); + }else if( strcmp(zAns,str.z)!=0 ){ + nErr++; + rc = 1; + utf8_printf(p->out, "%d: Expected: [%s]\n", tno, zAns); + utf8_printf(p->out, "%d: Got: [%s]\n", tno, str.z); + } + }else + { + utf8_printf(stderr, + "Unknown operation \"%s\" on selftest line %d\n", zOp, tno); + rc = 1; + break; + } + } + freeText(&str); + if( azTest!=azDefaultTest ) sqlite3_free_table(azTest); + utf8_printf(p->out, "%d errors out of %d tests\n", nErr, nTest); + }else + if( c=='s' && strncmp(azArg[0], "separator", n)==0 ){ if( nArg<2 || nArg>3 ){ raw_printf(stderr, "Usage: .separator COL ?ROW?\n"); @@ -4847,6 +5941,122 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else + if( c=='s' && n>=4 && strncmp(azArg[0],"sha3sum",n)==0 ){ + const char *zLike = 0; /* Which table to checksum. 0 means everything */ + int i; /* Loop counter */ + int bSchema = 0; /* Also hash the schema */ + int bSeparate = 0; /* Hash each table separately */ + int iSize = 224; /* Hash algorithm to use */ + int bDebug = 0; /* Only show the query that would have run */ + sqlite3_stmt *pStmt; /* For querying tables names */ + char *zSql; /* SQL to be run */ + char *zSep; /* Separator */ + ShellText sSql; /* Complete SQL for the query to run the hash */ + ShellText sQuery; /* Set of queries used to read all content */ + open_db(p, 0); + for(i=1; i1" + " UNION ALL SELECT 'sqlite_master'" + " ORDER BY 1 collate nocase"; + }else{ + zSql = "SELECT lower(name) FROM sqlite_master" + " WHERE type='table' AND coalesce(rootpage,0)>1" + " AND name NOT LIKE 'sqlite_%'" + " ORDER BY 1 collate nocase"; + } + sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + initText(&sQuery); + initText(&sSql); + appendText(&sSql, "WITH [sha3sum$query](a,b) AS(",0); + zSep = "VALUES("; + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zTab = (const char*)sqlite3_column_text(pStmt,0); + if( zLike && sqlite3_strlike(zLike, zTab, 0)!=0 ) continue; + if( strncmp(zTab, "sqlite_",7)!=0 ){ + appendText(&sQuery,"SELECT * FROM ", 0); + appendText(&sQuery,zTab,'"'); + appendText(&sQuery," NOT INDEXED;", 0); + }else if( strcmp(zTab, "sqlite_master")==0 ){ + appendText(&sQuery,"SELECT type,name,tbl_name,sql FROM sqlite_master" + " ORDER BY name;", 0); + }else if( strcmp(zTab, "sqlite_sequence")==0 ){ + appendText(&sQuery,"SELECT name,seq FROM sqlite_sequence" + " ORDER BY name;", 0); + }else if( strcmp(zTab, "sqlite_stat1")==0 ){ + appendText(&sQuery,"SELECT tbl,idx,stat FROM sqlite_stat1" + " ORDER BY tbl,idx;", 0); + }else if( strcmp(zTab, "sqlite_stat3")==0 + || strcmp(zTab, "sqlite_stat4")==0 ){ + appendText(&sQuery, "SELECT * FROM ", 0); + appendText(&sQuery, zTab, 0); + appendText(&sQuery, " ORDER BY tbl, idx, rowid;\n", 0); + } + appendText(&sSql, zSep, 0); + appendText(&sSql, sQuery.z, '\''); + sQuery.n = 0; + appendText(&sSql, ",", 0); + appendText(&sSql, zTab, '\''); + zSep = "),("; + } + sqlite3_finalize(pStmt); + if( bSeparate ){ + zSql = sqlite3_mprintf( + "%s))" + " SELECT lower(hex(sha3_query(a,%d))) AS hash, b AS label" + " FROM [sha3sum$query]", + sSql.z, iSize); + }else{ + zSql = sqlite3_mprintf( + "%s))" + " SELECT lower(hex(sha3_query(group_concat(a,''),%d))) AS hash" + " FROM [sha3sum$query]", + sSql.z, iSize); + } + freeText(&sQuery); + freeText(&sSql); + if( bDebug ){ + utf8_printf(p->out, "%s\n", zSql); + }else{ + shell_exec(p->db, zSql, shell_callback, p, 0); + } + sqlite3_free(zSql); + }else + if( c=='s' && (strncmp(azArg[0], "shell", n)==0 || strncmp(azArg[0],"system",n)==0) ){ @@ -4875,7 +6085,8 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = 1; goto meta_command_exit; } - utf8_printf(p->out, "%12.12s: %s\n","echo", azBool[p->echoOn!=0]); + utf8_printf(p->out, "%12.12s: %s\n","echo", + azBool[ShellHasFlag(p, SHFLG_Echo)]); utf8_printf(p->out, "%12.12s: %s\n","eqp", azBool[p->autoEQP&3]); utf8_printf(p->out, "%12.12s: %s\n","explain", p->mode==MODE_Explain ? "on" : p->autoExplain ? "auto" : "off"); @@ -5187,6 +6398,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } } }else +#endif /* !defined(SQLITE_UNTESTABLE) */ if( c=='t' && n>4 && strncmp(azArg[0], "timeout", n)==0 ){ open_db(p, 0); @@ -5223,7 +6435,6 @@ static int do_meta_command(char *zLine, ShellState *p){ } #endif }else -#endif /* !defined(SQLITE_UNTESTABLE) */ #if SQLITE_USER_AUTHENTICATION if( c=='u' && strncmp(azArg[0], "user", n)==0 ){ @@ -5297,7 +6508,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='v' && strncmp(azArg[0], "vfsinfo", n)==0 ){ const char *zDbName = nArg==2 ? azArg[1] : "main"; - sqlite3_vfs *pVfs; + sqlite3_vfs *pVfs = 0; if( p->db ){ sqlite3_file_control(p->db, zDbName, SQLITE_FCNTL_VFS_POINTER, &pVfs); if( pVfs ){ @@ -5440,7 +6651,7 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){ char *zErrMsg = 0; open_db(p, 0); - if( p->backslashOn ) resolve_backslashes(zSql); + if( ShellHasFlag(p,SHFLG_Backslash) ) resolve_backslashes(zSql); BEGIN_TIMER; rc = shell_exec(p->db, zSql, shell_callback, p, &zErrMsg); END_TIMER; @@ -5460,7 +6671,7 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){ utf8_printf(stderr, "%s %s\n", zPrefix, sqlite3_errmsg(p->db)); } return 1; - }else if( p->countChanges ){ + }else if( ShellHasFlag(p, SHFLG_CountChanges) ){ raw_printf(p->out, "changes: %3d total_changes: %d\n", sqlite3_changes(p->db), sqlite3_total_changes(p->db)); } @@ -5503,11 +6714,11 @@ static int process_input(ShellState *p, FILE *in){ } lineno++; if( nSql==0 && _all_whitespace(zLine) ){ - if( p->echoOn ) printf("%s\n", zLine); + if( ShellHasFlag(p, SHFLG_Echo) ) printf("%s\n", zLine); continue; } if( zLine && zLine[0]=='.' && nSql==0 ){ - if( p->echoOn ) printf("%s\n", zLine); + if( ShellHasFlag(p, SHFLG_Echo) ) printf("%s\n", zLine); rc = do_meta_command(zLine, p); if( rc==2 ){ /* exit requested */ break; @@ -5550,7 +6761,7 @@ static int process_input(ShellState *p, FILE *in){ p->outCount = 0; } }else if( nSql && _all_whitespace(zSql) ){ - if( p->echoOn ) printf("%s\n", zSql); + if( ShellHasFlag(p, SHFLG_Echo) ) printf("%s\n", zSql); nSql = 0; } } @@ -6019,7 +7230,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ }else if( strcmp(z,"-noheader")==0 ){ data.showHeader = 0; }else if( strcmp(z,"-echo")==0 ){ - data.echoOn = 1; + ShellSetFlag(&data, SHFLG_Echo); }else if( strcmp(z,"-eqp")==0 ){ data.autoEQP = 1; }else if( strcmp(z,"-eqpfull")==0 ){ @@ -6034,7 +7245,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ ** prior to sending the SQL into SQLite. Useful for injecting ** crazy bytes in the middle of SQL statements for testing and debugging. */ - data.backslashOn = 1; + ShellSetFlag(&data, SHFLG_Backslash); }else if( strcmp(z,"-bail")==0 ){ bail_on_error = 1; }else if( strcmp(z,"-version")==0 ){ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 1bcb62256c..97294c3f81 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -114,8 +114,8 @@ extern "C" { ** system. ^The SQLITE_SOURCE_ID macro evaluates to ** a string which identifies a particular check-in of SQLite ** within its configuration management system. ^The SQLITE_SOURCE_ID -** string contains the date and time of the check-in (UTC) and an SHA1 -** hash of the entire source tree. +** string contains the date and time of the check-in (UTC) and a SHA1 +** or SHA3-256 hash of the entire source tree. ** ** See also: [sqlite3_libversion()], ** [sqlite3_libversion_number()], [sqlite3_sourceid()], @@ -259,7 +259,11 @@ typedef struct sqlite3 sqlite3; */ #ifdef SQLITE_INT64_TYPE typedef SQLITE_INT64_TYPE sqlite_int64; - typedef unsigned SQLITE_INT64_TYPE sqlite_uint64; +# ifdef SQLITE_UINT64_TYPE + typedef SQLITE_UINT64_TYPE sqlite_uint64; +# else + typedef unsigned SQLITE_INT64_TYPE sqlite_uint64; +# endif #elif defined(_MSC_VER) || defined(__BORLANDC__) typedef __int64 sqlite_int64; typedef unsigned __int64 sqlite_uint64; @@ -572,7 +576,7 @@ int sqlite3_exec( ** file that were written at the application level might have changed ** and that adjacent bytes, even bytes within the same sector are ** guaranteed to be unchanged. The SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN -** flag indicate that a file cannot be deleted when open. The +** flag indicates that a file cannot be deleted when open. The ** SQLITE_IOCAP_IMMUTABLE flag indicates that the file is on ** read-only media and cannot be changed even by processes with ** elevated privileges. @@ -722,6 +726,9 @@ struct sqlite3_file { **
  • [SQLITE_IOCAP_ATOMIC64K] **
  • [SQLITE_IOCAP_SAFE_APPEND] **
  • [SQLITE_IOCAP_SEQUENTIAL] +**
  • [SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN] +**
  • [SQLITE_IOCAP_POWERSAFE_OVERWRITE] +**
  • [SQLITE_IOCAP_IMMUTABLE] ** ** ** The SQLITE_IOCAP_ATOMIC property means that all writes of @@ -2033,20 +2040,30 @@ int sqlite3_extended_result_codes(sqlite3*, int onoff); ** the table has a column of type [INTEGER PRIMARY KEY] then that column ** is another alias for the rowid. ** -** ^The sqlite3_last_insert_rowid(D) interface returns the [rowid] of the -** most recent successful [INSERT] into a rowid table or [virtual table] -** on database connection D. -** ^Inserts into [WITHOUT ROWID] tables are not recorded. -** ^If no successful [INSERT]s into rowid tables -** have ever occurred on the database connection D, -** then sqlite3_last_insert_rowid(D) returns zero. +** ^The sqlite3_last_insert_rowid(D) interface usually returns the [rowid] of +** the most recent successful [INSERT] into a rowid table or [virtual table] +** on database connection D. ^Inserts into [WITHOUT ROWID] tables are not +** recorded. ^If no successful [INSERT]s into rowid tables have ever occurred +** on the database connection D, then sqlite3_last_insert_rowid(D) returns +** zero. ** -** ^(If an [INSERT] occurs within a trigger or within a [virtual table] -** method, then this routine will return the [rowid] of the inserted -** row as long as the trigger or virtual table method is running. -** But once the trigger or virtual table method ends, the value returned -** by this routine reverts to what it was before the trigger or virtual -** table method began.)^ +** As well as being set automatically as rows are inserted into database +** tables, the value returned by this function may be set explicitly by +** [sqlite3_set_last_insert_rowid()] +** +** Some virtual table implementations may INSERT rows into rowid tables as +** part of committing a transaction (e.g. to flush data accumulated in memory +** to disk). In this case subsequent calls to this function return the rowid +** associated with these internal INSERT operations, which leads to +** unintuitive results. Virtual table implementations that do write to rowid +** tables in this way can avoid this problem by restoring the original +** rowid value using [sqlite3_set_last_insert_rowid()] before returning +** control to the user. +** +** ^(If an [INSERT] occurs within a trigger then this routine will +** return the [rowid] of the inserted row as long as the trigger is +** running. Once the trigger program ends, the value returned +** by this routine reverts to what it was before the trigger was fired.)^ ** ** ^An [INSERT] that fails due to a constraint violation is not a ** successful [INSERT] and does not change the value returned by this @@ -2073,6 +2090,16 @@ int sqlite3_extended_result_codes(sqlite3*, int onoff); */ sqlite3_int64 sqlite3_last_insert_rowid(sqlite3*); +/* +** CAPI3REF: Set the Last Insert Rowid value. +** METHOD: sqlite3 +** +** The sqlite3_set_last_insert_rowid(D, R) method allows the application to +** set the value returned by calling sqlite3_last_insert_rowid(D) to R +** without inserting a row into the database. +*/ +void sqlite3_set_last_insert_rowid(sqlite3*,sqlite3_int64); + /* ** CAPI3REF: Count The Number Of Rows Modified ** METHOD: sqlite3 @@ -3397,9 +3424,9 @@ int sqlite3_limit(sqlite3*, int id, int newVal); ** ** [[SQLITE_LIMIT_VDBE_OP]] ^(
    SQLITE_LIMIT_VDBE_OP
    **
    The maximum number of instructions in a virtual machine program -** used to implement an SQL statement. This limit is not currently -** enforced, though that might be added in some future release of -** SQLite.
    )^ +** used to implement an SQL statement. If [sqlite3_prepare_v2()] or +** the equivalent tries to allocate space for more than this many opcodes +** in a single prepared statement, an SQLITE_NOMEM error is returned.)^ ** ** [[SQLITE_LIMIT_FUNCTION_ARG]] ^(
    SQLITE_LIMIT_FUNCTION_ARG
    **
    The maximum number of arguments on a function.
    )^ @@ -3437,6 +3464,7 @@ int sqlite3_limit(sqlite3*, int id, int newVal); #define SQLITE_LIMIT_TRIGGER_DEPTH 10 #define SQLITE_LIMIT_WORKER_THREADS 11 + /* ** CAPI3REF: Compiling An SQL Statement ** KEYWORDS: {SQL statement compiler} @@ -5410,7 +5438,7 @@ void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); ** ^The update hook is not invoked when [WITHOUT ROWID] tables are modified. ** ** ^In the current implementation, the update hook -** is not invoked when duplication rows are deleted because of an +** is not invoked when conflicting rows are deleted because of an ** [ON CONFLICT | ON CONFLICT REPLACE] clause. ^Nor is the update hook ** invoked when rows are deleted using the [truncate optimization]. ** The exceptions defined in this paragraph might change in a future @@ -6192,6 +6220,12 @@ typedef struct sqlite3_blob sqlite3_blob; ** [database connection] error code and message accessible via ** [sqlite3_errcode()] and [sqlite3_errmsg()] and related functions. ** +** A BLOB referenced by sqlite3_blob_open() may be read using the +** [sqlite3_blob_read()] interface and modified by using +** [sqlite3_blob_write()]. The [BLOB handle] can be moved to a +** different row of the same table using the [sqlite3_blob_reopen()] +** interface. However, the column, table, or database of a [BLOB handle] +** cannot be changed after the [BLOB handle] is opened. ** ** ^(If the row that a BLOB handle points to is modified by an ** [UPDATE], [DELETE], or by [ON CONFLICT] side-effects @@ -6215,6 +6249,10 @@ typedef struct sqlite3_blob sqlite3_blob; ** ** To avoid a resource leak, every open [BLOB handle] should eventually ** be released by a call to [sqlite3_blob_close()]. +** +** See also: [sqlite3_blob_close()], +** [sqlite3_blob_reopen()], [sqlite3_blob_read()], +** [sqlite3_blob_bytes()], [sqlite3_blob_write()]. */ int sqlite3_blob_open( sqlite3*, @@ -6230,11 +6268,11 @@ int sqlite3_blob_open( ** CAPI3REF: Move a BLOB Handle to a New Row ** METHOD: sqlite3_blob ** -** ^This function is used to move an existing blob handle so that it points +** ^This function is used to move an existing [BLOB handle] so that it points ** to a different row of the same database table. ^The new row is identified ** by the rowid value passed as the second argument. Only the row can be ** changed. ^The database, table and column on which the blob handle is open -** remain the same. Moving an existing blob handle to a new row can be +** remain the same. Moving an existing [BLOB handle] to a new row is ** faster than closing the existing handle and opening a new one. ** ** ^(The new row must meet the same criteria as for [sqlite3_blob_open()] - @@ -8163,7 +8201,7 @@ int sqlite3_db_cacheflush(sqlite3*); ** ** ^The [sqlite3_preupdate_hook()] interface registers a callback function ** that is invoked prior to each [INSERT], [UPDATE], and [DELETE] operation -** on a [rowid table]. +** on a database table. ** ^At most one preupdate hook may be registered at a time on a single ** [database connection]; each call to [sqlite3_preupdate_hook()] overrides ** the previous setting. @@ -8172,9 +8210,9 @@ int sqlite3_db_cacheflush(sqlite3*); ** ^The third parameter to [sqlite3_preupdate_hook()] is passed through as ** the first parameter to callbacks. ** -** ^The preupdate hook only fires for changes to [rowid tables]; the preupdate -** hook is not invoked for changes to [virtual tables] or [WITHOUT ROWID] -** tables. +** ^The preupdate hook only fires for changes to real database tables; the +** preupdate hook is not invoked for changes to [virtual tables] or to +** system tables like sqlite_master or sqlite_stat1. ** ** ^The second parameter to the preupdate callback is a pointer to ** the [database connection] that registered the preupdate hook. @@ -8188,12 +8226,16 @@ int sqlite3_db_cacheflush(sqlite3*); ** databases.)^ ** ^The fifth parameter to the preupdate callback is the name of the ** table that is being modified. -** ^The sixth parameter to the preupdate callback is the initial [rowid] of the -** row being changes for SQLITE_UPDATE and SQLITE_DELETE changes and is -** undefined for SQLITE_INSERT changes. -** ^The seventh parameter to the preupdate callback is the final [rowid] of -** the row being changed for SQLITE_UPDATE and SQLITE_INSERT changes and is -** undefined for SQLITE_DELETE changes. +** +** For an UPDATE or DELETE operation on a [rowid table], the sixth +** parameter passed to the preupdate callback is the initial [rowid] of the +** row being modified or deleted. For an INSERT operation on a rowid table, +** or any operation on a WITHOUT ROWID table, the value of the sixth +** parameter is undefined. For an INSERT or UPDATE on a rowid table the +** seventh parameter is the final rowid value of the row being inserted +** or updated. The value of the seventh parameter passed to the callback +** function is not defined for operations on WITHOUT ROWID tables, or for +** INSERT operations on rowid tables. ** ** The [sqlite3_preupdate_old()], [sqlite3_preupdate_new()], ** [sqlite3_preupdate_count()], and [sqlite3_preupdate_depth()] interfaces diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 308498afc5..c06da27b45 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -103,19 +103,28 @@ # define _LARGEFILE_SOURCE 1 #endif -/* What version of GCC is being used. 0 means GCC is not being used */ -#ifdef __GNUC__ +/* The GCC_VERSION and MSVC_VERSION macros are used to +** conditionally include optimizations for each of these compilers. A +** value of 0 means that compiler is not being used. The +** SQLITE_DISABLE_INTRINSIC macro means do not use any compiler-specific +** optimizations, and hence set all compiler macros to 0 +** +** There was once also a CLANG_VERSION macro. However, we learn that the +** version numbers in clang are for "marketing" only and are inconsistent +** and unreliable. Fortunately, all versions of clang also recognize the +** gcc version numbers and have reasonable settings for gcc version numbers, +** so the GCC_VERSION macro will be set to a correct non-zero value even +** when compiling with clang. +*/ +#if defined(__GNUC__) && !defined(SQLITE_DISABLE_INTRINSIC) # define GCC_VERSION (__GNUC__*1000000+__GNUC_MINOR__*1000+__GNUC_PATCHLEVEL__) #else # define GCC_VERSION 0 #endif - -/* What version of CLANG is being used. 0 means CLANG is not being used */ -#ifdef __clang__ -# define CLANG_VERSION \ - (__clang_major__*1000000+__clang_minor__*1000+__clang_patchlevel__) +#if defined(_MSC_VER) && !defined(SQLITE_DISABLE_INTRINSIC) +# define MSVC_VERSION _MSC_VER #else -# define CLANG_VERSION 0 +# define MSVC_VERSION 0 #endif /* Needed for various definitions... */ @@ -247,6 +256,7 @@ # include # pragma intrinsic(_byteswap_ushort) # pragma intrinsic(_byteswap_ulong) +# pragma intrinsic(_byteswap_uint64) # pragma intrinsic(_ReadWriteBarrier) # else # include @@ -511,6 +521,18 @@ #include #include +/* +** Use a macro to replace memcpy() if compiled with SQLITE_INLINE_MEMCPY. +** This allows better measurements of where memcpy() is used when running +** cachegrind. But this macro version of memcpy() is very slow so it +** should not be used in production. This is a performance measurement +** hack only. +*/ +#ifdef SQLITE_INLINE_MEMCPY +# define memcpy(D,S,N) {char*xxd=(char*)(D);const char*xxs=(const char*)(S);\ + int xxn=(N);while(xxn-->0)*(xxd++)=*(xxs++);} +#endif + /* ** If compiling for a processor that lacks floating point support, ** substitute integer for floating-point @@ -775,32 +797,35 @@ typedef INT16_TYPE LogEst; ** ** For best performance, an attempt is made to guess at the byte-order ** using C-preprocessor macros. If that is unsuccessful, or if -** -DSQLITE_RUNTIME_BYTEORDER=1 is set, then byte-order is determined +** -DSQLITE_BYTEORDER=0 is set, then byte-order is determined ** at run-time. */ -#if (defined(i386) || defined(__i386__) || defined(_M_IX86) || \ +#ifndef SQLITE_BYTEORDER +# if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ - defined(__arm__)) && !defined(SQLITE_RUNTIME_BYTEORDER) -# define SQLITE_BYTEORDER 1234 -# define SQLITE_BIGENDIAN 0 -# define SQLITE_LITTLEENDIAN 1 -# define SQLITE_UTF16NATIVE SQLITE_UTF16LE + defined(__arm__) +# define SQLITE_BYTEORDER 1234 +# elif defined(sparc) || defined(__ppc__) +# define SQLITE_BYTEORDER 4321 +# else +# define SQLITE_BYTEORDER 0 +# endif #endif -#if (defined(sparc) || defined(__ppc__)) \ - && !defined(SQLITE_RUNTIME_BYTEORDER) -# define SQLITE_BYTEORDER 4321 +#if SQLITE_BYTEORDER==4321 # define SQLITE_BIGENDIAN 1 # define SQLITE_LITTLEENDIAN 0 # define SQLITE_UTF16NATIVE SQLITE_UTF16BE -#endif -#if !defined(SQLITE_BYTEORDER) +#elif SQLITE_BYTEORDER==1234 +# define SQLITE_BIGENDIAN 0 +# define SQLITE_LITTLEENDIAN 1 +# define SQLITE_UTF16NATIVE SQLITE_UTF16LE +#else # ifdef SQLITE_AMALGAMATION const int sqlite3one = 1; # else extern const int sqlite3one; # endif -# define SQLITE_BYTEORDER 0 /* 0 means "unknown at compile-time" */ # define SQLITE_BIGENDIAN (*(char *)(&sqlite3one)==0) # define SQLITE_LITTLEENDIAN (*(char *)(&sqlite3one)==1) # define SQLITE_UTF16NATIVE (SQLITE_BIGENDIAN?SQLITE_UTF16BE:SQLITE_UTF16LE) @@ -1309,6 +1334,7 @@ struct sqlite3 { u8 isTransactionSavepoint; /* True if the outermost savepoint is a TS */ u8 mTrace; /* zero or more SQLITE_TRACE flags */ u8 skipBtreeMutex; /* True if no shared-cache backends */ + u8 nSqlExec; /* Number of pending OP_SqlExec opcodes */ int nextPagesize; /* Pagesize after VACUUM if >0 */ u32 magic; /* Magic number for detect library misuse */ int nChange; /* Value returned by sqlite3_changes() */ @@ -1824,6 +1850,7 @@ struct Table { /* ... also used as column name list in a VIEW */ int tnum; /* Root BTree page for this table */ u32 nTabRef; /* Number of pointers to this Table */ + u32 tabFlags; /* Mask of TF_* values */ i16 iPKey; /* If not negative, use aCol[iPKey] as the rowid */ i16 nCol; /* Number of columns in this table */ LogEst nRowLogEst; /* Estimated rows in table - from sqlite_stat1 table */ @@ -1831,7 +1858,6 @@ struct Table { #ifdef SQLITE_ENABLE_COSTMULT LogEst costMult; /* Cost multiplier for using this table */ #endif - u8 tabFlags; /* Mask of TF_* values */ u8 keyConf; /* What to do in case of uniqueness conflict on iPKey */ #ifndef SQLITE_OMIT_ALTERTABLE int addColOffset; /* Offset in CREATE TABLE stmt to add a new column */ @@ -1855,15 +1881,17 @@ struct Table { ** the TF_OOOHidden attribute would apply in this case. Such tables require ** special handling during INSERT processing. */ -#define TF_Readonly 0x01 /* Read-only system table */ -#define TF_Ephemeral 0x02 /* An ephemeral table */ -#define TF_HasPrimaryKey 0x04 /* Table has a primary key */ -#define TF_Autoincrement 0x08 /* Integer primary key is autoincrement */ -#define TF_Virtual 0x10 /* Is a virtual table */ -#define TF_WithoutRowid 0x20 /* No rowid. PRIMARY KEY is the key */ -#define TF_NoVisibleRowid 0x40 /* No user-visible "rowid" column */ -#define TF_OOOHidden 0x80 /* Out-of-Order hidden columns */ - +#define TF_Readonly 0x0001 /* Read-only system table */ +#define TF_Ephemeral 0x0002 /* An ephemeral table */ +#define TF_HasPrimaryKey 0x0004 /* Table has a primary key */ +#define TF_Autoincrement 0x0008 /* Integer primary key is autoincrement */ +#define TF_HasStat1 0x0010 /* nRowLogEst set from sqlite_stat1 */ +#define TF_WithoutRowid 0x0020 /* No rowid. PRIMARY KEY is the key */ +#define TF_NoVisibleRowid 0x0040 /* No user-visible "rowid" column */ +#define TF_OOOHidden 0x0080 /* Out-of-Order hidden columns */ +#define TF_StatsUsed 0x0100 /* Query planner decisions affected by + ** Index.aiRowLogEst[] values */ +#define TF_HasNotNull 0x0200 /* Contains NOT NULL constraints */ /* ** Test to see whether or not a table is a virtual table. This is @@ -1871,7 +1899,7 @@ struct Table { ** table support is omitted from the build. */ #ifndef SQLITE_OMIT_VIRTUALTABLE -# define IsVirtual(X) (((X)->tabFlags & TF_Virtual)!=0) +# define IsVirtual(X) ((X)->nModuleArg) #else # define IsVirtual(X) 0 #endif @@ -2106,6 +2134,7 @@ struct Index { unsigned isResized:1; /* True if resizeIndexObject() has been called */ unsigned isCovering:1; /* True if this is a covering index */ unsigned noSkipScan:1; /* Do not try to use skip-scan if true */ + unsigned hasStat1:1; /* aiRowLogEst values come from sqlite_stat1 */ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 int nSample; /* Number of elements in aSample[] */ int nSampleCol; /* Size of IndexSample.anEq[] and so on */ @@ -2416,7 +2445,7 @@ struct Expr { struct ExprList { int nExpr; /* Number of expressions on the list */ struct ExprList_item { /* For each expression in the list */ - Expr *pExpr; /* The list of expressions */ + Expr *pExpr; /* The parse tree for this expression */ char *zName; /* Token associated with this expression */ char *zSpan; /* Original text of the expression */ u8 sortOrder; /* 1 for DESC or 0 for ASC */ @@ -2581,7 +2610,7 @@ struct SrcList { #define WHERE_SORTBYGROUP 0x0200 /* Support sqlite3WhereIsSorted() */ #define WHERE_SEEK_TABLE 0x0400 /* Do not defer seeks on main table */ #define WHERE_ORDERBY_LIMIT 0x0800 /* ORDERBY+LIMIT on the inner loop */ - /* 0x1000 not currently used */ +#define WHERE_SEEK_UNIQ_TABLE 0x1000 /* Do not defer seeks if unique */ /* 0x2000 not currently used */ #define WHERE_USE_LIMIT 0x4000 /* Use the LIMIT in cost estimates */ /* 0x8000 not currently used */ @@ -3042,13 +3071,11 @@ struct AuthContext { #define OPFLAG_NCHANGE 0x01 /* OP_Insert: Set to update db->nChange */ /* Also used in P2 (not P5) of OP_Delete */ #define OPFLAG_EPHEM 0x01 /* OP_Column: Ephemeral output is ok */ -#define OPFLAG_LASTROWID 0x02 /* Set to update db->lastRowid */ +#define OPFLAG_LASTROWID 0x20 /* Set to update db->lastRowid */ #define OPFLAG_ISUPDATE 0x04 /* This OP_Insert is an sql UPDATE */ #define OPFLAG_APPEND 0x08 /* This is likely to be an append */ #define OPFLAG_USESEEKRESULT 0x10 /* Try to avoid a seek in BtreeInsert() */ -#ifdef SQLITE_ENABLE_PREUPDATE_HOOK #define OPFLAG_ISNOOP 0x40 /* OP_Delete does pre-update-hook only */ -#endif #define OPFLAG_LENGTHARG 0x40 /* OP_Column only used for length() */ #define OPFLAG_TYPEOFARG 0x80 /* OP_Column only used for typeof() */ #define OPFLAG_BULKCSR 0x01 /* OP_Open** used to open bulk cursor */ @@ -3056,7 +3083,7 @@ struct AuthContext { #define OPFLAG_FORDELETE 0x08 /* OP_Open should use BTREE_FORDELETE */ #define OPFLAG_P2ISREG 0x10 /* P2 to OP_Open** is a register number */ #define OPFLAG_PERMUTE 0x01 /* OP_Compare: use the permutation */ -#define OPFLAG_SAVEPOSITION 0x02 /* OP_Delete: keep cursor position */ +#define OPFLAG_SAVEPOSITION 0x02 /* OP_Delete/Insert: save cursor pos */ #define OPFLAG_AUXDELETE 0x04 /* OP_Delete: index in a DELETE op */ /* @@ -3741,6 +3768,7 @@ void sqlite3Vacuum(Parse*,Token*); int sqlite3RunVacuum(char**, sqlite3*, int); char *sqlite3NameFromToken(sqlite3*, Token*); int sqlite3ExprCompare(Expr*, Expr*, int); +int sqlite3ExprCompareSkip(Expr*, Expr*, int); int sqlite3ExprListCompare(ExprList*, ExprList*, int); int sqlite3ExprImpliesExpr(Expr*, Expr*, int); void sqlite3ExprAnalyzeAggregates(NameContext*, Expr*); @@ -3779,6 +3807,11 @@ int sqlite3GenerateIndexKey(Parse*, Index*, int, int, int, int*,Index*,int); void sqlite3ResolvePartIdxLabel(Parse*,int); void sqlite3GenerateConstraintChecks(Parse*,Table*,int*,int,int,int,int, u8,u8,int,int*,int*); +#ifdef SQLITE_ENABLE_NULL_TRIM + void sqlite3SetMakeRecordP5(Vdbe*,Table*); +#else +# define sqlite3SetMakeRecordP5(A,B) +#endif void sqlite3CompleteInsertion(Parse*,Table*,int,int,int,int*,int,int,int); int sqlite3OpenTableAndIndices(Parse*, Table*, int, u8, int, u8*, int*, int*); void sqlite3BeginWriteOperation(Parse*, int, int); @@ -4057,8 +4090,10 @@ char sqlite3IndexColumnAffinity(sqlite3*, Index*, int); /* ** The interface to the LEMON-generated parser */ -void *sqlite3ParserAlloc(void*(*)(u64)); -void sqlite3ParserFree(void*, void(*)(void*)); +#ifndef SQLITE_AMALGAMATION + void *sqlite3ParserAlloc(void*(*)(u64)); + void sqlite3ParserFree(void*, void(*)(void*)); +#endif void sqlite3Parser(void*, int, Token, Parse*); #ifdef YYTRACKMAXSTACKDEPTH int sqlite3ParserStackPeak(void*); @@ -4168,6 +4203,7 @@ const char *sqlite3JournalModename(int); #define sqlite3FkDropTable(a,b,c) #define sqlite3FkOldmask(a,b) 0 #define sqlite3FkRequired(a,b,c,d) 0 + #define sqlite3FkReferences(a) 0 #endif #ifndef SQLITE_OMIT_FOREIGN_KEY void sqlite3FkDelete(sqlite3 *, Table*); diff --git a/src/sqliteLimit.h b/src/sqliteLimit.h index 0554e61581..28e7a41cc3 100644 --- a/src/sqliteLimit.h +++ b/src/sqliteLimit.h @@ -87,7 +87,7 @@ ** Not currently enforced. */ #ifndef SQLITE_MAX_VDBE_OP -# define SQLITE_MAX_VDBE_OP 25000 +# define SQLITE_MAX_VDBE_OP 250000000 #endif /* diff --git a/src/table.c b/src/table.c index a50d83ce63..c79255f990 100644 --- a/src/table.c +++ b/src/table.c @@ -17,8 +17,6 @@ ** if they are not used. */ #include "sqliteInt.h" -#include -#include #ifndef SQLITE_OMIT_GET_TABLE diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 1d87c9b331..9df023b45c 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -2307,7 +2307,7 @@ static int SQLITE_TCLAPI DbObjCmd( } in = fopen(zFile, "rb"); if( in==0 ){ - Tcl_AppendResult(interp, "Error: cannot open file: ", zFile, NULL); + Tcl_AppendResult(interp, "Error: cannot open file: ", zFile, (char*)0); sqlite3_finalize(pStmt); return TCL_ERROR; } @@ -2536,7 +2536,7 @@ static int SQLITE_TCLAPI DbObjCmd( int n = strlen30(z); if( n>2 && strncmp(z, "-argcount",n)==0 ){ if( i==(objc-2) ){ - Tcl_AppendResult(interp, "option requires an argument: ", z, 0); + Tcl_AppendResult(interp, "option requires an argument: ", z,(char*)0); return TCL_ERROR; } if( Tcl_GetIntFromObj(interp, objv[i+1], &nArg) ) return TCL_ERROR; @@ -2551,7 +2551,7 @@ static int SQLITE_TCLAPI DbObjCmd( flags |= SQLITE_DETERMINISTIC; }else{ Tcl_AppendResult(interp, "bad option \"", z, - "\": must be -argcount or -deterministic", 0 + "\": must be -argcount or -deterministic", (char*)0 ); return TCL_ERROR; } @@ -3208,7 +3208,7 @@ static int SQLITE_TCLAPI DbObjCmd( pObj = Tcl_NewStringObj((char*)sqlite3_value_text(pValue), -1); Tcl_SetObjResult(interp, pObj); }else{ - Tcl_AppendResult(interp, sqlite3_errmsg(pDb->db), 0); + Tcl_AppendResult(interp, sqlite3_errmsg(pDb->db), (char*)0); return TCL_ERROR; } } @@ -3405,10 +3405,6 @@ static int SQLITE_TCLAPI DbMain( } zErrMsg = 0; p = (SqliteDb*)Tcl_Alloc( sizeof(*p) ); - if( p==0 ){ - Tcl_SetResult(interp, (char *)"malloc failed", TCL_STATIC); - return TCL_ERROR; - } memset(p, 0, sizeof(*p)); zFile = Tcl_GetStringFromObj(objv[2], 0); zFile = Tcl_TranslateFileName(interp, zFile, &translatedFilename); diff --git a/src/test6.c b/src/test6.c index 5304bcc31f..849cdeb3b9 100644 --- a/src/test6.c +++ b/src/test6.c @@ -161,13 +161,13 @@ static CrashGlobal g = {0, 0, SQLITE_DEFAULT_SECTOR_SIZE, 0, 0}; static int sqlite3CrashTestEnable = 0; static void *crash_malloc(int nByte){ - return (void *)Tcl_Alloc((size_t)nByte); + return (void *)Tcl_AttemptAlloc((size_t)nByte); } static void crash_free(void *p){ Tcl_Free(p); } static void *crash_realloc(void *p, int n){ - return (void *)Tcl_Realloc(p, (size_t)n); + return (void *)Tcl_AttemptRealloc(p, (size_t)n); } /* @@ -315,8 +315,9 @@ static int writeListSync(CrashFile *pFile, int isCrash){ assert(pWrite->zBuf); #ifdef TRACE_CRASHTEST - printf("Trashing %d sectors @ %lld (sector %d) (%s)\n", - 1+iLast-iFirst, pWrite->iOffset, iFirst, pWrite->pFile->zName + printf("Trashing %d sectors (%d bytes) @ %lld (sector %d) (%s)\n", + 1+iLast-iFirst, (1+iLast-iFirst)*g.iSectorSize, + pWrite->iOffset, iFirst, pWrite->pFile->zName ); #endif @@ -827,7 +828,7 @@ static int SQLITE_TCLAPI crashNowCmd( } /* -** tclcmd: sqlite_crash_enable ENABLE +** tclcmd: sqlite_crash_enable ENABLE ?DEFAULT? ** ** Parameter ENABLE must be a boolean value. If true, then the "crash" ** vfs is added to the system. If false, it is removed. @@ -839,6 +840,7 @@ static int SQLITE_TCLAPI crashEnableCmd( Tcl_Obj *CONST objv[] ){ int isEnable; + int isDefault = 0; static sqlite3_vfs crashVfs = { 2, /* iVersion */ 0, /* szOsFile */ @@ -862,14 +864,17 @@ static int SQLITE_TCLAPI crashEnableCmd( 0, /* xCurrentTimeInt64 */ }; - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 1, objv, "ENABLE"); + if( objc!=2 && objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "ENABLE ?DEFAULT?"); return TCL_ERROR; } if( Tcl_GetBooleanFromObj(interp, objv[1], &isEnable) ){ return TCL_ERROR; } + if( objc==3 && Tcl_GetBooleanFromObj(interp, objv[2], &isDefault) ){ + return TCL_ERROR; + } if( (isEnable && crashVfs.pAppData) || (!isEnable && !crashVfs.pAppData) ){ return TCL_OK; @@ -880,7 +885,7 @@ static int SQLITE_TCLAPI crashEnableCmd( crashVfs.mxPathname = pOriginalVfs->mxPathname; crashVfs.pAppData = (void *)pOriginalVfs; crashVfs.szOsFile = sizeof(CrashFile) + pOriginalVfs->szOsFile; - sqlite3_vfs_register(&crashVfs, 0); + sqlite3_vfs_register(&crashVfs, isDefault); }else{ crashVfs.pAppData = 0; sqlite3_vfs_unregister(&crashVfs); diff --git a/src/test_blob.c b/src/test_blob.c index 7fa733bee9..118f210738 100644 --- a/src/test_blob.c +++ b/src/test_blob.c @@ -239,7 +239,11 @@ static int SQLITE_TCLAPI test_blob_read( } if( nByte>0 ){ - zBuf = (unsigned char *)Tcl_Alloc(nByte); + zBuf = (unsigned char *)Tcl_AttemptAlloc(nByte); + if( zBuf==0 ){ + Tcl_AppendResult(interp, "out of memory", 0); + return TCL_ERROR; + } } rc = sqlite3_blob_read(pBlob, zBuf, nByte, iOffset); if( rc==SQLITE_OK ){ diff --git a/src/test_config.c b/src/test_config.c index ff9964157e..e275bac632 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -746,6 +746,8 @@ Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY); LINKVAR( DEFAULT_CACHE_SIZE ); LINKVAR( DEFAULT_PAGE_SIZE ); LINKVAR( DEFAULT_FILE_FORMAT ); + LINKVAR( DEFAULT_SYNCHRONOUS ); + LINKVAR( DEFAULT_WAL_SYNCHRONOUS ); LINKVAR( MAX_ATTACHED ); LINKVAR( MAX_DEFAULT_PAGE_SIZE ); LINKVAR( MAX_WORKER_THREADS ); diff --git a/src/test_delete.c b/src/test_delete.c index 731f11e257..ca61965b27 100644 --- a/src/test_delete.c +++ b/src/test_delete.c @@ -80,7 +80,7 @@ static int sqlite3DeleteUnlinkIfExists(const char *zFile, int *pbExists){ ** Delete the database file identified by the string argument passed to this ** function. The string must contain a filename, not an SQLite URI. */ -int sqlite3_delete_database( +SQLITE_API int sqlite3_delete_database( const char *zFile /* File to delete */ ){ char *zBuf; /* Buffer to sprintf() filenames to */ diff --git a/src/test_journal.c b/src/test_journal.c index 4e63bccf76..8a449e888c 100644 --- a/src/test_journal.c +++ b/src/test_journal.c @@ -557,6 +557,9 @@ static int jtWrite( u32 pgno = (u32)(iOfst/p->nPagesize + 1); assert( (iAmt==1||iAmt==(int)p->nPagesize) && ((iOfst+iAmt)%p->nPagesize)==0 ); + /* The following assert() statements may fail if this layer is used + ** with a connection in "PRAGMA synchronous=off" mode. If they + ** fail with sync=normal or sync=full, this may indicate problem. */ assert( pgno<=p->nPage || p->nSync>0 ); assert( pgno>p->nPage || sqlite3BitvecTest(p->pWritable, pgno) ); } diff --git a/src/test_windirent.c b/src/test_windirent.c index 044bc7f414..ca78d345d9 100644 --- a/src/test_windirent.c +++ b/src/test_windirent.c @@ -63,6 +63,7 @@ LPDIR opendir( dirname = windirent_getenv("SystemDrive"); } + memset(&data, 0, sizeof(struct _finddata_t)); _snprintf(data.name, namesize, "%s\\*", dirname); dirp->d_handle = _findfirst(data.name, &data); @@ -71,12 +72,18 @@ LPDIR opendir( return NULL; } - /* TODO: Remove this block to allow hidden and system files. */ - if( data.attrib&_A_HIDDEN || data.attrib&_A_SYSTEM ){ + /* TODO: Remove this block to allow hidden and/or system files. */ + if( is_filtered(data) ){ +next: + + memset(&data, 0, sizeof(struct _finddata_t)); if( _findnext(dirp->d_handle, &data)==-1 ){ closedir(dirp); return NULL; } + + /* TODO: Remove this block to allow hidden and/or system files. */ + if( is_filtered(data) ) goto next; } dirp->d_first.d_attributes = data.attrib; @@ -105,11 +112,11 @@ LPDIRENT readdir( next: + memset(&data, 0, sizeof(struct _finddata_t)); if( _findnext(dirp->d_handle, &data)==-1 ) return NULL; - /* TODO: Remove this block to allow hidden and system files. */ - if( data.attrib&_A_HIDDEN ) goto next; - if( data.attrib&_A_SYSTEM ) goto next; + /* TODO: Remove this block to allow hidden and/or system files. */ + if( is_filtered(data) ) goto next; dirp->d_next.d_ino++; dirp->d_next.d_attributes = data.attrib; @@ -146,14 +153,14 @@ INT readdir_r( next: + memset(&data, 0, sizeof(struct _finddata_t)); if( _findnext(dirp->d_handle, &data)==-1 ){ *result = NULL; return ENOENT; } - /* TODO: Remove this block to allow hidden and system files. */ - if( data.attrib&_A_HIDDEN ) goto next; - if( data.attrib&_A_SYSTEM ) goto next; + /* TODO: Remove this block to allow hidden and/or system files. */ + if( is_filtered(data) ) goto next; entry->d_ino = (ino_t)-1; /* not available */ entry->d_attributes = data.attrib; diff --git a/src/test_windirent.h b/src/test_windirent.h index be454988e3..578e2a7c22 100644 --- a/src/test_windirent.h +++ b/src/test_windirent.h @@ -92,6 +92,17 @@ struct DIR { DIRENT d_next; /* DIRENT constructed based on "_findnext". */ }; +/* +** Provide a macro, for use by the implementation, to determine if a +** particular directory entry should be skipped over when searching for +** the next directory entry that should be returned by the readdir() or +** readdir_r() functions. +*/ + +#ifndef is_filtered +# define is_filtered(a) ((((a).attrib)&_A_HIDDEN) || (((a).attrib)&_A_SYSTEM)) +#endif + /* ** Provide the function prototype for the POSIX compatiable getenv() ** function. This function is not thread-safe. diff --git a/src/tokenize.c b/src/tokenize.c index c400dcd554..578ddc4f00 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -475,12 +475,15 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){ */ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ int nErr = 0; /* Number of errors encountered */ - int i; /* Loop counter */ void *pEngine; /* The LEMON-generated LALR(1) parser */ + int n = 0; /* Length of the next token token */ int tokenType; /* type of the next token */ int lastTokenParsed = -1; /* type of the previous token */ sqlite3 *db = pParse->db; /* The database connection */ int mxSqlLen; /* Max length of an SQL string */ +#ifdef sqlite3Parser_ENGINEALWAYSONSTACK + unsigned char zSpace[sizeof(yyParser)]; /* Space for parser engine object */ +#endif assert( zSql!=0 ); mxSqlLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH]; @@ -489,25 +492,27 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ } pParse->rc = SQLITE_OK; pParse->zTail = zSql; - i = 0; assert( pzErrMsg!=0 ); /* sqlite3ParserTrace(stdout, "parser: "); */ +#ifdef sqlite3Parser_ENGINEALWAYSONSTACK + pEngine = zSpace; + sqlite3ParserInit(pEngine); +#else pEngine = sqlite3ParserAlloc(sqlite3Malloc); if( pEngine==0 ){ sqlite3OomFault(db); return SQLITE_NOMEM_BKPT; } +#endif assert( pParse->pNewTable==0 ); assert( pParse->pNewTrigger==0 ); assert( pParse->nVar==0 ); assert( pParse->pVList==0 ); while( 1 ){ - assert( i>=0 ); - if( zSql[i]!=0 ){ - pParse->sLastToken.z = &zSql[i]; - pParse->sLastToken.n = sqlite3GetToken((u8*)&zSql[i],&tokenType); - i += pParse->sLastToken.n; - if( i>mxSqlLen ){ + if( zSql[0]!=0 ){ + n = sqlite3GetToken((u8*)zSql, &tokenType); + mxSqlLen -= n; + if( mxSqlLen<0 ){ pParse->rc = SQLITE_TOOBIG; break; } @@ -521,6 +526,7 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ }else{ tokenType = TK_SEMI; } + zSql -= n; } if( tokenType>=TK_SPACE ){ assert( tokenType==TK_SPACE || tokenType==TK_ILLEGAL ); @@ -529,18 +535,21 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ break; } if( tokenType==TK_ILLEGAL ){ - sqlite3ErrorMsg(pParse, "unrecognized token: \"%T\"", - &pParse->sLastToken); + sqlite3ErrorMsg(pParse, "unrecognized token: \"%.*s\"", n, zSql); break; } + zSql += n; }else{ + pParse->sLastToken.z = zSql; + pParse->sLastToken.n = n; sqlite3Parser(pEngine, tokenType, pParse->sLastToken, pParse); lastTokenParsed = tokenType; + zSql += n; if( pParse->rc!=SQLITE_OK || db->mallocFailed ) break; } } assert( nErr==0 ); - pParse->zTail = &zSql[i]; + pParse->zTail = zSql; #ifdef YYTRACKMAXSTACKDEPTH sqlite3_mutex_enter(sqlite3MallocMutex()); sqlite3StatusHighwater(SQLITE_STATUS_PARSER_STACK, @@ -548,7 +557,11 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ ); sqlite3_mutex_leave(sqlite3MallocMutex()); #endif /* YYDEBUG */ +#ifdef sqlite3Parser_ENGINEALWAYSONSTACK + sqlite3ParserFinalize(pEngine); +#else sqlite3ParserFree(pEngine, sqlite3_free); +#endif if( db->mallocFailed ){ pParse->rc = SQLITE_NOMEM_BKPT; } diff --git a/src/treeview.c b/src/treeview.c index 0ea512b5c6..61d4626a22 100644 --- a/src/treeview.c +++ b/src/treeview.c @@ -126,6 +126,10 @@ void sqlite3TreeViewWith(TreeView *pView, const With *pWith, u8 moreToFollow){ void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){ int n = 0; int cnt = 0; + if( p==0 ){ + sqlite3TreeViewLine(pView, "nil-SELECT"); + return; + } pView = sqlite3TreeViewPush(pView, moreToFollow); if( p->pWith ){ sqlite3TreeViewWith(pView, p->pWith, 1); diff --git a/src/update.c b/src/update.c index 5f89e31dac..8fca7eb988 100644 --- a/src/update.c +++ b/src/update.c @@ -71,12 +71,12 @@ void sqlite3ColumnDefault(Vdbe *v, Table *pTab, int i, int iReg){ if( pValue ){ sqlite3VdbeAppendP4(v, pValue, P4_MEM); } -#ifndef SQLITE_OMIT_FLOATING_POINT - if( pTab->aCol[i].affinity==SQLITE_AFF_REAL ){ - sqlite3VdbeAddOp1(v, OP_RealAffinity, iReg); - } -#endif } +#ifndef SQLITE_OMIT_FLOATING_POINT + if( pTab->aCol[i].affinity==SQLITE_AFF_REAL ){ + sqlite3VdbeAddOp1(v, OP_RealAffinity, iReg); + } +#endif } /* @@ -105,7 +105,7 @@ void sqlite3Update( int iDataCur; /* Cursor for the canonical data btree */ int iIdxCur; /* Cursor for the first index */ sqlite3 *db; /* The database structure */ - int *aRegIdx = 0; /* One register assigned to each index to be updated */ + int *aRegIdx = 0; /* First register in array assigned to each index */ int *aXRef = 0; /* aXRef[i] is the index in pChanges->a[] of the ** an expression for the i-th column of the table. ** aXRef[i]==-1 if the i-th column is not changed. */ @@ -117,10 +117,11 @@ void sqlite3Update( AuthContext sContext; /* The authorization context */ NameContext sNC; /* The name-context to resolve expressions in */ int iDb; /* Database containing the table being updated */ - int okOnePass; /* True for one-pass algorithm without the FIFO */ + int eOnePass; /* ONEPASS_XXX value from where.c */ int hasFK; /* True if foreign key processing is required */ int labelBreak; /* Jump here to break out of UPDATE loop */ int labelContinue; /* Jump here to continue next step of UPDATE loop */ + int flags; /* Flags for sqlite3WhereBegin() */ #ifndef SQLITE_OMIT_TRIGGER int isView; /* True when updating a view (INSTEAD OF trigger) */ @@ -131,6 +132,10 @@ void sqlite3Update( int iEph = 0; /* Ephemeral table holding all primary key values */ int nKey = 0; /* Number of elements in regKey for WITHOUT ROWID */ int aiCurOnePass[2]; /* The write cursors opened by WHERE_ONEPASS */ + int addrOpen = 0; /* Address of OP_OpenEphemeral */ + int iPk = 0; /* First of nPk cells holding PRIMARY KEY value */ + i16 nPk = 0; /* Number of components of the PRIMARY KEY */ + int bReplace = 0; /* True if REPLACE conflict resolution might happen */ /* Register Allocations */ int regRowCount = 0; /* A count of rows changed */ @@ -290,6 +295,11 @@ void sqlite3Update( if( iIdxCol<0 || aXRef[iIdxCol]>=0 ){ reg = ++pParse->nMem; pParse->nMem += pIdx->nColumn; + if( (onError==OE_Replace) + || (onError==OE_Default && pIdx->onError==OE_Replace) + ){ + bReplace = 1; + } break; } } @@ -297,6 +307,11 @@ void sqlite3Update( if( reg==0 ) aToOpen[j+1] = 0; aRegIdx[j] = reg; } + if( bReplace ){ + /* If REPLACE conflict resolution might be invoked, open cursors on all + ** indexes in case they are needed to delete records. */ + memset(aToOpen, 1, nIdx+1); + } /* Begin generating code. */ v = sqlite3GetVdbe(pParse); @@ -349,51 +364,79 @@ void sqlite3Update( } #endif - /* Begin the database scan - */ + /* Initialize the count of updated rows */ + if( (db->flags & SQLITE_CountRows) && !pParse->pTriggerTab ){ + regRowCount = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 0, regRowCount); + } + if( HasRowid(pTab) ){ sqlite3VdbeAddOp3(v, OP_Null, 0, regRowSet, regOldRowid); - pWInfo = sqlite3WhereBegin( - pParse, pTabList, pWhere, 0, 0, - WHERE_ONEPASS_DESIRED | WHERE_SEEK_TABLE, iIdxCur - ); - if( pWInfo==0 ) goto update_cleanup; - okOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); - - /* Remember the rowid of every item to be updated. - */ - sqlite3VdbeAddOp2(v, OP_Rowid, iDataCur, regOldRowid); - if( !okOnePass ){ - sqlite3VdbeAddOp2(v, OP_RowSetAdd, regRowSet, regOldRowid); - } - - /* End the database scan loop. - */ - sqlite3WhereEnd(pWInfo); }else{ - int iPk; /* First of nPk memory cells holding PRIMARY KEY value */ - i16 nPk; /* Number of components of the PRIMARY KEY */ - int addrOpen; /* Address of the OpenEphemeral instruction */ - assert( pPk!=0 ); nPk = pPk->nKeyCol; iPk = pParse->nMem+1; pParse->nMem += nPk; regKey = ++pParse->nMem; iEph = pParse->nTab++; + sqlite3VdbeAddOp2(v, OP_Null, 0, iPk); addrOpen = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iEph, nPk); sqlite3VdbeSetP4KeyInfo(pParse, pPk); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, - WHERE_ONEPASS_DESIRED, iIdxCur); - if( pWInfo==0 ) goto update_cleanup; - okOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); + } + + /* Begin the database scan. + ** + ** Do not consider a single-pass strategy for a multi-row update if + ** there are any triggers or foreign keys to process, or rows may + ** be deleted as a result of REPLACE conflict handling. Any of these + ** things might disturb a cursor being used to scan through the table + ** or index, causing a single-pass approach to malfunction. */ + flags = WHERE_ONEPASS_DESIRED|WHERE_SEEK_UNIQ_TABLE; + if( !pParse->nested && !pTrigger && !hasFK && !chngKey && !bReplace ){ + flags |= WHERE_ONEPASS_MULTIROW; + } + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, flags, iIdxCur); + if( pWInfo==0 ) goto update_cleanup; + + /* A one-pass strategy that might update more than one row may not + ** be used if any column of the index used for the scan is being + ** updated. Otherwise, if there is an index on "b", statements like + ** the following could create an infinite loop: + ** + ** UPDATE t1 SET b=b+1 WHERE b>? + ** + ** Fall back to ONEPASS_OFF if where.c has selected a ONEPASS_MULTI + ** strategy that uses an index for which one or more columns are being + ** updated. */ + eOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); + if( eOnePass==ONEPASS_MULTI ){ + int iCur = aiCurOnePass[1]; + if( iCur>=0 && iCur!=iDataCur && aToOpen[iCur-iBaseCur] ){ + eOnePass = ONEPASS_OFF; + } + assert( iCur!=iDataCur || !HasRowid(pTab) ); + } + + if( HasRowid(pTab) ){ + /* Read the rowid of the current row of the WHERE scan. In ONEPASS_OFF + ** mode, write the rowid into the FIFO. In either of the one-pass modes, + ** leave it in register regOldRowid. */ + sqlite3VdbeAddOp2(v, OP_Rowid, iDataCur, regOldRowid); + if( eOnePass==ONEPASS_OFF ){ + sqlite3VdbeAddOp2(v, OP_RowSetAdd, regRowSet, regOldRowid); + } + }else{ + /* Read the PK of the current row into an array of registers. In + ** ONEPASS_OFF mode, serialize the array into a record and store it in + ** the ephemeral table. Or, in ONEPASS_SINGLE or MULTI mode, change + ** the OP_OpenEphemeral instruction to a Noop (the ephemeral table + ** is not required) and leave the PK fields in the array of registers. */ for(i=0; iaiColumn[i]>=0 ); - sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, pPk->aiColumn[i], - iPk+i); + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur,pPk->aiColumn[i],iPk+i); } - if( okOnePass ){ + if( eOnePass ){ sqlite3VdbeChangeToNoop(v, addrOpen); nKey = nPk; regKey = iPk; @@ -402,50 +445,42 @@ void sqlite3Update( sqlite3IndexAffinityStr(db, pPk), nPk); sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iEph, regKey, iPk, nPk); } - sqlite3WhereEnd(pWInfo); } - /* Initialize the count of updated rows - */ - if( (db->flags & SQLITE_CountRows) && !pParse->pTriggerTab ){ - regRowCount = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Integer, 0, regRowCount); + if( eOnePass!=ONEPASS_MULTI ){ + sqlite3WhereEnd(pWInfo); } labelBreak = sqlite3VdbeMakeLabel(v); if( !isView ){ - /* - ** Open every index that needs updating. Note that if any - ** index could potentially invoke a REPLACE conflict resolution - ** action, then we need to open all indices because we might need - ** to be deleting some records. - */ - if( onError==OE_Replace ){ - memset(aToOpen, 1, nIdx+1); - }else{ - for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - if( pIdx->onError==OE_Replace ){ - memset(aToOpen, 1, nIdx+1); - break; - } - } - } - if( okOnePass ){ + int addrOnce = 0; + + /* Open every index that needs updating. */ + if( eOnePass!=ONEPASS_OFF ){ if( aiCurOnePass[0]>=0 ) aToOpen[aiCurOnePass[0]-iBaseCur] = 0; if( aiCurOnePass[1]>=0 ) aToOpen[aiCurOnePass[1]-iBaseCur] = 0; } + + if( eOnePass==ONEPASS_MULTI && (nIdx-(aiCurOnePass[1]>=0))>0 ){ + addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + } sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, 0, iBaseCur, aToOpen, 0, 0); + if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); } /* Top of the update loop */ - if( okOnePass ){ - if( aToOpen[iDataCur-iBaseCur] && !isView ){ + if( eOnePass!=ONEPASS_OFF ){ + if( !isView && aiCurOnePass[0]!=iDataCur && aiCurOnePass[1]!=iDataCur ){ assert( pPk ); sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelBreak, regKey, nKey); VdbeCoverageNeverTaken(v); } - labelContinue = labelBreak; + if( eOnePass==ONEPASS_SINGLE ){ + labelContinue = labelBreak; + }else{ + labelContinue = sqlite3VdbeMakeLabel(v); + } sqlite3VdbeAddOp2(v, OP_IsNull, pPk ? regKey : regOldRowid, labelBreak); VdbeCoverageIf(v, pPk==0); VdbeCoverageIf(v, pPk!=0); @@ -570,7 +605,6 @@ void sqlite3Update( if( !isView ){ int addr1 = 0; /* Address of jump instruction */ - int bReplace = 0; /* True if REPLACE conflict resolution might happen */ /* Do constraint checks. */ assert( regOldRowid>0 ); @@ -606,14 +640,18 @@ void sqlite3Update( assert( regNew==regNewRowid+1 ); #ifdef SQLITE_ENABLE_PREUPDATE_HOOK sqlite3VdbeAddOp3(v, OP_Delete, iDataCur, - OPFLAG_ISUPDATE | ((hasFK || chngKey || pPk!=0) ? 0 : OPFLAG_ISNOOP), + OPFLAG_ISUPDATE | ((hasFK || chngKey) ? 0 : OPFLAG_ISNOOP), regNewRowid ); + if( eOnePass==ONEPASS_MULTI ){ + assert( hasFK==0 && chngKey==0 ); + sqlite3VdbeChangeP5(v, OPFLAG_SAVEPOSITION); + } if( !pParse->nested ){ sqlite3VdbeAppendP4(v, pTab, P4_TABLE); } #else - if( hasFK || chngKey || pPk!=0 ){ + if( hasFK || chngKey ){ sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, 0); } #endif @@ -626,8 +664,11 @@ void sqlite3Update( } /* Insert the new index entries and the new record. */ - sqlite3CompleteInsertion(pParse, pTab, iDataCur, iIdxCur, - regNewRowid, aRegIdx, 1, 0, 0); + sqlite3CompleteInsertion( + pParse, pTab, iDataCur, iIdxCur, regNewRowid, aRegIdx, + OPFLAG_ISUPDATE | (eOnePass==ONEPASS_MULTI ? OPFLAG_SAVEPOSITION : 0), + 0, 0 + ); /* Do any ON CASCADE, SET NULL or SET DEFAULT operations required to ** handle rows (possibly in other tables) that refer via a foreign key @@ -649,8 +690,11 @@ void sqlite3Update( /* Repeat the above with the next record to be updated, until ** all record selected by the WHERE clause have been updated. */ - if( okOnePass ){ + if( eOnePass==ONEPASS_SINGLE ){ /* Nothing to do at end-of-loop for a single-pass */ + }else if( eOnePass==ONEPASS_MULTI ){ + sqlite3VdbeResolveLabel(v, labelContinue); + sqlite3WhereEnd(pWInfo); }else if( pPk ){ sqlite3VdbeResolveLabel(v, labelContinue); sqlite3VdbeAddOp2(v, OP_Next, iEph, addrTop); VdbeCoverage(v); diff --git a/src/util.c b/src/util.c index ca14ab8526..9b6f4f9e1d 100644 --- a/src/util.c +++ b/src/util.c @@ -1140,13 +1140,11 @@ u32 sqlite3Get4byte(const u8 *p){ u32 x; memcpy(&x,p,4); return x; -#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ - && (GCC_VERSION>=4003000 || CLANG_VERSION>=3000000) +#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 u32 x; memcpy(&x,p,4); return __builtin_bswap32(x); -#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ - && defined(_MSC_VER) && _MSC_VER>=1300 +#elif SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 u32 x; memcpy(&x,p,4); return _byteswap_ulong(x); @@ -1158,12 +1156,10 @@ u32 sqlite3Get4byte(const u8 *p){ void sqlite3Put4byte(unsigned char *p, u32 v){ #if SQLITE_BYTEORDER==4321 memcpy(p,&v,4); -#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ - && (GCC_VERSION>=4003000 || CLANG_VERSION>=3000000) +#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 u32 x = __builtin_bswap32(v); memcpy(p,&x,4); -#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ - && defined(_MSC_VER) && _MSC_VER>=1300 +#elif SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 u32 x = _byteswap_ulong(v); memcpy(p,&x,4); #else @@ -1279,8 +1275,7 @@ int sqlite3SafetyCheckSickOrOk(sqlite3 *db){ ** overflow, leave *pA unchanged and return 1. */ int sqlite3AddInt64(i64 *pA, i64 iB){ -#if !defined(SQLITE_DISABLE_INTRINSIC) \ - && (GCC_VERSION>=5004000 || CLANG_VERSION>=4000000) +#if GCC_VERSION>=5004000 return __builtin_add_overflow(*pA, iB, pA); #else i64 iA = *pA; @@ -1300,8 +1295,7 @@ int sqlite3AddInt64(i64 *pA, i64 iB){ #endif } int sqlite3SubInt64(i64 *pA, i64 iB){ -#if !defined(SQLITE_DISABLE_INTRINSIC) \ - && (GCC_VERSION>=5004000 || CLANG_VERSION>=4000000) +#if GCC_VERSION>=5004000 return __builtin_sub_overflow(*pA, iB, pA); #else testcase( iB==SMALLEST_INT64+1 ); @@ -1316,8 +1310,7 @@ int sqlite3SubInt64(i64 *pA, i64 iB){ #endif } int sqlite3MulInt64(i64 *pA, i64 iB){ -#if !defined(SQLITE_DISABLE_INTRINSIC) \ - && (GCC_VERSION>=5004000 || CLANG_VERSION>=4000000) +#if GCC_VERSION>=5004000 return __builtin_mul_overflow(*pA, iB, pA); #else i64 iA = *pA; diff --git a/src/vacuum.c b/src/vacuum.c index b642361913..2df98a4e21 100644 --- a/src/vacuum.c +++ b/src/vacuum.c @@ -98,8 +98,25 @@ static int execSqlF(sqlite3 *db, char **pzErrMsg, const char *zSql, ...){ */ void sqlite3Vacuum(Parse *pParse, Token *pNm){ Vdbe *v = sqlite3GetVdbe(pParse); - int iDb = pNm ? sqlite3TwoPartName(pParse, pNm, pNm, &pNm) : 0; - if( v && (iDb>=2 || iDb==0) ){ + int iDb = 0; + if( v==0 ) return; + if( pNm ){ +#ifndef SQLITE_BUG_COMPATIBLE_20160819 + /* Default behavior: Report an error if the argument to VACUUM is + ** not recognized */ + iDb = sqlite3TwoPartName(pParse, pNm, pNm, &pNm); + if( iDb<0 ) return; +#else + /* When SQLITE_BUG_COMPATIBLE_20160819 is defined, unrecognized arguments + ** to VACUUM are silently ignored. This is a back-out of a bug fix that + ** occurred on 2016-08-19 (https://www.sqlite.org/src/info/083f9e6270). + ** The buggy behavior is required for binary compatibility with some + ** legacy applications. */ + iDb = sqlite3FindDb(pParse->db, pNm); + if( iDb<0 ) iDb = 0; +#endif + } + if( iDb!=1 ){ sqlite3VdbeAddOp1(v, OP_Vacuum, iDb); sqlite3VdbeUsesBtree(v, iDb); } diff --git a/src/vdbe.c b/src/vdbe.c index 89ab832989..c4c9bccfda 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -403,9 +403,7 @@ void sqlite3VdbeMemPrettyPrint(Mem *pMem, char *zBuf){ }else{ c = 's'; } - - sqlite3_snprintf(100, zCsr, "%c", c); - zCsr += sqlite3Strlen30(zCsr); + *(zCsr++) = c; sqlite3_snprintf(100, zCsr, "%d[", pMem->n); zCsr += sqlite3Strlen30(zCsr); for(i=0; i<16 && in; i++){ @@ -417,9 +415,7 @@ void sqlite3VdbeMemPrettyPrint(Mem *pMem, char *zBuf){ if( z<32 || z>126 ) *zCsr++ = '.'; else *zCsr++ = z; } - - sqlite3_snprintf(100, zCsr, "]%s", encnames[pMem->enc]); - zCsr += sqlite3Strlen30(zCsr); + *(zCsr++) = ']'; if( f & MEM_Zero ){ sqlite3_snprintf(100, zCsr,"+%dz",pMem->u.nZero); zCsr += sqlite3Strlen30(zCsr); @@ -583,8 +579,6 @@ int sqlite3VdbeExec( Mem *pIn2 = 0; /* 2nd input operand */ Mem *pIn3 = 0; /* 3rd input operand */ Mem *pOut = 0; /* Output operand */ - int *aPermute = 0; /* Permutation of columns for OP_Compare */ - i64 lastRowid = db->lastRowid; /* Saved value of the last insert ROWID */ #ifdef VDBE_PROFILE u64 start; /* CPU clock count at start of opcode */ #endif @@ -599,7 +593,6 @@ int sqlite3VdbeExec( } assert( p->rc==SQLITE_OK || (p->rc&0xff)==SQLITE_BUSY ); assert( p->bIsReader || p->readOnly!=0 ); - p->rc = SQLITE_OK; p->iCurrentTime = 0; assert( p->explain==0 ); p->pResultSet = 0; @@ -960,7 +953,6 @@ case OP_Halt: { p->nFrame--; sqlite3VdbeSetChanges(db, p->nChange); pcx = sqlite3VdbeFrameRestore(pFrame); - lastRowid = db->lastRowid; if( pOp->p2==OE_Ignore ){ /* Instruction pcx is the OP_Program that invoked the sub-program ** currently being halted. If the p2 instruction of this OP_Halt @@ -1195,7 +1187,7 @@ case OP_Variable: { /* out2 */ if( sqlite3VdbeMemTooBig(pVar) ){ goto too_big; } - pOut = out2Prerelease(p, pOp); + pOut = &aMem[pOp->p2]; sqlite3VdbeMemShallowCopy(pOut, pVar, MEM_Static); UPDATE_MAX_BLOBSIZE(pOut); break; @@ -1673,23 +1665,21 @@ case OP_Function: { for(i=pCtx->argc-1; i>=0; i--) pCtx->argv[i] = &aMem[pOp->p2+i]; } - memAboutToChange(p, pCtx->pOut); + memAboutToChange(p, pOut); #ifdef SQLITE_DEBUG for(i=0; iargc; i++){ assert( memIsValid(pCtx->argv[i]) ); REGISTER_TRACE(pOp->p2+i, pCtx->argv[i]); } #endif - MemSetTypeFlag(pCtx->pOut, MEM_Null); + MemSetTypeFlag(pOut, MEM_Null); pCtx->fErrorOrAux = 0; - db->lastRowid = lastRowid; (*pCtx->pFunc->xSFunc)(pCtx, pCtx->argc, pCtx->argv);/* IMP: R-24505-23230 */ - lastRowid = db->lastRowid; /* Remember rowid changes made by xSFunc */ /* If the function returned an error, throw an exception */ if( pCtx->fErrorOrAux ){ if( pCtx->isError ){ - sqlite3VdbeError(p, "%s", sqlite3_value_text(pCtx->pOut)); + sqlite3VdbeError(p, "%s", sqlite3_value_text(pOut)); rc = pCtx->isError; } sqlite3VdbeDeleteAuxData(db, &p->pAuxData, pCtx->iOp, pOp->p1); @@ -1698,12 +1688,12 @@ case OP_Function: { /* Copy the result of the function into register P3 */ if( pOut->flags & (MEM_Str|MEM_Blob) ){ - sqlite3VdbeChangeEncoding(pCtx->pOut, encoding); - if( sqlite3VdbeMemTooBig(pCtx->pOut) ) goto too_big; + sqlite3VdbeChangeEncoding(pOut, encoding); + if( sqlite3VdbeMemTooBig(pOut) ) goto too_big; } - REGISTER_TRACE(pOp->p3, pCtx->pOut); - UPDATE_MAX_BLOBSIZE(pCtx->pOut); + REGISTER_TRACE(pOp->p3, pOut); + UPDATE_MAX_BLOBSIZE(pOut); break; } @@ -2140,8 +2130,8 @@ case OP_ElseNotEq: { /* same as TK_ESCAPE, jump */ /* Opcode: Permutation * * * P4 * ** -** Set the permutation used by the OP_Compare operator to be the array -** of integers in P4. +** Set the permutation used by the OP_Compare operator in the next +** instruction. The permutation is stored in the P4 operand. ** ** The permutation is only valid until the next OP_Compare that has ** the OPFLAG_PERMUTE bit set in P5. Typically the OP_Permutation should @@ -2153,7 +2143,8 @@ case OP_ElseNotEq: { /* same as TK_ESCAPE, jump */ case OP_Permutation: { assert( pOp->p4type==P4_INTARRAY ); assert( pOp->p4.ai ); - aPermute = pOp->p4.ai + 1; + assert( pOp[1].opcode==OP_Compare ); + assert( pOp[1].p5 & OPFLAG_PERMUTE ); break; } @@ -2186,15 +2177,24 @@ case OP_Compare: { int idx; CollSeq *pColl; /* Collating sequence to use on this term */ int bRev; /* True for DESCENDING sort order */ + int *aPermute; /* The permutation */ - if( (pOp->p5 & OPFLAG_PERMUTE)==0 ) aPermute = 0; + if( (pOp->p5 & OPFLAG_PERMUTE)==0 ){ + aPermute = 0; + }else{ + assert( pOp>aOp ); + assert( pOp[-1].opcode==OP_Permutation ); + assert( pOp[-1].p4type==P4_INTARRAY ); + aPermute = pOp[-1].p4.ai + 1; + assert( aPermute!=0 ); + } n = pOp->p3; pKeyInfo = pOp->p4.pKeyInfo; assert( n>0 ); assert( pKeyInfo!=0 ); p1 = pOp->p1; p2 = pOp->p2; -#if SQLITE_DEBUG +#ifdef SQLITE_DEBUG if( aPermute ){ int k, mx = 0; for(k=0; kmx ) mx = aPermute[k]; @@ -2220,7 +2220,6 @@ case OP_Compare: { break; } } - aPermute = 0; break; } @@ -2658,8 +2657,13 @@ case OP_Column: { ** 2. the length(X) function if X is a blob, and ** 3. if the content length is zero. ** So we might as well use bogus content rather than reading - ** content from disk. */ - static u8 aZero[8]; /* This is the bogus content */ + ** content from disk. + ** + ** Although sqlite3VdbeSerialGet() may read at most 8 bytes from the + ** buffer passed to it, debugging function VdbeMemPrettyPrint() may + ** read up to 16. So 16 bytes of bogus content is supplied. + */ + static u8 aZero[16]; /* This is the bogus content */ sqlite3VdbeSerialGet(aZero, t, pDest); }else{ rc = sqlite3VdbeMemFromBtree(pC->uc.pCursor, aOffset[p2], len, pDest); @@ -2777,6 +2781,20 @@ case OP_MakeRecord: { }while( zAffinity[0] ); } +#ifdef SQLITE_ENABLE_NULL_TRIM + /* NULLs can be safely trimmed from the end of the record, as long as + ** as the schema format is 2 or more and none of the omitted columns + ** have a non-NULL default value. Also, the record must be left with + ** at least one field. If P5>0 then it will be one more than the + ** index of the right-most column with a non-NULL default value */ + if( pOp->p5 ){ + while( (pLast->flags & MEM_Null)!=0 && nField>pOp->p5 ){ + pLast--; + nField--; + } + } +#endif + /* Loop through the elements that will make up the record to figure ** out how much space is required for the new record. */ @@ -4397,7 +4415,7 @@ case OP_InsertInt: { assert( pC!=0 ); assert( pC->eCurType==CURTYPE_BTREE ); assert( pC->uc.pCursor!=0 ); - assert( pC->isTable ); + assert( (pOp->p5 & OPFLAG_ISNOOP) || pC->isTable ); assert( pOp->p4type==P4_TABLE || pOp->p4type>=P4_STATIC ); REGISTER_TRACE(pOp->p2, pData); @@ -4413,14 +4431,13 @@ case OP_InsertInt: { } if( pOp->p4type==P4_TABLE && HAS_UPDATE_HOOK(db) ){ - assert( pC->isTable ); assert( pC->iDb>=0 ); zDb = db->aDb[pC->iDb].zDbSName; pTab = pOp->p4.pTab; - assert( HasRowid(pTab) ); + assert( (pOp->p5 & OPFLAG_ISNOOP) || HasRowid(pTab) ); op = ((pOp->p5 & OPFLAG_ISUPDATE) ? SQLITE_UPDATE : SQLITE_INSERT); }else{ - pTab = 0; /* Not needed. Silence a comiler warning. */ + pTab = 0; /* Not needed. Silence a compiler warning. */ zDb = 0; /* Not needed. Silence a compiler warning. */ } @@ -4432,10 +4449,11 @@ case OP_InsertInt: { ){ sqlite3VdbePreUpdateHook(p, pC, SQLITE_INSERT, zDb, pTab, x.nKey, pOp->p2); } + if( pOp->p5 & OPFLAG_ISNOOP ) break; #endif if( pOp->p5 & OPFLAG_NCHANGE ) p->nChange++; - if( pOp->p5 & OPFLAG_LASTROWID ) db->lastRowid = lastRowid = x.nKey; + if( pOp->p5 & OPFLAG_LASTROWID ) db->lastRowid = x.nKey; if( pData->flags & MEM_Null ){ x.pData = 0; x.nData = 0; @@ -4452,7 +4470,7 @@ case OP_InsertInt: { } x.pKey = 0; rc = sqlite3BtreeInsert(pC->uc.pCursor, &x, - (pOp->p5 & OPFLAG_APPEND)!=0, seekResult + (pOp->p5 & (OPFLAG_APPEND|OPFLAG_SAVEPOSITION)), seekResult ); pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; @@ -4544,8 +4562,11 @@ case OP_Delete: { #ifdef SQLITE_ENABLE_PREUPDATE_HOOK /* Invoke the pre-update-hook if required. */ - if( db->xPreUpdateCallback && pOp->p4.pTab && HasRowid(pTab) ){ - assert( !(opflags & OPFLAG_ISUPDATE) || (aMem[pOp->p3].flags & MEM_Int) ); + if( db->xPreUpdateCallback && pOp->p4.pTab ){ + assert( !(opflags & OPFLAG_ISUPDATE) + || HasRowid(pTab)==0 + || (aMem[pOp->p3].flags & MEM_Int) + ); sqlite3VdbePreUpdateHook(p, pC, (opflags & OPFLAG_ISUPDATE) ? SQLITE_UPDATE : SQLITE_DELETE, zDb, pTab, pC->movetoTarget, @@ -4663,7 +4684,7 @@ case OP_SorterData: { break; } -/* Opcode: RowData P1 P2 * * * +/* Opcode: RowData P1 P2 P3 * * ** Synopsis: r[P2]=data ** ** Write into register P2 the complete row content for the row at @@ -4677,14 +4698,26 @@ case OP_SorterData: { ** ** If the P1 cursor must be pointing to a valid row (not a NULL row) ** of a real table, not a pseudo-table. +** +** If P3!=0 then this opcode is allowed to make an ephermeral pointer +** into the database page. That means that the content of the output +** register will be invalidated as soon as the cursor moves - including +** moves caused by other cursors that "save" the the current cursors +** position in order that they can write to the same table. If P3==0 +** then a copy of the data is made into memory. P3!=0 is faster, but +** P3==0 is safer. +** +** If P3!=0 then the content of the P2 register is unsuitable for use +** in OP_Result and any OP_Result will invalidate the P2 register content. +** The P2 register content is invalidated by opcodes like OP_Function or +** by any use of another cursor pointing to the same table. */ case OP_RowData: { VdbeCursor *pC; BtCursor *pCrsr; u32 n; - pOut = &aMem[pOp->p2]; - memAboutToChange(p, pOut); + pOut = out2Prerelease(p, pOp); assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; @@ -4715,14 +4748,9 @@ case OP_RowData: { goto too_big; } testcase( n==0 ); - if( sqlite3VdbeMemClearAndResize(pOut, MAX(n,32)) ){ - goto no_mem; - } - pOut->n = n; - MemSetTypeFlag(pOut, MEM_Blob); - rc = sqlite3BtreePayload(pCrsr, 0, n, pOut->z); + rc = sqlite3VdbeMemFromBtree(pCrsr, 0, n, pOut); if( rc ) goto abort_due_to_error; - pOut->enc = SQLITE_UTF8; /* In case the blob is ever cast to text */ + if( !pOp->p3 ) Deephemeralize(pOut); UPDATE_MAX_BLOBSIZE(pOut); REGISTER_TRACE(pOp->p2, pOut); break; @@ -4851,6 +4879,33 @@ case OP_Last: { /* jump */ break; } +/* Opcode: IfSmaller P1 P2 P3 * * +** +** Estimate the number of rows in the table P1. Jump to P2 if that +** estimate is less than approximately 2**(0.1*P3). +*/ +case OP_IfSmaller: { /* jump */ + VdbeCursor *pC; + BtCursor *pCrsr; + int res; + i64 sz; + + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + pCrsr = pC->uc.pCursor; + assert( pCrsr ); + rc = sqlite3BtreeFirst(pCrsr, &res); + if( rc ) goto abort_due_to_error; + if( res==0 ){ + sz = sqlite3BtreeRowCountEst(pCrsr); + if( ALWAYS(sz>=0) && sqlite3LogEst((u64)sz)p3 ) res = 1; + } + VdbeBranchTaken(res!=0,2); + if( res ) goto jump_to_p2; + break; +} + /* Opcode: SorterSort P1 P2 * * * ** @@ -5110,7 +5165,7 @@ case OP_IdxInsert: { /* in2 */ x.aMem = aMem + pOp->p3; x.nMem = (u16)pOp->p4.i; rc = sqlite3BtreeInsert(pC->uc.pCursor, &x, - (pOp->p5 & OPFLAG_APPEND)!=0, + (pOp->p5 & (OPFLAG_APPEND|OPFLAG_SAVEPOSITION)), ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0) ); assert( pC->deferredMoveto==0 ); @@ -5232,7 +5287,6 @@ case OP_IdxRowid: { /* out2 */ }else{ pOut = out2Prerelease(p, pOp); pOut->u.i = rowid; - pOut->flags = MEM_Int; } }else{ assert( pOp->opcode==OP_IdxRowid ); @@ -5496,6 +5550,18 @@ case OP_CreateTable: { /* out2 */ break; } +/* Opcode: SqlExec * * * P4 * +** +** Run the SQL statement or statements specified in the P4 string. +*/ +case OP_SqlExec: { + db->nSqlExec++; + rc = sqlite3_exec(db, pOp->p4.z, 0, 0, 0); + db->nSqlExec--; + if( rc ) goto abort_due_to_error; + break; +} + /* Opcode: ParseSchema P1 * * P4 * ** ** Read and parse all entries from the SQLITE_MASTER table of database P1 @@ -5616,7 +5682,7 @@ case OP_DropTrigger: { ** register P1 the text of an error message describing any problems. ** If no problems are found, store a NULL in register P1. ** -** The register P3 contains the maximum number of allowed errors. +** The register P3 contains one less than the maximum number of allowed errors. ** At most reg(P3) errors will be reported. ** In other words, the analysis stops as soon as reg(P1) errors are ** seen. Reg(P1) is updated with the number of errors remaining. @@ -5649,14 +5715,14 @@ case OP_IntegrityCk: { assert( pOp->p5nDb ); assert( DbMaskTest(p->btreeMask, pOp->p5) ); z = sqlite3BtreeIntegrityCheck(db->aDb[pOp->p5].pBt, aRoot, nRoot, - (int)pnErr->u.i, &nErr); - pnErr->u.i -= nErr; + (int)pnErr->u.i+1, &nErr); sqlite3VdbeMemSetNull(pIn1); if( nErr==0 ){ assert( z==0 ); }else if( z==0 ){ goto no_mem; }else{ + pnErr->u.i -= nErr-1; sqlite3VdbeMemSetStr(pIn1, z, -1, SQLITE_UTF8, sqlite3_free); } UPDATE_MAX_BLOBSIZE(pIn1); @@ -5874,7 +5940,7 @@ case OP_Program: { /* jump */ p->nFrame++; pFrame->pParent = p->pFrame; - pFrame->lastRowid = lastRowid; + pFrame->lastRowid = db->lastRowid; pFrame->nChange = p->nChange; pFrame->nDbChange = p->db->nChange; assert( pFrame->pAuxData==0 ); @@ -6823,7 +6889,7 @@ case OP_VUpdate: { sqlite3VtabImportErrmsg(p, pVtab); if( rc==SQLITE_OK && pOp->p1 ){ assert( nArg>1 && apArg[0] && (apArg[0]->flags&MEM_Null) ); - db->lastRowid = lastRowid = rowid; + db->lastRowid = rowid; } if( (rc&0xff)==SQLITE_CONSTRAINT && pOp->p4.pVtab->bConstraint ){ if( pOp->p5==OE_Ignore ){ @@ -6923,7 +6989,11 @@ case OP_Init: { /* jump */ sqlite3_free(z); }else #endif - { + if( db->nVdbeExec>1 ){ + char *z = sqlite3MPrintf(db, "-- %s", zTrace); + (void)db->xTrace(SQLITE_TRACE_STMT, db->pTraceArg, p, z); + sqlite3DbFree(db, z); + }else{ (void)db->xTrace(SQLITE_TRACE_STMT, db->pTraceArg, p, zTrace); } } @@ -7059,7 +7129,6 @@ abort_due_to_error: ** release the mutexes on btrees that were acquired at the ** top. */ vdbe_return: - db->lastRowid = lastRowid; testcase( nVmStep>0 ); p->aCounter[SQLITE_STMTSTATUS_VM_STEP] += (int)nVmStep; sqlite3VdbeLeave(p); diff --git a/src/vdbe.h b/src/vdbe.h index feaf116807..a35f3be344 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -41,8 +41,7 @@ typedef struct SubProgram SubProgram; struct VdbeOp { u8 opcode; /* What operation to perform */ signed char p4type; /* One of the P4_xxx constants for p4 */ - u8 notUsed1; - u8 p5; /* Fifth parameter is an unsigned character */ + u16 p5; /* Fifth parameter is an unsigned 16-bit integer */ int p1; /* First operand */ int p2; /* Second parameter (often the jump destination) */ int p3; /* The third parameter */ @@ -194,7 +193,7 @@ void sqlite3VdbeChangeOpcode(Vdbe*, u32 addr, u8); void sqlite3VdbeChangeP1(Vdbe*, u32 addr, int P1); void sqlite3VdbeChangeP2(Vdbe*, u32 addr, int P2); void sqlite3VdbeChangeP3(Vdbe*, u32 addr, int P3); -void sqlite3VdbeChangeP5(Vdbe*, u8 P5); +void sqlite3VdbeChangeP5(Vdbe*, u16 P5); void sqlite3VdbeJumpHere(Vdbe*, int addr); int sqlite3VdbeChangeToNoop(Vdbe*, int addr); int sqlite3VdbeDeletePriorOpcode(Vdbe*, u8 op); diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 3e6bb06889..989cdfd346 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -434,6 +434,7 @@ struct PreUpdate { i64 iKey2; /* Second key value passed to hook */ Mem *aNew; /* Array of new.* values */ Table *pTab; /* Schema object being upated */ + Index *pPk; /* PK index if pTab is WITHOUT ROWID */ }; /* diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 32794de9a0..1f2699dc4d 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -154,7 +154,8 @@ int sqlite3_clear_bindings(sqlite3_stmt *pStmt){ sqlite3VdbeMemRelease(&p->aVar[i]); p->aVar[i].flags = MEM_Null; } - if( p->isPrepareV2 && p->expmask ){ + assert( p->isPrepareV2 || p->expmask==0 ); + if( p->expmask ){ p->expired = 1; } sqlite3_mutex_leave(mutex); @@ -1258,9 +1259,8 @@ static int vdbeUnbind(Vdbe *p, int i){ ** as if there had been a schema change, on the first sqlite3_step() call ** following any change to the bindings of that parameter. */ - if( p->isPrepareV2 && - ((i<32 && p->expmask & ((u32)1 << i)) || p->expmask==0xffffffff) - ){ + assert( p->isPrepareV2 || p->expmask==0 ); + if( p->expmask!=0 && (p->expmask & (i>=31 ? 0x80000000 : (u32)1<expired = 1; } return SQLITE_OK; @@ -1523,10 +1523,12 @@ int sqlite3_transfer_bindings(sqlite3_stmt *pFromStmt, sqlite3_stmt *pToStmt){ if( pFrom->nVar!=pTo->nVar ){ return SQLITE_ERROR; } - if( pTo->isPrepareV2 && pTo->expmask ){ + assert( pTo->isPrepareV2 || pTo->expmask==0 ); + if( pTo->expmask ){ pTo->expired = 1; } - if( pFrom->isPrepareV2 && pFrom->expmask ){ + assert( pFrom->isPrepareV2 || pFrom->expmask==0 ); + if( pFrom->expmask ){ pFrom->expired = 1; } return sqlite3TransferBindings(pFromStmt, pToStmt); @@ -1660,6 +1662,7 @@ static UnpackedRecord *vdbeUnpackRecord( */ int sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlite3_value **ppValue){ PreUpdate *p = db->pPreUpdate; + Mem *pMem; int rc = SQLITE_OK; /* Test that this call is being made from within an SQLITE_DELETE or @@ -1668,6 +1671,9 @@ int sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlite3_value **ppValue){ rc = SQLITE_MISUSE_BKPT; goto preupdate_old_out; } + if( p->pPk ){ + iIdx = sqlite3ColumnOfIndex(p->pPk, iIdx); + } if( iIdx>=p->pCsr->nField || iIdx<0 ){ rc = SQLITE_RANGE; goto preupdate_old_out; @@ -1693,17 +1699,14 @@ int sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlite3_value **ppValue){ p->aRecord = aRec; } - if( iIdx>=p->pUnpacked->nField ){ + pMem = *ppValue = &p->pUnpacked->aMem[iIdx]; + if( iIdx==p->pTab->iPKey ){ + sqlite3VdbeMemSetInt64(pMem, p->iKey1); + }else if( iIdx>=p->pUnpacked->nField ){ *ppValue = (sqlite3_value *)columnNullValue(); - }else{ - Mem *pMem = *ppValue = &p->pUnpacked->aMem[iIdx]; - *ppValue = &p->pUnpacked->aMem[iIdx]; - if( iIdx==p->pTab->iPKey ){ - sqlite3VdbeMemSetInt64(pMem, p->iKey1); - }else if( p->pTab->aCol[iIdx].affinity==SQLITE_AFF_REAL ){ - if( pMem->flags & MEM_Int ){ - sqlite3VdbeMemRealify(pMem); - } + }else if( p->pTab->aCol[iIdx].affinity==SQLITE_AFF_REAL ){ + if( pMem->flags & MEM_Int ){ + sqlite3VdbeMemRealify(pMem); } } @@ -1756,6 +1759,9 @@ int sqlite3_preupdate_new(sqlite3 *db, int iIdx, sqlite3_value **ppValue){ rc = SQLITE_MISUSE_BKPT; goto preupdate_new_out; } + if( p->pPk && p->op!=SQLITE_UPDATE ){ + iIdx = sqlite3ColumnOfIndex(p->pPk, iIdx); + } if( iIdx>=p->pCsr->nField || iIdx<0 ){ rc = SQLITE_RANGE; goto preupdate_new_out; @@ -1776,13 +1782,11 @@ int sqlite3_preupdate_new(sqlite3 *db, int iIdx, sqlite3_value **ppValue){ } p->pNewUnpacked = pUnpack; } - if( iIdx>=pUnpack->nField ){ + pMem = &pUnpack->aMem[iIdx]; + if( iIdx==p->pTab->iPKey ){ + sqlite3VdbeMemSetInt64(pMem, p->iKey2); + }else if( iIdx>=pUnpack->nField ){ pMem = (sqlite3_value *)columnNullValue(); - }else{ - pMem = &pUnpack->aMem[iIdx]; - if( iIdx==p->pTab->iPKey ){ - sqlite3VdbeMemSetInt64(pMem, p->iKey2); - } } }else{ /* For an UPDATE, memory cell (p->iNewReg+1+iIdx) contains the required diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 8e912865fa..a393439899 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -57,6 +57,7 @@ void sqlite3VdbeError(Vdbe *p, const char *zFormat, ...){ void sqlite3VdbeSetSql(Vdbe *p, const char *z, int n, int isPrepareV2){ assert( isPrepareV2==1 || isPrepareV2==0 ); if( p==0 ) return; + if( !isPrepareV2 ) p->expmask = 0; #if defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_ENABLE_SQLLOG) if( !isPrepareV2 ) return; #endif @@ -85,6 +86,7 @@ void sqlite3VdbeSwap(Vdbe *pA, Vdbe *pB){ pA->zSql = pB->zSql; pB->zSql = zTmp; pB->isPrepareV2 = pA->isPrepareV2; + pB->expmask = pA->expmask; } /* @@ -115,6 +117,12 @@ static int growOpArray(Vdbe *v, int nOp){ UNUSED_PARAMETER(nOp); #endif + /* Ensure that the size of a VDBE does not grow too large */ + if( nNew > p->db->aLimit[SQLITE_LIMIT_VDBE_OP] ){ + sqlite3OomFault(p->db); + return SQLITE_NOMEM; + } + assert( nOp<=(1024/sizeof(Op)) ); assert( nNew>=(p->nOpAlloc+nOp) ); pNew = sqlite3DbRealloc(p->db, v->aOp, nNew*sizeof(Op)); @@ -783,7 +791,7 @@ void sqlite3VdbeChangeP2(Vdbe *p, u32 addr, int val){ void sqlite3VdbeChangeP3(Vdbe *p, u32 addr, int val){ sqlite3VdbeGetOp(p,addr)->p3 = val; } -void sqlite3VdbeChangeP5(Vdbe *p, u8 p5){ +void sqlite3VdbeChangeP5(Vdbe *p, u16 p5){ assert( p->nOp>0 || p->db->mallocFailed ); if( p->nOp>0 ) p->aOp[p->nOp-1].p5 = p5; } @@ -2514,60 +2522,59 @@ static void checkActiveVdbeCnt(sqlite3 *db){ ** If an IO error occurs, an SQLITE_IOERR_XXX error code is returned. ** Otherwise SQLITE_OK. */ -int sqlite3VdbeCloseStatement(Vdbe *p, int eOp){ +static SQLITE_NOINLINE int vdbeCloseStatement(Vdbe *p, int eOp){ sqlite3 *const db = p->db; int rc = SQLITE_OK; + int i; + const int iSavepoint = p->iStatement-1; - /* If p->iStatement is greater than zero, then this Vdbe opened a - ** statement transaction that should be closed here. The only exception - ** is that an IO error may have occurred, causing an emergency rollback. - ** In this case (db->nStatement==0), and there is nothing to do. - */ - if( db->nStatement && p->iStatement ){ - int i; - const int iSavepoint = p->iStatement-1; + assert( eOp==SAVEPOINT_ROLLBACK || eOp==SAVEPOINT_RELEASE); + assert( db->nStatement>0 ); + assert( p->iStatement==(db->nStatement+db->nSavepoint) ); - assert( eOp==SAVEPOINT_ROLLBACK || eOp==SAVEPOINT_RELEASE); - assert( db->nStatement>0 ); - assert( p->iStatement==(db->nStatement+db->nSavepoint) ); - - for(i=0; inDb; i++){ - int rc2 = SQLITE_OK; - Btree *pBt = db->aDb[i].pBt; - if( pBt ){ - if( eOp==SAVEPOINT_ROLLBACK ){ - rc2 = sqlite3BtreeSavepoint(pBt, SAVEPOINT_ROLLBACK, iSavepoint); - } - if( rc2==SQLITE_OK ){ - rc2 = sqlite3BtreeSavepoint(pBt, SAVEPOINT_RELEASE, iSavepoint); - } - if( rc==SQLITE_OK ){ - rc = rc2; - } - } - } - db->nStatement--; - p->iStatement = 0; - - if( rc==SQLITE_OK ){ + for(i=0; inDb; i++){ + int rc2 = SQLITE_OK; + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ if( eOp==SAVEPOINT_ROLLBACK ){ - rc = sqlite3VtabSavepoint(db, SAVEPOINT_ROLLBACK, iSavepoint); + rc2 = sqlite3BtreeSavepoint(pBt, SAVEPOINT_ROLLBACK, iSavepoint); + } + if( rc2==SQLITE_OK ){ + rc2 = sqlite3BtreeSavepoint(pBt, SAVEPOINT_RELEASE, iSavepoint); } if( rc==SQLITE_OK ){ - rc = sqlite3VtabSavepoint(db, SAVEPOINT_RELEASE, iSavepoint); + rc = rc2; } } + } + db->nStatement--; + p->iStatement = 0; - /* If the statement transaction is being rolled back, also restore the - ** database handles deferred constraint counter to the value it had when - ** the statement transaction was opened. */ + if( rc==SQLITE_OK ){ if( eOp==SAVEPOINT_ROLLBACK ){ - db->nDeferredCons = p->nStmtDefCons; - db->nDeferredImmCons = p->nStmtDefImmCons; + rc = sqlite3VtabSavepoint(db, SAVEPOINT_ROLLBACK, iSavepoint); } + if( rc==SQLITE_OK ){ + rc = sqlite3VtabSavepoint(db, SAVEPOINT_RELEASE, iSavepoint); + } + } + + /* If the statement transaction is being rolled back, also restore the + ** database handles deferred constraint counter to the value it had when + ** the statement transaction was opened. */ + if( eOp==SAVEPOINT_ROLLBACK ){ + db->nDeferredCons = p->nStmtDefCons; + db->nDeferredImmCons = p->nStmtDefImmCons; } return rc; } +int sqlite3VdbeCloseStatement(Vdbe *p, int eOp){ + if( p->db->nStatement && p->iStatement ){ + return vdbeCloseStatement(p, eOp); + } + return SQLITE_OK; +} + /* ** This function is called when a transaction opened by the database @@ -2627,13 +2634,13 @@ int sqlite3VdbeHalt(Vdbe *p){ ** one, or the complete transaction if there is no statement transaction. */ + if( p->magic!=VDBE_MAGIC_RUN ){ + return SQLITE_OK; + } if( db->mallocFailed ){ p->rc = SQLITE_NOMEM_BKPT; } closeAllCursors(p); - if( p->magic!=VDBE_MAGIC_RUN ){ - return SQLITE_OK; - } checkActiveVdbeCnt(db); /* No commit or rollback needed if the program never started or if the @@ -3583,7 +3590,7 @@ void sqlite3VdbeRecordUnpack( p->nField = u; } -#if SQLITE_DEBUG +#ifdef SQLITE_DEBUG /* ** This function compares two index or table record keys in the same way ** as the sqlite3VdbeRecordCompare() routine. Unlike VdbeRecordCompare(), @@ -3688,7 +3695,7 @@ debugCompareEnd: } #endif -#if SQLITE_DEBUG +#ifdef SQLITE_DEBUG /* ** Count the number of fields (a.k.a. columns) in the record given by ** pKey,nKey. The verify that this count is less than or equal to the @@ -4571,8 +4578,8 @@ sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe *v, int iVar, u8 aff){ */ void sqlite3VdbeSetVarmask(Vdbe *v, int iVar){ assert( iVar>0 ); - if( iVar>32 ){ - v->expmask = 0xffffffff; + if( iVar>=32 ){ + v->expmask |= 0x80000000; }else{ v->expmask |= ((u32)1 << (iVar-1)); } @@ -4605,10 +4612,10 @@ void sqlite3VtabImportErrmsg(Vdbe *p, sqlite3_vtab *pVtab){ ** This function is used to free UnpackedRecord structures allocated by ** the vdbeUnpackRecord() function found in vdbeapi.c. */ -static void vdbeFreeUnpacked(sqlite3 *db, UnpackedRecord *p){ +static void vdbeFreeUnpacked(sqlite3 *db, int nField, UnpackedRecord *p){ if( p ){ int i; - for(i=0; inField; i++){ + for(i=0; iaMem[i]; if( pMem->zMalloc ) sqlite3VdbeMemRelease(pMem); } @@ -4641,10 +4648,15 @@ void sqlite3VdbePreUpdateHook( assert( db->pPreUpdate==0 ); memset(&preupdate, 0, sizeof(PreUpdate)); - if( op==SQLITE_UPDATE ){ - iKey2 = v->aMem[iReg].u.i; + if( HasRowid(pTab)==0 ){ + iKey1 = iKey2 = 0; + preupdate.pPk = sqlite3PrimaryKeyIndex(pTab); }else{ - iKey2 = iKey1; + if( op==SQLITE_UPDATE ){ + iKey2 = v->aMem[iReg].u.i; + }else{ + iKey2 = iKey1; + } } assert( pCsr->nField==pTab->nCol @@ -4667,8 +4679,8 @@ void sqlite3VdbePreUpdateHook( db->xPreUpdateCallback(db->pPreUpdateArg, db, op, zDb, zTbl, iKey1, iKey2); db->pPreUpdate = 0; sqlite3DbFree(db, preupdate.aRecord); - vdbeFreeUnpacked(db, preupdate.pUnpacked); - vdbeFreeUnpacked(db, preupdate.pNewUnpacked); + vdbeFreeUnpacked(db, preupdate.keyinfo.nField+1, preupdate.pUnpacked); + vdbeFreeUnpacked(db, preupdate.keyinfo.nField+1, preupdate.pNewUnpacked); if( preupdate.aNew ){ int i; for(i=0; inField; i++){ diff --git a/src/vdbeblob.c b/src/vdbeblob.c index 520d16c985..5692c1c606 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -23,10 +23,9 @@ */ typedef struct Incrblob Incrblob; struct Incrblob { - int flags; /* Copy of "flags" passed to sqlite3_blob_open() */ int nByte; /* Size of open blob, in bytes */ int iOffset; /* Byte offset of blob in cursor data */ - int iCol; /* Table column this handle is open on */ + u16 iCol; /* Table column this handle is open on */ BtCursor *pCsr; /* Cursor pointing at blob row */ sqlite3_stmt *pStmt; /* Statement holding cursor open */ sqlite3 *db; /* The associated database */ @@ -57,17 +56,27 @@ static int blobSeekToRow(Incrblob *p, sqlite3_int64 iRow, char **pzErr){ char *zErr = 0; /* Error message */ Vdbe *v = (Vdbe *)p->pStmt; - /* Set the value of the SQL statements only variable to integer iRow. - ** This is done directly instead of using sqlite3_bind_int64() to avoid - ** triggering asserts related to mutexes. + /* Set the value of register r[1] in the SQL statement to integer iRow. + ** This is done directly as a performance optimization */ - assert( v->aVar[0].flags&MEM_Int ); - v->aVar[0].u.i = iRow; + v->aMem[1].flags = MEM_Int; + v->aMem[1].u.i = iRow; - rc = sqlite3_step(p->pStmt); + /* If the statement has been run before (and is paused at the OP_ResultRow) + ** then back it up to the point where it does the OP_SeekRowid. This could + ** have been down with an extra OP_Goto, but simply setting the program + ** counter is faster. */ + if( v->pc>3 ){ + v->pc = 3; + rc = sqlite3VdbeExec(v); + }else{ + rc = sqlite3_step(p->pStmt); + } if( rc==SQLITE_ROW ){ VdbeCursor *pC = v->apCsr[0]; - u32 type = pC->aType[p->iCol]; + u32 type = pC->nHdrParsed>p->iCol ? pC->aType[p->iCol] : 0; + testcase( pC->nHdrParsed==p->iCol ); + testcase( pC->nHdrParsed==p->iCol+1 ); if( type<12 ){ zErr = sqlite3MPrintf(p->db, "cannot open value of type %s", type==0?"null": type==7?"real": "integer" @@ -112,7 +121,7 @@ int sqlite3_blob_open( const char *zTable, /* The table containing the blob */ const char *zColumn, /* The column containing the blob */ sqlite_int64 iRow, /* The row containing the glob */ - int flags, /* True -> read/write access, false -> read-only */ + int wrFlag, /* True -> read/write access, false -> read-only */ sqlite3_blob **ppBlob /* Handle for accessing the blob returned here */ ){ int nAttempt = 0; @@ -134,7 +143,7 @@ int sqlite3_blob_open( return SQLITE_MISUSE_BKPT; } #endif - flags = !!flags; /* flags = (flags ? 1 : 0); */ + wrFlag = !!wrFlag; /* wrFlag = (wrFlag ? 1 : 0); */ sqlite3_mutex_enter(db->mutex); @@ -194,9 +203,8 @@ int sqlite3_blob_open( /* If the value is being opened for writing, check that the ** column is not indexed, and that it is not part of a foreign key. - ** It is against the rules to open a column to which either of these - ** descriptions applies for writing. */ - if( flags ){ + */ + if( wrFlag ){ const char *zFault = 0; Index *pIdx; #ifndef SQLITE_OMIT_FOREIGN_KEY @@ -257,18 +265,17 @@ int sqlite3_blob_open( static const VdbeOpList openBlob[] = { {OP_TableLock, 0, 0, 0}, /* 0: Acquire a read or write lock */ {OP_OpenRead, 0, 0, 0}, /* 1: Open a cursor */ - {OP_Variable, 1, 1, 0}, /* 2: Move ?1 into reg[1] */ - {OP_NotExists, 0, 7, 1}, /* 3: Seek the cursor */ - {OP_Column, 0, 0, 1}, /* 4 */ - {OP_ResultRow, 1, 0, 0}, /* 5 */ - {OP_Goto, 0, 2, 0}, /* 6 */ - {OP_Halt, 0, 0, 0}, /* 7 */ + /* blobSeekToRow() will initialize r[1] to the desired rowid */ + {OP_NotExists, 0, 5, 1}, /* 2: Seek the cursor to rowid=r[1] */ + {OP_Column, 0, 0, 1}, /* 3 */ + {OP_ResultRow, 1, 0, 0}, /* 4 */ + {OP_Halt, 0, 0, 0}, /* 5 */ }; Vdbe *v = (Vdbe *)pBlob->pStmt; int iDb = sqlite3SchemaToIndex(db, pTab->pSchema); VdbeOp *aOp; - sqlite3VdbeAddOp4Int(v, OP_Transaction, iDb, flags, + sqlite3VdbeAddOp4Int(v, OP_Transaction, iDb, wrFlag, pTab->pSchema->schema_cookie, pTab->pSchema->iGeneration); sqlite3VdbeChangeP5(v, 1); @@ -285,7 +292,7 @@ int sqlite3_blob_open( #else aOp[0].p1 = iDb; aOp[0].p2 = pTab->tnum; - aOp[0].p3 = flags; + aOp[0].p3 = wrFlag; sqlite3VdbeChangeP4(v, 1, pTab->zName, P4_TRANSIENT); } if( db->mallocFailed==0 ){ @@ -293,7 +300,7 @@ int sqlite3_blob_open( /* Remove either the OP_OpenWrite or OpenRead. Set the P2 ** parameter of the other to pTab->tnum. */ - if( flags ) aOp[1].opcode = OP_OpenWrite; + if( wrFlag ) aOp[1].opcode = OP_OpenWrite; aOp[1].p2 = pTab->tnum; aOp[1].p3 = iDb; @@ -306,23 +313,21 @@ int sqlite3_blob_open( */ aOp[1].p4type = P4_INT32; aOp[1].p4.i = pTab->nCol+1; - aOp[4].p2 = pTab->nCol; + aOp[3].p2 = pTab->nCol; - pParse->nVar = 1; + pParse->nVar = 0; pParse->nMem = 1; pParse->nTab = 1; sqlite3VdbeMakeReady(v, pParse); } } - pBlob->flags = flags; pBlob->iCol = iCol; pBlob->db = db; sqlite3BtreeLeaveAll(db); if( db->mallocFailed ){ goto blob_open_out; } - sqlite3_bind_int64(pBlob->pStmt, 1, iRow); rc = blobSeekToRow(pBlob, iRow, &zErr); } while( (++nAttempt)pSchema); assert( iDb>=0 ); - pTable->tabFlags |= TF_Virtual; - pTable->nModuleArg = 0; + assert( pTable->nModuleArg==0 ); addModuleArgument(db, pTable, sqlite3NameFromToken(db, pModuleName)); addModuleArgument(db, pTable, 0); addModuleArgument(db, pTable, sqlite3DbStrDup(db, pTable->zName)); @@ -628,7 +627,7 @@ int sqlite3VtabCallConnect(Parse *pParse, Table *pTab){ int rc; assert( pTab ); - if( (pTab->tabFlags & TF_Virtual)==0 || sqlite3GetVTable(db, pTab) ){ + if( !IsVirtual(pTab) || sqlite3GetVTable(db, pTab) ){ return SQLITE_OK; } @@ -698,7 +697,7 @@ int sqlite3VtabCallCreate(sqlite3 *db, int iDb, const char *zTab, char **pzErr){ const char *zMod; pTab = sqlite3FindTable(db, zTab, db->aDb[iDb].zDbSName); - assert( pTab && (pTab->tabFlags & TF_Virtual)!=0 && !pTab->pVTable ); + assert( pTab && IsVirtual(pTab) && !pTab->pVTable ); /* Locate the required virtual table module */ zMod = pTab->azModuleArg[0]; @@ -752,7 +751,7 @@ int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ return SQLITE_MISUSE_BKPT; } pTab = pCtx->pTab; - assert( (pTab->tabFlags & TF_Virtual)!=0 ); + assert( IsVirtual(pTab) ); pParse = sqlite3StackAllocZero(db, sizeof(*pParse)); if( pParse==0 ){ @@ -766,7 +765,7 @@ int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ && pParse->pNewTable && !db->mallocFailed && !pParse->pNewTable->pSelect - && (pParse->pNewTable->tabFlags & TF_Virtual)==0 + && !IsVirtual(pParse->pNewTable) ){ if( !pTab->aCol ){ Table *pNew = pParse->pNewTable; @@ -1055,7 +1054,7 @@ FuncDef *sqlite3VtabOverloadFunction( if( pExpr->op!=TK_COLUMN ) return pDef; pTab = pExpr->pTab; if( NEVER(pTab==0) ) return pDef; - if( (pTab->tabFlags & TF_Virtual)==0 ) return pDef; + if( !IsVirtual(pTab) ) return pDef; pVtab = sqlite3GetVTable(db, pTab)->pVtab; assert( pVtab!=0 ); assert( pVtab->pModule!=0 ); @@ -1150,8 +1149,7 @@ int sqlite3VtabEponymousTableInit(Parse *pParse, Module *pMod){ pMod->pEpoTab = pTab; pTab->nTabRef = 1; pTab->pSchema = db->aDb[0].pSchema; - pTab->tabFlags |= TF_Virtual; - pTab->nModuleArg = 0; + assert( pTab->nModuleArg==0 ); pTab->iPKey = -1; addModuleArgument(db, pTab, sqlite3DbStrDup(db, pTab->zName)); addModuleArgument(db, pTab, 0); @@ -1222,7 +1220,7 @@ int sqlite3_vtab_config(sqlite3 *db, int op, ...){ if( !p ){ rc = SQLITE_MISUSE_BKPT; }else{ - assert( p->pTab==0 || (p->pTab->tabFlags & TF_Virtual)!=0 ); + assert( p->pTab==0 || IsVirtual(p->pTab) ); p->pVTable->bConstraint = (u8)va_arg(ap, int); } break; diff --git a/src/walker.c b/src/walker.c index d1b1e96a2d..a7123d8ab0 100644 --- a/src/walker.c +++ b/src/walker.c @@ -27,11 +27,11 @@ ** ** WRC_Continue Continue descending down the tree. ** -** WRC_Prune Do not descend into child nodes. But allow +** WRC_Prune Do not descend into child nodes, but allow ** the walk to continue with sibling nodes. ** ** WRC_Abort Do no more callbacks. Unwind the stack and -** return the top-level walk call. +** return from the top-level walk call. ** ** The return value from this routine is WRC_Abort to abandon the tree walk ** and WRC_Continue to continue. diff --git a/src/where.c b/src/where.c index 81cc1f131b..4f65695a3f 100644 --- a/src/where.c +++ b/src/where.c @@ -209,7 +209,8 @@ static WhereTerm *whereScanNext(WhereScan *pScan){ if( pTerm->leftCursor==iCur && pTerm->u.leftColumn==iColumn && (iColumn!=XN_EXPR - || sqlite3ExprCompare(pTerm->pExpr->pLeft,pScan->pIdxExpr,iCur)==0) + || sqlite3ExprCompareSkip(pTerm->pExpr->pLeft, + pScan->pIdxExpr,iCur)==0) && (pScan->iEquiv<=1 || !ExprHasProperty(pTerm->pExpr, EP_FromJoin)) ){ if( (pTerm->eOperator & WO_EQUIV)!=0 @@ -308,6 +309,7 @@ static WhereTerm *whereScanInit( iColumn = pIdx->aiColumn[j]; if( iColumn==XN_EXPR ){ pScan->pIdxExpr = pIdx->aColExpr->a[j].pExpr; + pScan->zCollName = pIdx->azColl[j]; }else if( iColumn==pIdx->pTable->iPKey ){ iColumn = XN_ROWID; }else if( iColumn>=0 ){ @@ -515,14 +517,16 @@ static LogEst estLog(LogEst N){ ** value stored in its output register. */ static void translateColumnToCopy( - Vdbe *v, /* The VDBE containing code to translate */ + Parse *pParse, /* Parsing context */ int iStart, /* Translate from this opcode to the end */ int iTabCur, /* OP_Column/OP_Rowid references to this table */ int iRegister, /* The first column is in this register */ int bIncrRowid /* If non-zero, transform OP_rowid to OP_AddImm(1) */ ){ + Vdbe *v = pParse->pVdbe; VdbeOp *pOp = sqlite3VdbeGetOp(v, iStart); int iEnd = sqlite3VdbeCurrentAddr(v); + if( pParse->db->mallocFailed ) return; for(; iStartp1!=iTabCur ) continue; if( pOp->opcode==OP_Column ){ @@ -800,7 +804,9 @@ static void constructAutomaticIndex( if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue); if( pTabItem->fg.viaCoroutine ){ sqlite3VdbeChangeP2(v, addrCounter, regBase+n); - translateColumnToCopy(v, addrTop, pLevel->iTabCur, pTabItem->regResult, 1); + testcase( pParse->db->mallocFailed ); + translateColumnToCopy(pParse, addrTop, pLevel->iTabCur, + pTabItem->regResult, 1); sqlite3VdbeGoto(v, addrTop); pTabItem->fg.viaCoroutine = 0; }else{ @@ -2385,6 +2391,11 @@ static int whereLoopAddBtreeIndex( continue; } + if( IsUniqueIndex(pProbe) && saved_nEq==pProbe->nKeyCol-1 ){ + pBuilder->bldFlags |= SQLITE_BLDF_UNIQUE; + }else{ + pBuilder->bldFlags |= SQLITE_BLDF_INDEXED; + } pNew->wsFlags = saved_wsFlags; pNew->u.btree.nEq = saved_nEq; pNew->u.btree.nBtm = saved_nBtm; @@ -2932,7 +2943,15 @@ static int whereLoopAddBtree( } } + pBuilder->bldFlags = 0; rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 0); + if( pBuilder->bldFlags==SQLITE_BLDF_INDEXED ){ + /* If a non-unique index is used, or if a prefix of the key for + ** unique index is used (making the index functionally non-unique) + ** then the sqlite_stat1 data becomes important for scoring the + ** plan */ + pTab->tabFlags |= TF_StatsUsed; + } #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 sqlite3Stat4ProbeFree(pBuilder->pRec); pBuilder->nRecValid = 0; @@ -4112,9 +4131,9 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ && nRowEst ){ Bitmask notUsed; - int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pDistinctSet, pFrom, + int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pResultSet, pFrom, WHERE_DISTINCTBY, nLoop-1, pFrom->aLoop[nLoop-1], ¬Used); - if( rc==pWInfo->pDistinctSet->nExpr ){ + if( rc==pWInfo->pResultSet->nExpr ){ pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; } } @@ -4351,7 +4370,7 @@ WhereInfo *sqlite3WhereBegin( SrcList *pTabList, /* FROM clause: A list of all tables to be scanned */ Expr *pWhere, /* The WHERE clause */ ExprList *pOrderBy, /* An ORDER BY (or GROUP BY) clause, or NULL */ - ExprList *pDistinctSet, /* Try not to output two rows that duplicate these */ + ExprList *pResultSet, /* Query result set. Req'd for DISTINCT */ u16 wctrlFlags, /* The WHERE_* flags defined in sqliteInt.h */ int iAuxArg /* If WHERE_OR_SUBCLAUSE is set, index cursor number ** If WHERE_USE_LIMIT, then the limit amount */ @@ -4427,7 +4446,7 @@ WhereInfo *sqlite3WhereBegin( pWInfo->pParse = pParse; pWInfo->pTabList = pTabList; pWInfo->pOrderBy = pOrderBy; - pWInfo->pDistinctSet = pDistinctSet; + pWInfo->pResultSet = pResultSet; pWInfo->aiCurOnePass[0] = pWInfo->aiCurOnePass[1] = -1; pWInfo->nLevel = nTabList; pWInfo->iBreak = pWInfo->iContinue = sqlite3VdbeMakeLabel(v); @@ -4505,13 +4524,13 @@ WhereInfo *sqlite3WhereBegin( if( db->mallocFailed ) goto whereBeginError; if( wctrlFlags & WHERE_WANT_DISTINCT ){ - if( isDistinctRedundant(pParse, pTabList, &pWInfo->sWC, pDistinctSet) ){ + if( isDistinctRedundant(pParse, pTabList, &pWInfo->sWC, pResultSet) ){ /* The DISTINCT marking is pointless. Ignore it. */ pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; }else if( pOrderBy==0 ){ /* Try to ORDER BY the result set to make distinct processing easier */ pWInfo->wctrlFlags |= WHERE_DISTINCTBY; - pWInfo->pOrderBy = pDistinctSet; + pWInfo->pOrderBy = pResultSet; } } @@ -4587,10 +4606,10 @@ WhereInfo *sqlite3WhereBegin( #endif /* Attempt to omit tables from the join that do not effect the result */ if( pWInfo->nLevel>=2 - && pDistinctSet!=0 + && pResultSet!=0 && OptimizationEnabled(db, SQLITE_OmitNoopJoin) ){ - Bitmask tabUsed = sqlite3WhereExprListUsage(pMaskSet, pDistinctSet); + Bitmask tabUsed = sqlite3WhereExprListUsage(pMaskSet, pResultSet); if( sWLB.pOrderBy ){ tabUsed |= sqlite3WhereExprListUsage(pMaskSet, sWLB.pOrderBy); } @@ -4905,8 +4924,9 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ ** the co-routine into OP_Copy of result contained in a register. ** OP_Rowid becomes OP_Null. */ - if( pTabItem->fg.viaCoroutine && !db->mallocFailed ){ - translateColumnToCopy(v, pLevel->addrBody, pLevel->iTabCur, + if( pTabItem->fg.viaCoroutine ){ + testcase( pParse->db->mallocFailed ); + translateColumnToCopy(pParse, pLevel->addrBody, pLevel->iTabCur, pTabItem->regResult, 0); continue; } @@ -4949,7 +4969,8 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ pOp->p2 = x; pOp->p1 = pLevel->iIdxCur; } - assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 || x>=0 ); + assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 || x>=0 + || pWInfo->eOnePass ); }else if( pOp->opcode==OP_Rowid ){ pOp->p1 = pLevel->iIdxCur; pOp->opcode = OP_IdxRowid; diff --git a/src/whereInt.h b/src/whereInt.h index fd6ebe77b4..f065fae6ba 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -395,8 +395,13 @@ struct WhereLoopBuilder { UnpackedRecord *pRec; /* Probe for stat4 (if required) */ int nRecValid; /* Number of valid fields currently in pRec */ #endif + unsigned int bldFlags; /* SQLITE_BLDF_* flags */ }; +/* Allowed values for WhereLoopBuider.bldFlags */ +#define SQLITE_BLDF_INDEXED 0x0001 /* An index is used */ +#define SQLITE_BLDF_UNIQUE 0x0002 /* All keys of a UNIQUE index used */ + /* ** The WHERE clause processing routine has two halves. The ** first part does the start of the WHERE loop and the second @@ -411,7 +416,7 @@ struct WhereInfo { Parse *pParse; /* Parsing and code generating context */ SrcList *pTabList; /* List of tables in the join */ ExprList *pOrderBy; /* The ORDER BY clause or NULL */ - ExprList *pDistinctSet; /* DISTINCT over all these values */ + ExprList *pResultSet; /* Result set of the query */ LogEst iLimit; /* LIMIT if wctrlFlags has WHERE_USE_LIMIT */ int aiCurOnePass[2]; /* OP_OpenWrite cursors for the ONEPASS opt */ int iContinue; /* Jump here to continue with next record */ diff --git a/src/wherecode.c b/src/wherecode.c index 032c56fef3..4fd5e16fac 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -1062,6 +1062,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( Vdbe *v; /* The prepared stmt under constructions */ struct SrcList_item *pTabItem; /* FROM clause term being coded */ int addrBrk; /* Jump here to break out of the loop */ + int addrHalt; /* addrBrk for the outermost loop */ int addrCont; /* Jump here to continue with next cycle */ int iRowidReg = 0; /* Rowid is stored in this register, if not zero */ int iReleaseReg = 0; /* Temp register to free before returning */ @@ -1103,6 +1104,11 @@ Bitmask sqlite3WhereCodeOneLoopStart( VdbeComment((v, "init LEFT JOIN no-match flag")); } + /* Compute a safe address to jump to if we discover that the table for + ** this loop is empty and can never contribute content. */ + for(j=iLevel; j>0 && pWInfo->a[j].iLeftJoin==0; j--){} + addrHalt = pWInfo->a[j].addrBrk; + /* Special case of a FROM clause subquery implemented as a co-routine */ if( pTabItem->fg.viaCoroutine ){ int regYield = pTabItem->regReturn; @@ -1287,7 +1293,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( sqlite3ExprCacheAffinityChange(pParse, r1, 1); sqlite3ReleaseTempReg(pParse, rTemp); }else{ - sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iCur, addrBrk); + sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iCur, addrHalt); VdbeCoverageIf(v, bRev==0); VdbeCoverageIf(v, bRev!=0); } @@ -1591,7 +1597,10 @@ Bitmask sqlite3WhereCodeOneLoopStart( if( omitTable ){ /* pIdx is a covering index. No need to access the main table. */ }else if( HasRowid(pIdx->pTable) ){ - if( (pWInfo->wctrlFlags & WHERE_SEEK_TABLE)!=0 ){ + if( (pWInfo->wctrlFlags & WHERE_SEEK_TABLE) || ( + (pWInfo->wctrlFlags & WHERE_SEEK_UNIQ_TABLE) + && (pWInfo->eOnePass==ONEPASS_SINGLE) + )){ iRowidReg = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, iRowidReg); sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); @@ -1930,7 +1939,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( codeCursorHint(pTabItem, pWInfo, pLevel, 0); pLevel->op = aStep[bRev]; pLevel->p1 = iCur; - pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk); + pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrHalt); VdbeCoverageIf(v, bRev==0); VdbeCoverageIf(v, bRev!=0); pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; diff --git a/src/whereexpr.c b/src/whereexpr.c index 4bb161044e..248b5349db 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -213,15 +213,6 @@ static int isLikeOrGlob( #endif pList = pExpr->x.pList; pLeft = pList->a[1].pExpr; - if( pLeft->op!=TK_COLUMN - || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT - || IsVirtual(pLeft->pTab) /* Value might be numeric */ - ){ - /* IMP: R-02065-49465 The left-hand side of the LIKE or GLOB operator must - ** be the name of an indexed column with TEXT affinity. */ - return 0; - } - assert( pLeft->iColumn!=(-1) ); /* Because IPK never has AFF_TEXT */ pRight = sqlite3ExprSkipCollate(pList->a[0].pExpr); op = pRight->op; @@ -238,6 +229,23 @@ static int isLikeOrGlob( z = pRight->u.zToken; } if( z ){ + + /* If the RHS begins with a digit or a minus sign, then the LHS must + ** be an ordinary column (not a virtual table column) with TEXT affinity. + ** Otherwise the LHS might be numeric and "lhs >= rhs" would be false + ** even though "lhs LIKE rhs" is true. But if the RHS does not start + ** with a digit or '-', then "lhs LIKE rhs" will always be false if + ** the LHS is numeric and so the optimization still works. + */ + if( sqlite3Isdigit(z[0]) || z[0]=='-' ){ + if( pLeft->op!=TK_COLUMN + || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT + || IsVirtual(pLeft->pTab) /* Value might be numeric */ + ){ + sqlite3ValueFree(pVal); + return 0; + } + } cnt = 0; while( (c=z[cnt])!=0 && c!=wc[0] && c!=wc[1] && c!=wc[2] ){ cnt++; @@ -866,7 +874,7 @@ static int exprMightBeIndexed( if( pIdx->aColExpr==0 ) continue; for(i=0; inKeyCol; i++){ if( pIdx->aiColumn[i]!=XN_EXPR ) continue; - if( sqlite3ExprCompare(pExpr, pIdx->aColExpr->a[i].pExpr, iCur)==0 ){ + if( sqlite3ExprCompareSkip(pExpr, pIdx->aColExpr->a[i].pExpr, iCur)==0 ){ *piCur = iCur; *piColumn = XN_EXPR; return 1; @@ -913,6 +921,7 @@ static void exprAnalyze( Parse *pParse = pWInfo->pParse; /* Parsing context */ sqlite3 *db = pParse->db; /* Database connection */ unsigned char eOp2; /* op2 value for LIKE/REGEXP/GLOB */ + int nLeft; /* Number of elements on left side vector */ if( db->mallocFailed ){ return; @@ -942,6 +951,10 @@ static void exprAnalyze( prereqAll |= x; extraRight = x-1; /* ON clause terms may not be used with an index ** on left table of a LEFT JOIN. Ticket #3015 */ + if( (prereqAll>>1)>=x ){ + sqlite3ErrorMsg(pParse, "ON clause references tables to its right"); + return; + } } pTerm->prereqAll = prereqAll; pTerm->leftCursor = -1; @@ -1184,13 +1197,12 @@ static void exprAnalyze( ** is not a sub-select. */ if( pWC->op==TK_AND && (pExpr->op==TK_EQ || pExpr->op==TK_IS) - && sqlite3ExprIsVector(pExpr->pLeft) + && (nLeft = sqlite3ExprVectorSize(pExpr->pLeft))>1 + && sqlite3ExprVectorSize(pExpr->pRight)==nLeft && ( (pExpr->pLeft->flags & EP_xIsSelect)==0 - || (pExpr->pRight->flags & EP_xIsSelect)==0 - )){ - int nLeft = sqlite3ExprVectorSize(pExpr->pLeft); + || (pExpr->pRight->flags & EP_xIsSelect)==0) + ){ int i; - assert( nLeft==sqlite3ExprVectorSize(pExpr->pRight) ); for(i=0; ioomErr ) return; nNew = p->nAlloc*2 + 100 + n; - zNew = sqlite3_realloc(p->z, nNew); + zNew = sqlite3_realloc64(p->z, nNew); if( zNew==0 ){ sqlite3_free(p->z); memset(p, 0, sizeof(*p)); @@ -468,7 +468,7 @@ static void StrAppend(Str *p, const char *z){ p->z = zNew; p->nAlloc = nNew; } - memcpy(p->z + p->n, z, n); + memcpy(p->z + p->n, z, (int)n); p->n += n; p->z[p->n] = 0; } @@ -647,7 +647,7 @@ static void runSql(sqlite3 *db, const char *zSql, unsigned runFlags){ int main(int argc, char **argv){ int i; /* Loop counter */ int nDb = 0; /* Number of databases to fuzz */ - const char **azDb = 0; /* Names of the databases (limit: 20) */ + char **azDb = 0; /* Names of the databases (limit: 20) */ int verboseFlag = 0; /* True for extra output */ int noLookaside = 0; /* Disable lookaside if true */ int vdbeLimitFlag = 0; /* Stop after 100,000 VDBE ops */ @@ -660,7 +660,7 @@ int main(int argc, char **argv){ unsigned runFlags = 0; /* Flags passed to runSql */ for(i=1; i=argc-1 ) fatalError("missing arguments on %s", argv[i]); zMsg = argv[++i]; }else + if( strcmp(z,"native-malloc")==0 ){ + nativeMalloc = 1; + }else if( strcmp(z,"native-vfs")==0 ){ nativeFlag = 1; }else @@ -972,7 +977,7 @@ int main(int argc, char **argv){ /* Process each source database separately */ for(iSrcDb=0; iSrcDbzName); + SQLITE_OPEN_READWRITE, pDfltVfs->zName); if( rc ){ fatalError("cannot open source database %s - %s", azSrcDb[iSrcDb], sqlite3_errmsg(db)); @@ -1013,7 +1018,7 @@ int main(int argc, char **argv){ ossFuzzThisDb = sqlite3_column_int(pStmt,1); if( verboseFlag ) printf("Config: oss-fuzz=%d\n", ossFuzzThisDb); } - if( strcmp(zName, "limit-mem")==0 ){ + if( strcmp(zName, "limit-mem")==0 && !nativeMalloc ){ #if !defined(SQLITE_ENABLE_MEMSYS3) && !defined(SQLITE_ENABLE_MEMSYS5) fatalError("the limit-mem option requires -DSQLITE_ENABLE_MEMSYS5" " or _MEMSYS3"); @@ -1047,6 +1052,8 @@ int main(int argc, char **argv){ sqlite3_close(db); return 0; } + rc = sqlite3_exec(db, "PRAGMA query_only=1;", 0, 0, 0); + if( rc ) fatalError("cannot set database to query-only"); if( zExpDb!=0 || zExpSql!=0 ){ sqlite3_create_function(db, "writefile", 2, SQLITE_UTF8, 0, writefileFunc, 0, 0); @@ -1141,14 +1148,19 @@ int main(int argc, char **argv){ } /* Limit available memory, if requested */ - if( nMemThisDb>0 ){ - sqlite3_shutdown(); + sqlite3_shutdown(); + if( nMemThisDb>0 && !nativeMalloc ){ pHeap = realloc(pHeap, nMemThisDb); if( pHeap==0 ){ fatalError("failed to allocate %d bytes of heap memory", nMem); } sqlite3_config(SQLITE_CONFIG_HEAP, pHeap, nMemThisDb, 128); } + + /* Disable lookaside with the --native-malloc option */ + if( nativeMalloc ){ + sqlite3_config(SQLITE_CONFIG_LOOKASIDE, 0, 0); + } /* Reset the in-memory virtual filesystem */ formatVfs(); @@ -1205,9 +1217,13 @@ int main(int argc, char **argv){ runSql(db, (char*)pSql->a, runFlags); }while( timeoutTest ); setAlarm(0); + sqlite3_exec(db, "PRAGMA temp_store_directory=''", 0, 0, 0); sqlite3_close(db); } - if( sqlite3_memory_used()>0 ) fatalError("memory leak"); + if( sqlite3_memory_used()>0 ){ + fatalError("memory leak: %lld bytes outstanding", + sqlite3_memory_used()); + } reformatVfs(); nTest++; g.zTestName[0] = 0; diff --git a/test/hook.test b/test/hook.test index 8f095c9e8d..9ba220cded 100644 --- a/test/hook.test +++ b/test/hook.test @@ -849,9 +849,62 @@ do_preupdate_test 7.6.4 { } # No preupdate callbacks for modifying sqlite_master. -do_preupdate_test 8.1 { - CREATE TABLE x1(x, y); -} { +do_preupdate_test 8.1 { CREATE TABLE x1(x, y); } { } +do_preupdate_test 8.2 { ALTER TABLE x1 ADD COLUMN z } { } +do_preupdate_test 8.3 { ALTER TABLE x1 RENAME TO y1 } { } +do_preupdate_test 8.4 { CREATE INDEX y1x ON y1(x) } { } +do_preupdate_test 8.5 { CREATE VIEW v1 AS SELECT * FROM y1 } { } +do_preupdate_test 8.6 { DROP TABLE y1 } { } + +#------------------------------------------------------------------------- +reset_db +db preupdate hook preupdate_hook +do_execsql_test 9.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE TABLE t2(a, b INTEGER PRIMARY KEY); } +do_preupdate_test 9.1 { + INSERT INTO t1 VALUES(456, NULL, NULL); +} { + INSERT main t1 456 456 0 456 {} {} +} +do_execsql_test 9.2 { + ALTER TABLE t1 ADD COLUMN d; +} +do_preupdate_test 9.3 { + INSERT INTO t1(a, b, c) VALUES(457, NULL, NULL); +} { + INSERT main t1 457 457 0 457 {} {} {} +} +do_preupdate_test 9.4 { + DELETE FROM t1 WHERE a=456 +} { + DELETE main t1 456 456 0 456 {} {} {} +} +do_preupdate_test 9.5 { + INSERT INTO t2 DEFAULT VALUES; +} { + INSERT main t2 1 1 0 {} 1 +} +do_preupdate_test 9.6 { + INSERT INTO t1 DEFAULT VALUES; +} { + INSERT main t1 458 458 0 458 {} {} {} +} + + +do_execsql_test 10.0 { + CREATE TABLE t3(a, b INTEGER PRIMARY KEY); +} +do_preupdate_test 10.1 { + INSERT INTO t3 DEFAULT VALUES +} { + INSERT main t3 1 1 0 {} 1 +} +do_execsql_test 10.2 { SELECT * FROM t3 } {{} 1} +do_preupdate_test 10.3 { + DELETE FROM t3 WHERE b=1 +} {DELETE main t3 1 1 0 {} 1} + finish_test diff --git a/test/hook2.test b/test/hook2.test new file mode 100644 index 0000000000..9ae1103a7d --- /dev/null +++ b/test/hook2.test @@ -0,0 +1,218 @@ +# 2017 Jan 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# The tests in this file focus on the pre-update hook. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix hook2 + +ifcapable !preupdate { + finish_test + return +} + +#------------------------------------------------------------------------- +proc do_preupdate_test {tn sql x} { + set X [list] + foreach elem $x {lappend X $elem} + uplevel do_test $tn [list " + set ::preupdate \[list\] + execsql { $sql } + set ::preupdate + "] [list $X] +} + +proc preupdate_hook {args} { + set type [lindex $args 0] + eval lappend ::preupdate $args + if {$type != "INSERT"} { + for {set i 0} {$i < [db preupdate count]} {incr i} { + lappend ::preupdate [db preupdate old $i] + } + } + if {$type != "DELETE"} { + for {set i 0} {$i < [db preupdate count]} {incr i} { + set rc [catch { db preupdate new $i } v] + lappend ::preupdate $v + } + } +} + +#------------------------------------------------------------------------- +# Simple tests - INSERT, UPDATE and DELETE on a WITHOUT ROWID table. +# +db preupdate hook preupdate_hook +do_execsql_test 1.0 { + CREATE TABLE t1(a PRIMARY KEY, b) WITHOUT ROWID; +} +do_preupdate_test 1.1 { + INSERT INTO t1 VALUES('one', 1); +} { + INSERT main t1 0 0 one 1 +} +do_preupdate_test 1.2 { + UPDATE t1 SET b=2 WHERE a='one'; +} { + UPDATE main t1 0 0 one 1 one 2 +} +do_preupdate_test 1.3 { + DELETE FROM t1 WHERE a='one'; +} { + DELETE main t1 0 0 one 2 +} + +#------------------------------------------------------------------------- +# Some more complex tests for the pre-update callback on WITHOUT ROWID +# tables. +# +# 2.1.1 - INSERT statement. +# 2.1.2 - INSERT INTO ... SELECT statement. +# 2.1.3 - REPLACE INTO ... (PK conflict) +# 2.1.4 - REPLACE INTO ... (other index conflicts) +# 2.1.5 - REPLACE INTO ... (both PK and other index conflicts) +# +# 2.2.1 - DELETE statement. +# 2.2.2 - DELETE statement that uses the truncate optimization. +# +# 2.3.1 - UPDATE statement. +# 2.3.2 - UPDATE statement that modifies the PK. +# 2.3.3 - UPDATE OR REPLACE ... (PK conflict). +# 2.3.4 - UPDATE OR REPLACE ... (other index conflicts) +# 2.3.4 - UPDATE OR REPLACE ... (both PK and other index conflicts) +# +do_execsql_test 2.0 { + CREATE TABLE t2(a DEFAULT 4, b, c, PRIMARY KEY(b, c)) WITHOUT ROWID; + CREATE UNIQUE INDEX t2a ON t2(a); +} + +do_preupdate_test 2.1.1 { + INSERT INTO t2(b, c) VALUES(1, 1); +} { + INSERT main t2 0 0 4 1 1 +} + +do_execsql_test 2.1.2.0 { + CREATE TABLE d1(a DEFAULT 4, b, c, PRIMARY KEY(b, c)) WITHOUT ROWID; + CREATE UNIQUE INDEX d1a ON d1(a); + INSERT INTO d1 VALUES(1, 2, 3); + INSERT INTO d1 VALUES(11, 12, 13); +} +do_preupdate_test 2.1.2.1 { + INSERT INTO t2 SELECT * FROM d1; +} { + INSERT main t2 0 0 1 2 3 + INSERT main t2 0 0 11 12 13 +} +do_preupdate_test 2.1.2.2 { + INSERT INTO t2 SELECT a+20, b+20, c+20 FROM d1; +} { + INSERT main t2 0 0 21 22 23 + INSERT main t2 0 0 31 32 33 +} +do_execsql_test 2.1.2.3 { + SELECT * FROM t2 ORDER BY b, c; +} { + 4 1 1 + 1 2 3 + 11 12 13 + 21 22 23 + 31 32 33 +} +do_preupdate_test 2.1.3 { + REPLACE INTO t2 VALUES(45, 22, 23); +} { + DELETE main t2 0 0 21 22 23 + INSERT main t2 0 0 45 22 23 +} +do_preupdate_test 2.1.4 { + REPLACE INTO t2 VALUES(11, 100, 100); +} { + DELETE main t2 0 0 11 12 13 + INSERT main t2 0 0 11 100 100 +} +do_preupdate_test 2.1.5 { + REPLACE INTO t2(c, b) VALUES(33, 32) +} { + DELETE main t2 0 0 4 1 1 + DELETE main t2 0 0 31 32 33 + INSERT main t2 0 0 4 32 33 +} + +do_execsql_test 2.2.0 { + SELECT * FROM t2 ORDER BY b,c; +} { + 1 2 3 + 45 22 23 + 4 32 33 + 11 100 100 +} +do_preupdate_test 2.2.1 { + DELETE FROM t2 WHERE b=22; +} { + DELETE main t2 0 0 45 22 23 +} +do_preupdate_test 2.2.2 { + DELETE FROM t2; +} { + DELETE main t2 0 0 1 2 3 + DELETE main t2 0 0 4 32 33 + DELETE main t2 0 0 11 100 100 +} + +do_execsql_test 2.3.0 { + CREATE TABLE t3(x, y PRIMARY KEY, z UNIQUE) WITHOUT ROWID; + INSERT INTO t3 VALUES('a', 'b', 'c'); + INSERT INTO t3 VALUES('d', 'e', 'f'); + + INSERT INTO t3 VALUES(1, 1, 1); + INSERT INTO t3 VALUES(2, 2, 2); + INSERT INTO t3 VALUES(3, 3, 3); +} + +do_preupdate_test 2.3.1 { + UPDATE t3 SET x=4 WHERE y IN ('b', 'e', 'x'); +} { + UPDATE main t3 0 0 a b c 4 b c + UPDATE main t3 0 0 d e f 4 e f +} + +do_preupdate_test 2.3.2 { + UPDATE t3 SET y=y||y WHERE z IN('c', 'f'); +} { + UPDATE main t3 0 0 4 b c 4 bb c + UPDATE main t3 0 0 4 e f 4 ee f +} + +do_preupdate_test 2.3.3 { + UPDATE OR REPLACE t3 SET y='bb' WHERE z='f' +} { + DELETE main t3 0 0 4 bb c + UPDATE main t3 0 0 4 ee f 4 bb f +} + +do_preupdate_test 2.3.4 { + UPDATE OR REPLACE t3 SET z=2 WHERE y=1; +} { + DELETE main t3 0 0 2 2 2 + UPDATE main t3 0 0 1 1 1 1 1 2 +} + +do_preupdate_test 2.3.5 { + UPDATE OR REPLACE t3 SET z=2, y='bb' WHERE y=3; +} { + DELETE main t3 0 0 1 1 2 + DELETE main t3 0 0 4 bb f + UPDATE main t3 0 0 3 3 3 3 bb 2 +} + + +finish_test diff --git a/test/incrblobfault.test b/test/incrblobfault.test index 0c1d93d460..10c2c8ecb4 100644 --- a/test/incrblobfault.test +++ b/test/incrblobfault.test @@ -52,7 +52,7 @@ do_faultsim_test 2 -prep { error [sqlite3_errmsg db] } } -test { - faultsim_test_result {1 {no such rowid: -1}} + faultsim_test_result {1 {no such rowid: -1}} {1 {disk I/O error}} close $::blob } diff --git a/test/indexexpr1.test b/test/indexexpr1.test index cd72430d91..2b375a5855 100644 --- a/test/indexexpr1.test +++ b/test/indexexpr1.test @@ -370,4 +370,14 @@ do_execsql_test indexexpr1-1200.4 { 0 0 0 2 0 4 2 0 2 2 4 0 } +# Ticket https://www.sqlite.org/src/tktview/eb703ba7b50c1a +# Incorrect result using an index on an expression with a collating function +# +do_execsql_test indexexpr1-1300.1 { + CREATE TABLE t1300(a INTEGER PRIMARY KEY, b); + INSERT INTO t1300 VALUES(1,'coffee'),(2,'COFFEE'),(3,'stress'),(4,'STRESS'); + CREATE INDEX t1300bexpr ON t1300( substr(b,4) ); + SELECT a FROM t1300 WHERE substr(b,4)='ess' COLLATE nocase ORDER BY +a; +} {3 4} + finish_test diff --git a/test/join2.test b/test/join2.test index 0f558c5a3d..9372e770c3 100644 --- a/test/join2.test +++ b/test/join2.test @@ -12,10 +12,10 @@ # # This file implements tests for joins, including outer joins. # -# $Id: join2.test,v 1.2 2005/01/21 03:12:16 danielk1977 Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl +set testprefix join2 do_test join2-1.1 { execsql { @@ -72,4 +72,24 @@ ifcapable subquery { } {1 11 111 1111 2 22 {} {} 3 33 {} {}} } +#------------------------------------------------------------------------- +# Check that ticket [25e335f802ddc] has been resolved. It should be an +# error for the ON clause of a LEFT JOIN to refer to a table to its right. +# +do_execsql_test 2.0 { + CREATE TABLE aa(a); + CREATE TABLE bb(b); + CREATE TABLE cc(c); + INSERT INTO aa VALUES('one'); + INSERT INTO bb VALUES('one'); + INSERT INTO cc VALUES('one'); +} + +do_catchsql_test 2.1 { + SELECT * FROM aa LEFT JOIN cc ON (a=b) JOIN bb ON (b=c); +} {1 {ON clause references tables to its right}} +do_catchsql_test 2.2 { + SELECT * FROM aa JOIN cc ON (a=b) JOIN bb ON (b=c); +} {0 {one one one}} + finish_test diff --git a/test/json104.test b/test/json104.test new file mode 100644 index 0000000000..b5313f01b5 --- /dev/null +++ b/test/json104.test @@ -0,0 +1,130 @@ +# 2017-03-22 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements tests for json_patch(A,B) SQL function. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +ifcapable !json1 { + finish_test + return +} + +# This is the example from pages 2 and 3 of RFC-7396 +do_execsql_test json104-100 { + SELECT json_patch('{ + "a": "b", + "c": { + "d": "e", + "f": "g" + } + }','{ + "a":"z", + "c": { + "f": null + } + }'); +} {{{"a":"z","c":{"d":"e"}}}} + + +# This is the example from pages 4 and 5 of RFC-7396 +do_execsql_test json104-110 { + SELECT json_patch('{ + "title": "Goodbye!", + "author" : { + "givenName" : "John", + "familyName" : "Doe" + }, + "tags":[ "example", "sample" ], + "content": "This will be unchanged" + }','{ + "title": "Hello!", + "phoneNumber": "+01-123-456-7890", + "author": { + "familyName": null + }, + "tags": [ "example" ] + }'); +} {{{"title":"Hello!","author":{"givenName":"John"},"tags":["example"],"content":"This will be unchanged","phoneNumber":"+01-123-456-7890"}}} + +do_execsql_test json104-200 { + SELECT json_patch('[1,2,3]','{"x":null}'); +} {{{}}} +do_execsql_test json104-210 { + SELECT json_patch('[1,2,3]','{"x":null,"y":1,"z":null}'); +} {{{"y":1}}} +do_execsql_test json104-220 { + SELECT json_patch('{}','{"a":{"bb":{"ccc":null}}}'); +} {{{"a":{"bb":{}}}}} +do_execsql_test json104-221 { + SELECT json_patch('{}','{"a":{"bb":{"ccc":[1,null,3]}}}'); +} {{{"a":{"bb":{"ccc":[1,null,3]}}}}} +do_execsql_test json104-222 { + SELECT json_patch('{}','{"a":{"bb":{"ccc":[1,{"dddd":null},3]}}}'); +} {{{"a":{"bb":{"ccc":[1,{"dddd":null},3]}}}}} + +# Example test cases at the end of the RFC-7396 document +do_execsql_test json104-300 { + SELECT json_patch('{"a":"b"}','{"a":"c"}'); +} {{{"a":"c"}}} +do_execsql_test json104-300a { + SELECT coalesce(json_patch(null,'{"a":"c"}'), 'real-null'); +} {{real-null}} +do_execsql_test json104-301 { + SELECT json_patch('{"a":"b"}','{"b":"c"}'); +} {{{"a":"b","b":"c"}}} +do_execsql_test json104-302 { + SELECT json_patch('{"a":"b"}','{"a":null}'); +} {{{}}} +do_execsql_test json104-303 { + SELECT json_patch('{"a":"b","b":"c"}','{"a":null}'); +} {{{"b":"c"}}} +do_execsql_test json104-304 { + SELECT json_patch('{"a":["b"]}','{"a":"c"}'); +} {{{"a":"c"}}} +do_execsql_test json104-305 { + SELECT json_patch('{"a":"c"}','{"a":["b"]}'); +} {{{"a":["b"]}}} +do_execsql_test json104-306 { + SELECT json_patch('{"a":{"b":"c"}}','{"a":{"b":"d","c":null}}'); +} {{{"a":{"b":"d"}}}} +do_execsql_test json104-307 { + SELECT json_patch('{"a":[{"b":"c"}]}','{"a":[1]}'); +} {{{"a":[1]}}} +do_execsql_test json104-308 { + SELECT json_patch('["a","b"]','["c","d"]'); +} {{["c","d"]}} +do_execsql_test json104-309 { + SELECT json_patch('{"a":"b"}','["c"]'); +} {{["c"]}} +do_execsql_test json104-310 { + SELECT json_patch('{"a":"foo"}','null'); +} {{null}} +do_execsql_test json104-310a { + SELECT coalesce(json_patch('{"a":"foo"}',null), 'real-null'); +} {{real-null}} +do_execsql_test json104-311 { + SELECT json_patch('{"a":"foo"}','"bar"'); +} {{"bar"}} +do_execsql_test json104-312 { + SELECT json_patch('{"e":null}','{"a":1}'); +} {{{"e":null,"a":1}}} +do_execsql_test json104-313 { + SELECT json_patch('[1,2]','{"a":"b","c":null}'); +} {{{"a":"b"}}} +do_execsql_test json104-314 { + SELECT json_patch('{}','{"a":{"bb":{"ccc":null}}}'); +} {{{"a":{"bb":{}}}}} + + + +finish_test diff --git a/test/kvtest.c b/test/kvtest.c index 877605aced..4fa733f1a7 100644 --- a/test/kvtest.c +++ b/test/kvtest.c @@ -61,32 +61,41 @@ ** ./kvtest run x1 --count 10000 --max-id 1000000 */ static const char zHelp[] = -"Usage: kvhelp COMMAND ARGS...\n" +"Usage: kvtest COMMAND ARGS...\n" "\n" -" kvhelp init DBFILE --count N --size M --pagesize X\n" +" kvtest init DBFILE --count N --size M --pagesize X\n" "\n" " Generate a new test database file named DBFILE containing N\n" " BLOBs each of size M bytes. The page size of the new database\n" -" file will be X\n" +" file will be X. Additional options:\n" "\n" -" kvhelp export DBFILE DIRECTORY\n" +" --variance V Randomly vary M by plus or minus V\n" +"\n" +" kvtest export DBFILE DIRECTORY\n" "\n" " Export all the blobs in the kv table of DBFILE into separate\n" " files in DIRECTORY.\n" "\n" -" kvhelp run DBFILE [options]\n" +" kvtest stat DBFILE\n" +"\n" +" Display summary information about DBFILE\n" +"\n" +" kvtest run DBFILE [options]\n" "\n" " Run a performance test. DBFILE can be either the name of a\n" " database or a directory containing sample files. Options:\n" "\n" -" --asc Read blobs in ascending order\n" -" --blob-api Use the BLOB API\n" -" --cache-size N Database cache size\n" -" --count N Read N blobs\n" -" --desc Read blobs in descending order\n" -" --max-id N Maximum blob key to use\n" -" --random Read blobs in a random order\n" -" --start N Start reading with this blob key\n" +" --asc Read blobs in ascending order\n" +" --blob-api Use the BLOB API\n" +" --cache-size N Database cache size\n" +" --count N Read N blobs\n" +" --desc Read blobs in descending order\n" +" --max-id N Maximum blob key to use\n" +" --mmap N Mmap as much as N bytes of DBFILE\n" +" --jmode MODE Set MODE journal mode prior to starting\n" +" --random Read blobs in a random order\n" +" --start N Start reading with this blob key\n" +" --stats Output operating stats before exiting\n" ; /* Reference resources used */ @@ -131,6 +140,64 @@ static void fatalError(const char *zFormat, ...){ exit(1); } +/* +** Return the value of a hexadecimal digit. Return -1 if the input +** is not a hex digit. +*/ +static int hexDigitValue(char c){ + if( c>='0' && c<='9' ) return c - '0'; + if( c>='a' && c<='f' ) return c - 'a' + 10; + if( c>='A' && c<='F' ) return c - 'A' + 10; + return -1; +} + +/* +** Interpret zArg as an integer value, possibly with suffixes. +*/ +static int integerValue(const char *zArg){ + int v = 0; + static const struct { char *zSuffix; int iMult; } aMult[] = { + { "KiB", 1024 }, + { "MiB", 1024*1024 }, + { "GiB", 1024*1024*1024 }, + { "KB", 1000 }, + { "MB", 1000000 }, + { "GB", 1000000000 }, + { "K", 1000 }, + { "M", 1000000 }, + { "G", 1000000000 }, + }; + int i; + int isNeg = 0; + if( zArg[0]=='-' ){ + isNeg = 1; + zArg++; + }else if( zArg[0]=='+' ){ + zArg++; + } + if( zArg[0]=='0' && zArg[1]=='x' ){ + int x; + zArg += 2; + while( (x = hexDigitValue(zArg[0]))>=0 ){ + v = (v<<4) + x; + zArg++; + } + }else{ + while( zArg[0]>='0' && zArg[0]<='9' ){ + v = v*10 + zArg[0] - '0'; + zArg++; + } + } + for(i=0; i65536 || ((pgsz-1)&pgsz)!=0 ){ fatalError("the --pagesize must be power of 2 between 512 and 65536"); } @@ -235,9 +308,9 @@ static int initMain(int argc, char **argv){ "BEGIN;\n" "CREATE TABLE kv(k INTEGER PRIMARY KEY, v BLOB);\n" "WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<%d)" - " INSERT INTO kv(k,v) SELECT x, randomblob(%d) FROM c;\n" + " INSERT INTO kv(k,v) SELECT x, randomblob(%d+(random()%%(%d))) FROM c;\n" "COMMIT;\n", - pgsz, nCount, sz + pgsz, nCount, sz, iVariance+1 ); rc = sqlite3_exec(db, zSql, 0, 0, &zErrMsg); if( rc ) fatalError("database create failed: %s", zErrMsg); @@ -246,6 +319,65 @@ static int initMain(int argc, char **argv){ return 0; } +/* +** Analyze an existing database file. Report its content. +*/ +static int statMain(int argc, char **argv){ + char *zDb; + int i, rc; + sqlite3 *db; + char *zSql; + sqlite3_stmt *pStmt; + + assert( strcmp(argv[1],"stat")==0 ); + assert( argc>=3 ); + zDb = argv[2]; + for(i=3; i #include +#include +#include #include "sqlite3.h" +/* Global debugging settings. OSS-Fuzz will have all debugging turned +** off. But if LLVMFuzzerTestOneInput() is called interactively from +** the ossshell utility program, then these flags might be set. +*/ +static unsigned mDebug = 0; +#define FUZZ_SQL_TRACE 0x0001 /* Set an sqlite3_trace() callback */ +#define FUZZ_SHOW_MAX_DELAY 0x0002 /* Show maximum progress callback delay */ +#define FUZZ_SHOW_ERRORS 0x0004 /* Print error messages from SQLite */ + +/* The ossshell utility program invokes this interface to see the +** debugging flags. Unused by OSS-Fuzz. +*/ +void ossfuzz_set_debug_flags(unsigned x){ + mDebug = x; +} + +/* Return the current real-world time in milliseconds since the +** Julian epoch (-4714-11-24). +*/ +static sqlite3_int64 timeOfDay(void){ + static sqlite3_vfs *clockVfs = 0; + sqlite3_int64 t; + if( clockVfs==0 ) clockVfs = sqlite3_vfs_find(0); + if( clockVfs->iVersion>=2 && clockVfs->xCurrentTimeInt64!=0 ){ + clockVfs->xCurrentTimeInt64(clockVfs, &t); + }else{ + double r; + clockVfs->xCurrentTime(clockVfs, &r); + t = (sqlite3_int64)(r*86400000.0); + } + return t; +} + +/* An instance of the following object is passed by pointer as the +** client data to various callbacks. +*/ +typedef struct FuzzCtx { + sqlite3 *db; /* The database connection */ + sqlite3_int64 iCutoffTime; /* Stop processing at this time. */ + sqlite3_int64 iLastCb; /* Time recorded for previous progress callback */ + sqlite3_int64 mxInterval; /* Longest interval between two progress calls */ + unsigned nCb; /* Number of progress callbacks */ +} FuzzCtx; + #ifndef SQLITE_OMIT_PROGRESS_CALLBACK /* -** Progress handler callback +** Progress handler callback. +** +** The argument is the cutoff-time after which all processing should +** stop. So return non-zero if the cut-off time is exceeded. */ -static int progress_handler(void *pReturn) { - return *(int*)pReturn; +static int progress_handler(void *pClientData) { + FuzzCtx *p = (FuzzCtx*)pClientData; + sqlite3_int64 iNow = timeOfDay(); + int rc = iNow>=p->iCutoffTime; + sqlite3_int64 iDiff = iNow - p->iLastCb; + if( iDiff > p->mxInterval ) p->mxInterval = iDiff; + p->nCb++; + return rc; } #endif @@ -31,14 +86,14 @@ static int exec_handler(void *pCnt, int argc, char **argv, char **namev){ ** fuzzed input. */ int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { - int progressArg = 0; /* 1 causes progress handler abort */ int execCnt = 0; /* Abort row callback when count reaches zero */ char *zErrMsg = 0; /* Error message returned by sqlite_exec() */ - sqlite3 *db; /* The database connection */ uint8_t uSelector; /* First byte of input data[] */ int rc; /* Return code from various interfaces */ char *zSql; /* Zero-terminated copy of data[] */ + FuzzCtx cx; /* Fuzzing context */ + memset(&cx, 0, sizeof(cx)); if( size<3 ) return 0; /* Early out if unsufficient data */ /* Extract the selector byte from the beginning of the input. But only @@ -51,22 +106,26 @@ int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { } /* Open the database connection. Only use an in-memory database. */ - rc = sqlite3_open_v2("fuzz.db", &db, + rc = sqlite3_open_v2("fuzz.db", &cx.db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_MEMORY, 0); if( rc ) return 0; #ifndef SQLITE_OMIT_PROGRESS_CALLBACK - /* Bit 0 of the selector enables progress callbacks. Bit 1 is the - ** return code from progress callbacks */ - if( uSelector & 1 ){ - sqlite3_progress_handler(db, 4, progress_handler, (void*)&progressArg); - } + /* Invoke the progress handler frequently to check to see if we + ** are taking too long. The progress handler will return true + ** (which will block further processing) if more than 10 seconds have + ** elapsed since the start of the test. + */ + cx.iLastCb = timeOfDay(); + cx.iCutoffTime = cx.iLastCb + 10000; /* Now + 10 seconds */ + sqlite3_progress_handler(cx.db, 10, progress_handler, (void*)&cx); #endif - uSelector >>= 1; - progressArg = uSelector & 1; uSelector >>= 1; - /* Bit 2 of the selector enables foreign key constraints */ - sqlite3_db_config(db, SQLITE_DBCONFIG_ENABLE_FKEY, uSelector&1, &rc); + /* Set a limit on the maximum size of a prepared statement */ + sqlite3_limit(cx.db, SQLITE_LIMIT_VDBE_OP, 25000); + + /* Bit 1 of the selector enables foreign key constraints */ + sqlite3_db_config(cx.db, SQLITE_DBCONFIG_ENABLE_FKEY, uSelector&1, &rc); uSelector >>= 1; /* Remaining bits of the selector determine a limit on the number of @@ -76,11 +135,22 @@ int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { /* Run the SQL. The sqlite_exec() interface expects a zero-terminated ** string, so make a copy. */ zSql = sqlite3_mprintf("%.*s", (int)size, data); - sqlite3_exec(db, zSql, exec_handler, (void*)&execCnt, &zErrMsg); + sqlite3_exec(cx.db, zSql, exec_handler, (void*)&execCnt, &zErrMsg); + + /* Show any errors */ + if( (mDebug & FUZZ_SHOW_ERRORS)!=0 && zErrMsg ){ + printf("Error: %s\n", zErrMsg); + } /* Cleanup and return */ sqlite3_free(zErrMsg); sqlite3_free(zSql); - sqlite3_close(db); + sqlite3_exec(cx.db, "PRAGMA temp_store_directory=''", 0, 0, 0); + sqlite3_close(cx.db); + + if( mDebug & FUZZ_SHOW_MAX_DELAY ){ + printf("Progress callback count....... %d\n", cx.nCb); + printf("Max time between callbacks.... %d ms\n", (int)cx.mxInterval); + } return 0; } diff --git a/test/ossshell.c b/test/ossshell.c index 15902a9122..00cc3391c8 100644 --- a/test/ossshell.c +++ b/test/ossshell.c @@ -9,6 +9,7 @@ #include #include #include +#include #include "sqlite3.h" /* @@ -16,6 +17,13 @@ */ int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size); +/* Must match equivalent #defines in ossfuzz.c */ +#define FUZZ_SQL_TRACE 0x0001 /* Set an sqlite3_trace() callback */ +#define FUZZ_SHOW_MAX_DELAY 0x0002 /* Show maximum progress callback delay */ +#define FUZZ_SHOW_ERRORS 0x0004 /* Show SQL errors */ +extern void ossfuzz_set_debug_flags(unsigned); + + /* ** Read files named on the command-line and invoke the fuzzer for @@ -27,9 +35,32 @@ int main(int argc, char **argv){ int nErr = 0; uint8_t *zBuf = 0; size_t sz; + unsigned mDebug = 0; for(i=1; ible](x INTEGER PRIMARY KEY, y) WITHOUT ROWID; + INSERT INTO [ta<>ble] VALUES(1,null), (12,''), (23,1), + (34,2.25), (45,'hello'), (56,x'807f'); + } + catchcmd test2.db {.dump --preserve-rowids} +} {0 {PRAGMA foreign_keys=OFF; +BEGIN TRANSACTION; +CREATE TABLE [ta<>ble](x INTEGER PRIMARY KEY, y) WITHOUT ROWID; +INSERT INTO "ta<>ble" VALUES(1,NULL); +INSERT INTO "ta<>ble" VALUES(12,''); +INSERT INTO "ta<>ble" VALUES(23,1); +INSERT INTO "ta<>ble" VALUES(34,2.25); +INSERT INTO "ta<>ble" VALUES(45,'hello'); +INSERT INTO "ta<>ble" VALUES(56,X'807f'); +COMMIT;}} + +# Do not record rowids if the rowid is inaccessible +# +do_test shell1-4.1.5 { + db close + forcedelete test2.db + sqlite3 db test2.db + db eval { + CREATE TABLE t1(_ROWID_,rowid,oid); + INSERT INTO t1 VALUES(1,null,'alpha'), (12,'',99), (23,1,x'b0b1b2'); + } + catchcmd test2.db {.dump --preserve-rowids} +} {0 {PRAGMA foreign_keys=OFF; +BEGIN TRANSACTION; +CREATE TABLE t1(_ROWID_,rowid,oid); +INSERT INTO t1 VALUES(1,NULL,'alpha'); +INSERT INTO t1 VALUES(12,'',99); +INSERT INTO t1 VALUES(23,1,X'b0b1b2'); COMMIT;}} # Test the output of ".mode insert" diff --git a/test/speedtest1.c b/test/speedtest1.c index 02d55d5974..db3a558a38 100644 --- a/test/speedtest1.c +++ b/test/speedtest1.c @@ -33,7 +33,7 @@ static const char zHelp[] = " --size N Relative test size. Default=100\n" " --stats Show statistics at the end\n" " --temp N N from 0 to 9. 0: no temp table. 9: all temp tables\n" - " --testset T Run test-set T\n" + " --testset T Run test-set T (main, cte, rtree, orm, debug)\n" " --trace Turn on SQL tracing\n" " --threads N Use up to N threads for sorting\n" " --utf16be Set text encoding to UTF-16BE\n" @@ -61,9 +61,6 @@ static const char zHelp[] = #if SQLITE_VERSION_NUMBER<3005000 # define sqlite3_int64 sqlite_int64 #endif -#ifdef SQLITE_ENABLE_RBU -# include "sqlite3rbu.h" -#endif /* All global state is held in this structure */ static struct Global { @@ -1181,10 +1178,10 @@ void testset_rtree(int p1, int p2){ unsigned mxCoord; unsigned x0, x1, y0, y1, z0, z1; unsigned iStep; - int *aCheck = sqlite3_malloc( sizeof(int)*g.szTest*100 ); + int *aCheck = sqlite3_malloc( sizeof(int)*g.szTest*500 ); mxCoord = 15000; - n = g.szTest*100; + n = g.szTest*500; speedtest1_begin_test(100, "%d INSERTs into an r-tree", n); speedtest1_exec("BEGIN"); speedtest1_exec("CREATE VIRTUAL TABLE rt1 USING rtree(id,x0,x1,y0,y1,z0,z1)"); @@ -1207,11 +1204,11 @@ void testset_rtree(int p1, int p2){ speedtest1_end_test(); speedtest1_begin_test(101, "Copy from rtree to a regular table"); - speedtest1_exec(" TABLE t1(id INTEGER PRIMARY KEY,x0,x1,y0,y1,z0,z1)"); + speedtest1_exec("CREATE TABLE t1(id INTEGER PRIMARY KEY,x0,x1,y0,y1,z0,z1)"); speedtest1_exec("INSERT INTO t1 SELECT * FROM rt1"); speedtest1_end_test(); - n = g.szTest*20; + n = g.szTest*100; speedtest1_begin_test(110, "%d one-dimensional intersect slice queries", n); speedtest1_prepare("SELECT count(*) FROM rt1 WHERE x0>=?1 AND x1<=?2"); iStep = mxCoord/n; @@ -1224,7 +1221,7 @@ void testset_rtree(int p1, int p2){ speedtest1_end_test(); if( g.bVerify ){ - n = g.szTest*20; + n = g.szTest*100; speedtest1_begin_test(111, "Verify result from 1-D intersect slice queries"); speedtest1_prepare("SELECT count(*) FROM t1 WHERE x0>=?1 AND x1<=?2"); iStep = mxCoord/n; @@ -1240,7 +1237,7 @@ void testset_rtree(int p1, int p2){ speedtest1_end_test(); } - n = g.szTest*20; + n = g.szTest*100; speedtest1_begin_test(120, "%d one-dimensional overlap slice queries", n); speedtest1_prepare("SELECT count(*) FROM rt1 WHERE y1>=?1 AND y0<=?2"); iStep = mxCoord/n; @@ -1253,7 +1250,7 @@ void testset_rtree(int p1, int p2){ speedtest1_end_test(); if( g.bVerify ){ - n = g.szTest*20; + n = g.szTest*100; speedtest1_begin_test(121, "Verify result from 1-D overlap slice queries"); speedtest1_prepare("SELECT count(*) FROM t1 WHERE y1>=?1 AND y0<=?2"); iStep = mxCoord/n; @@ -1270,7 +1267,7 @@ void testset_rtree(int p1, int p2){ } - n = g.szTest*20; + n = g.szTest*100; speedtest1_begin_test(125, "%d custom geometry callback queries", n); sqlite3_rtree_geometry_callback(g.db, "xslice", xsliceGeometryCallback, 0); speedtest1_prepare("SELECT count(*) FROM rt1 WHERE id MATCH xslice(?1,?2)"); @@ -1286,7 +1283,7 @@ void testset_rtree(int p1, int p2){ } speedtest1_end_test(); - n = g.szTest*80; + n = g.szTest*400; speedtest1_begin_test(130, "%d three-dimensional intersect box queries", n); speedtest1_prepare("SELECT count(*) FROM rt1 WHERE x1>=?1 AND x0<=?2" " AND y1>=?1 AND y0<=?2 AND z1>=?1 AND z0<=?2"); @@ -1299,7 +1296,7 @@ void testset_rtree(int p1, int p2){ } speedtest1_end_test(); - n = g.szTest*100; + n = g.szTest*500; speedtest1_begin_test(140, "%d rowid queries", n); speedtest1_prepare("SELECT * FROM rt1 WHERE id=?1"); for(i=1; i<=n; i++){ @@ -1310,6 +1307,275 @@ void testset_rtree(int p1, int p2){ } #endif /* SQLITE_ENABLE_RTREE */ +/* +** A testset that does key/value storage on tables with many columns. +** This is the kind of workload generated by ORMs such as CoreData. +*/ +void testset_orm(void){ + unsigned i, j, n; + unsigned nRow; + unsigned x1, len; + char zNum[2000]; /* A number name */ + static const char zType[] = /* Types for all non-PK columns, in order */ + "IBBIIITIVVITBTBFBFITTFBTBVBVIFTBBFITFFVBIFIVBVVVBTVTIBBFFIVIBTB" + "TVTTFTVTVFFIITIFBITFTTFFFVBIIBTTITFTFFVVVFIIITVBBVFFTVVB"; + + nRow = n = g.szTest*250; + speedtest1_begin_test(100, "Fill %d rows", n); + speedtest1_exec( + "BEGIN;" + "CREATE TABLE ZLOOKSLIKECOREDATA (" + " ZPK INTEGER PRIMARY KEY," + " ZTERMFITTINGHOUSINGCOMMAND INTEGER," + " ZBRIEFGOBYDODGERHEIGHT BLOB," + " ZCAPABLETRIPDOORALMOND BLOB," + " ZDEPOSITPAIRCOLLEGECOMET INTEGER," + " ZFRAMEENTERSIMPLEMOUTH INTEGER," + " ZHOPEFULGATEHOLECHALK INTEGER," + " ZSLEEPYUSERGRANDBOWL TIMESTAMP," + " ZDEWPEACHCAREERCELERY INTEGER," + " ZHANGERLITHIUMDINNERMEET VARCHAR," + " ZCLUBRELEASELIZARDADVICE VARCHAR," + " ZCHARGECLICKHUMANEHIRE INTEGER," + " ZFINGERDUEPIZZAOPTION TIMESTAMP," + " ZFLYINGDOCTORTABLEMELODY BLOB," + " ZLONGFINLEAVEIMAGEOIL TIMESTAMP," + " ZFAMILYVISUALOWNERMATTER BLOB," + " ZGOLDYOUNGINITIALNOSE FLOAT," + " ZCAUSESALAMITERMCYAN BLOB," + " ZSPREADMOTORBISCUITBACON FLOAT," + " ZGIFTICEFISHGLUEHAIR INTEGER," + " ZNOTICEPEARPOLICYJUICE TIMESTAMP," + " ZBANKBUFFALORECOVERORBIT TIMESTAMP," + " ZLONGDIETESSAYNATURE FLOAT," + " ZACTIONRANGEELEGANTNEUTRON BLOB," + " ZCADETBRIGHTPLANETBANK TIMESTAMP," + " ZAIRFORGIVEHEADFROG BLOB," + " ZSHARKJUSTFRUITMOVIE VARCHAR," + " ZFARMERMORNINGMIRRORCONCERN BLOB," + " ZWOODPOETRYCOBBLERBENCH VARCHAR," + " ZHAFNIUMSCRIPTSALADMOTOR INTEGER," + " ZPROBLEMCLUBPOPOVERJELLY FLOAT," + " ZEIGHTLEADERWORKERMOST TIMESTAMP," + " ZGLASSRESERVEBARIUMMEAL BLOB," + " ZCLAMBITARUGULAFAJITA BLOB," + " ZDECADEJOYOUSWAVEHABIT FLOAT," + " ZCOMPANYSUMMERFIBERELF INTEGER," + " ZTREATTESTQUILLCHARGE TIMESTAMP," + " ZBROWBALANCEKEYCHOWDER FLOAT," + " ZPEACHCOPPERDINNERLAKE FLOAT," + " ZDRYWALLBEYONDBROWNBOWL VARCHAR," + " ZBELLYCRASHITEMLACK BLOB," + " ZTENNISCYCLEBILLOFFICER INTEGER," + " ZMALLEQUIPTHANKSGLUE FLOAT," + " ZMISSREPLYHUMANLIVING INTEGER," + " ZKIWIVISUALPRIDEAPPLE VARCHAR," + " ZWISHHITSKINMOTOR BLOB," + " ZCALMRACCOONPROGRAMDEBIT VARCHAR," + " ZSHINYASSISTLIVINGCRAB VARCHAR," + " ZRESOLVEWRISTWRAPAPPLE VARCHAR," + " ZAPPEALSIMPLESECONDHOUSING BLOB," + " ZCORNERANCHORTAPEDIVER TIMESTAMP," + " ZMEMORYREQUESTSOURCEBIG VARCHAR," + " ZTRYFACTKEEPMILK TIMESTAMP," + " ZDIVERPAINTLEATHEREASY INTEGER," + " ZSORTMISTYQUOTECABBAGE BLOB," + " ZTUNEGASBUFFALOCAPITAL BLOB," + " ZFILLSTOPLAWJOYFUL FLOAT," + " ZSTEELCAREFULPLATENUMBER FLOAT," + " ZGIVEVIVIDDIVINEMEANING INTEGER," + " ZTREATPACKFUTURECONVERT VARCHAR," + " ZCALMLYGEMFINISHEFFECT INTEGER," + " ZCABBAGESOCKEASEMINUTE BLOB," + " ZPLANETFAMILYPUREMEMORY TIMESTAMP," + " ZMERRYCRACKTRAINLEADER BLOB," + " ZMINORWAYPAPERCLASSY TIMESTAMP," + " ZEAGLELINEMINEMAIL VARCHAR," + " ZRESORTYARDGREENLET TIMESTAMP," + " ZYARDOREGANOVIVIDJEWEL TIMESTAMP," + " ZPURECAKEVIVIDNEATLY FLOAT," + " ZASKCONTACTMONITORFUN TIMESTAMP," + " ZMOVEWHOGAMMAINCH VARCHAR," + " ZLETTUCEBIRDMEETDEBATE TIMESTAMP," + " ZGENENATURALHEARINGKITE VARCHAR," + " ZMUFFINDRYERDRAWFORTUNE FLOAT," + " ZGRAYSURVEYWIRELOVE FLOAT," + " ZPLIERSPRINTASKOREGANO INTEGER," + " ZTRAVELDRIVERCONTESTLILY INTEGER," + " ZHUMORSPICESANDKIDNEY TIMESTAMP," + " ZARSENICSAMPLEWAITMUON INTEGER," + " ZLACEADDRESSGROUNDCAREFUL FLOAT," + " ZBAMBOOMESSWASABIEVENING BLOB," + " ZONERELEASEAVERAGENURSE INTEGER," + " ZRADIANTWHENTRYCARD TIMESTAMP," + " ZREWARDINSIDEMANGOINTENSE FLOAT," + " ZNEATSTEWPARTIRON TIMESTAMP," + " ZOUTSIDEPEAHENCOUNTICE TIMESTAMP," + " ZCREAMEVENINGLIPBRANCH FLOAT," + " ZWHALEMATHAVOCADOCOPPER FLOAT," + " ZLIFEUSELEAFYBELL FLOAT," + " ZWEALTHLINENGLEEFULDAY VARCHAR," + " ZFACEINVITETALKGOLD BLOB," + " ZWESTAMOUNTAFFECTHEARING INTEGER," + " ZDELAYOUTCOMEHORNAGENCY INTEGER," + " ZBIGTHINKCONVERTECONOMY BLOB," + " ZBASEGOUDAREGULARFORGIVE TIMESTAMP," + " ZPATTERNCLORINEGRANDCOLBY TIMESTAMP," + " ZCYANBASEFEEDADROIT INTEGER," + " ZCARRYFLOORMINNOWDRAGON TIMESTAMP," + " ZIMAGEPENCILOTHERBOTTOM FLOAT," + " ZXENONFLIGHTPALEAPPLE TIMESTAMP," + " ZHERRINGJOKEFEATUREHOPEFUL FLOAT," + " ZCAPYEARLYRIVETBRUSH FLOAT," + " ZAGEREEDFROGBASKET VARCHAR," + " ZUSUALBODYHALIBUTDIAMOND VARCHAR," + " ZFOOTTAPWORDENTRY VARCHAR," + " ZDISHKEEPBLESTMONITOR FLOAT," + " ZBROADABLESOLIDCASUAL INTEGER," + " ZSQUAREGLEEFULCHILDLIGHT INTEGER," + " ZHOLIDAYHEADPONYDETAIL INTEGER," + " ZGENERALRESORTSKYOPEN TIMESTAMP," + " ZGLADSPRAYKIDNEYGUPPY VARCHAR," + " ZSWIMHEAVYMENTIONKIND BLOB," + " ZMESSYSULFURDREAMFESTIVE BLOB," + " ZSKYSKYCLASSICBRIEF VARCHAR," + " ZDILLASKHOKILEMON FLOAT," + " ZJUNIORSHOWPRESSNOVA FLOAT," + " ZSIZETOEAWARDFRESH TIMESTAMP," + " ZKEYFAILAPRICOTMETAL VARCHAR," + " ZHANDYREPAIRPROTONAIRPORT VARCHAR," + " ZPOSTPROTEINHANDLEACTOR BLOB" + ");" + ); + speedtest1_prepare( + "INSERT INTO ZLOOKSLIKECOREDATA(ZPK,ZAIRFORGIVEHEADFROG," + "ZGIFTICEFISHGLUEHAIR,ZDELAYOUTCOMEHORNAGENCY,ZSLEEPYUSERGRANDBOWL," + "ZGLASSRESERVEBARIUMMEAL,ZBRIEFGOBYDODGERHEIGHT," + "ZBAMBOOMESSWASABIEVENING,ZFARMERMORNINGMIRRORCONCERN," + "ZTREATPACKFUTURECONVERT,ZCAUSESALAMITERMCYAN,ZCALMRACCOONPROGRAMDEBIT," + "ZHOLIDAYHEADPONYDETAIL,ZWOODPOETRYCOBBLERBENCH,ZHAFNIUMSCRIPTSALADMOTOR," + "ZUSUALBODYHALIBUTDIAMOND,ZOUTSIDEPEAHENCOUNTICE,ZDIVERPAINTLEATHEREASY," + "ZWESTAMOUNTAFFECTHEARING,ZSIZETOEAWARDFRESH,ZDEWPEACHCAREERCELERY," + "ZSTEELCAREFULPLATENUMBER,ZCYANBASEFEEDADROIT,ZCALMLYGEMFINISHEFFECT," + "ZHANDYREPAIRPROTONAIRPORT,ZGENENATURALHEARINGKITE,ZBROADABLESOLIDCASUAL," + "ZPOSTPROTEINHANDLEACTOR,ZLACEADDRESSGROUNDCAREFUL,ZIMAGEPENCILOTHERBOTTOM," + "ZPROBLEMCLUBPOPOVERJELLY,ZPATTERNCLORINEGRANDCOLBY,ZNEATSTEWPARTIRON," + "ZAPPEALSIMPLESECONDHOUSING,ZMOVEWHOGAMMAINCH,ZTENNISCYCLEBILLOFFICER," + "ZSHARKJUSTFRUITMOVIE,ZKEYFAILAPRICOTMETAL,ZCOMPANYSUMMERFIBERELF," + "ZTERMFITTINGHOUSINGCOMMAND,ZRESORTYARDGREENLET,ZCABBAGESOCKEASEMINUTE," + "ZSQUAREGLEEFULCHILDLIGHT,ZONERELEASEAVERAGENURSE,ZBIGTHINKCONVERTECONOMY," + "ZPLIERSPRINTASKOREGANO,ZDECADEJOYOUSWAVEHABIT,ZDRYWALLBEYONDBROWNBOWL," + "ZCLUBRELEASELIZARDADVICE,ZWHALEMATHAVOCADOCOPPER,ZBELLYCRASHITEMLACK," + "ZLETTUCEBIRDMEETDEBATE,ZCAPABLETRIPDOORALMOND,ZRADIANTWHENTRYCARD," + "ZCAPYEARLYRIVETBRUSH,ZAGEREEDFROGBASKET,ZSWIMHEAVYMENTIONKIND," + "ZTRAVELDRIVERCONTESTLILY,ZGLADSPRAYKIDNEYGUPPY,ZBANKBUFFALORECOVERORBIT," + "ZFINGERDUEPIZZAOPTION,ZCLAMBITARUGULAFAJITA,ZLONGFINLEAVEIMAGEOIL," + "ZLONGDIETESSAYNATURE,ZJUNIORSHOWPRESSNOVA,ZHOPEFULGATEHOLECHALK," + "ZDEPOSITPAIRCOLLEGECOMET,ZWEALTHLINENGLEEFULDAY,ZFILLSTOPLAWJOYFUL," + "ZTUNEGASBUFFALOCAPITAL,ZGRAYSURVEYWIRELOVE,ZCORNERANCHORTAPEDIVER," + "ZREWARDINSIDEMANGOINTENSE,ZCADETBRIGHTPLANETBANK,ZPLANETFAMILYPUREMEMORY," + "ZTREATTESTQUILLCHARGE,ZCREAMEVENINGLIPBRANCH,ZSKYSKYCLASSICBRIEF," + "ZARSENICSAMPLEWAITMUON,ZBROWBALANCEKEYCHOWDER,ZFLYINGDOCTORTABLEMELODY," + "ZHANGERLITHIUMDINNERMEET,ZNOTICEPEARPOLICYJUICE,ZSHINYASSISTLIVINGCRAB," + "ZLIFEUSELEAFYBELL,ZFACEINVITETALKGOLD,ZGENERALRESORTSKYOPEN," + "ZPURECAKEVIVIDNEATLY,ZKIWIVISUALPRIDEAPPLE,ZMESSYSULFURDREAMFESTIVE," + "ZCHARGECLICKHUMANEHIRE,ZHERRINGJOKEFEATUREHOPEFUL,ZYARDOREGANOVIVIDJEWEL," + "ZFOOTTAPWORDENTRY,ZWISHHITSKINMOTOR,ZBASEGOUDAREGULARFORGIVE," + "ZMUFFINDRYERDRAWFORTUNE,ZACTIONRANGEELEGANTNEUTRON,ZTRYFACTKEEPMILK," + "ZPEACHCOPPERDINNERLAKE,ZFRAMEENTERSIMPLEMOUTH,ZMERRYCRACKTRAINLEADER," + "ZMEMORYREQUESTSOURCEBIG,ZCARRYFLOORMINNOWDRAGON,ZMINORWAYPAPERCLASSY," + "ZDILLASKHOKILEMON,ZRESOLVEWRISTWRAPAPPLE,ZASKCONTACTMONITORFUN," + "ZGIVEVIVIDDIVINEMEANING,ZEIGHTLEADERWORKERMOST,ZMISSREPLYHUMANLIVING," + "ZXENONFLIGHTPALEAPPLE,ZSORTMISTYQUOTECABBAGE,ZEAGLELINEMINEMAIL," + "ZFAMILYVISUALOWNERMATTER,ZSPREADMOTORBISCUITBACON,ZDISHKEEPBLESTMONITOR," + "ZMALLEQUIPTHANKSGLUE,ZGOLDYOUNGINITIALNOSE,ZHUMORSPICESANDKIDNEY)" + "VALUES(?1,?26,?20,?93,?8,?33,?3,?81,?28,?60,?18,?47,?109,?29,?30,?104,?86," + "?54,?92,?117,?9,?58,?97,?61,?119,?73,?107,?120,?80,?99,?31,?96,?85,?50,?71," + "?42,?27,?118,?36,?2,?67,?62,?108,?82,?94,?76,?35,?40,?11,?88,?41,?72,?4," + "?83,?102,?103,?112,?77,?111,?22,?13,?34,?15,?23,?116,?7,?5,?90,?57,?56," + "?75,?51,?84,?25,?63,?37,?87,?114,?79,?38,?14,?10,?21,?48,?89,?91,?110," + "?69,?45,?113,?12,?101,?68,?105,?46,?95,?74,?24,?53,?39,?6,?64,?52,?98," + "?65,?115,?49,?70,?59,?32,?44,?100,?55,?66,?16,?19,?106,?43,?17,?78);" + ); + for(i=0; i=argc-1 ) fatal_error("missing argument on %s\n", argv[i]); pageSize = integerValue(argv[++i]); @@ -1618,6 +1879,8 @@ int main(int argc, char **argv){ testset_main(); }else if( strcmp(zTSet,"debug1")==0 ){ testset_debug1(); + }else if( strcmp(zTSet,"orm")==0 ){ + testset_orm(); }else if( strcmp(zTSet,"cte")==0 ){ testset_cte(); }else if( strcmp(zTSet,"rtree")==0 ){ diff --git a/test/sync2.test b/test/sync2.test new file mode 100644 index 0000000000..4f6ed7754c --- /dev/null +++ b/test/sync2.test @@ -0,0 +1,160 @@ +# 2017 March 16 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# +# Specificly, it tests that "PRAGMA synchronous" appears to work. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix sync2 + +# +# These tests are only applicable when pager pragma are +# enabled. Also, since every test uses an ATTACHed database, they +# are only run when ATTACH is enabled. +# +ifcapable !pager_pragmas||!attach { + finish_test + return +} +if {$::tcl_platform(platform)!="unix" + || [permutation] == "journaltest" + || [permutation] == "inmemory_journal" +} { + finish_test + return +} + +proc execsql_sync {sql} { + set s $::sqlite_sync_count + set res [execsql $sql] + concat [expr $::sqlite_sync_count-$s] $res +} + +proc do_execsql_sync_test {tn sql res} { + uplevel [list do_test $tn [list execsql_sync $sql] [list {*}$res]] +} + +#----------------------------------------------------------------------- +# Tests for journal mode. +# +sqlite3 db test.db +do_execsql_test 1.0 { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); +} + +do_execsql_sync_test 1.1 { INSERT INTO t1 VALUES(3, 4) } 4 + +# synchronous=normal. So, 1 sync on the directory, 1 on the journal, 1 +# on the db file. 3 in total. +do_execsql_test 1.2.1 { PRAGMA main.synchronous = NORMAL } +do_execsql_test 1.2.2 { PRAGMA main.synchronous } 1 +do_execsql_sync_test 1.2.3 { INSERT INTO t1 VALUES(5, 6) } 3 + +# synchronous=off. No syncs. +do_execsql_test 1.3.1 { PRAGMA main.synchronous = OFF } +do_execsql_test 1.3.2 { PRAGMA main.synchronous } 0 +do_execsql_sync_test 1.3.3 { INSERT INTO t1 VALUES(7, 8) } 0 + +# synchronous=full, journal_mode=delete. So, 1 sync on the directory, +# 2 on the journal, 1 on the db file. 4 in total. +do_execsql_test 1.4.1 { PRAGMA main.synchronous = FULL } +do_execsql_test 1.4.2 { PRAGMA main.synchronous } 2 +do_execsql_sync_test 1.4.3 { INSERT INTO t1 VALUES(9, 10) } 4 + +#----------------------------------------------------------------------- +# Tests for wal mode. +# +do_execsql_test 1.5 { PRAGMA journal_mode = wal } {wal} + +# sync=full, journal_mode=wal. One sync on the directory, two on the +# wal file. +do_execsql_sync_test 1.6 { INSERT INTO t1 VALUES(11, 12) } 3 + +# One sync on the wal file. +do_execsql_sync_test 1.7 { INSERT INTO t1 VALUES(13, 14) } 1 + +# No syncs. +do_execsql_test 1.8.1 { PRAGMA main.synchronous = NORMAL } +do_execsql_test 1.8.2 { PRAGMA main.synchronous } 1 +do_execsql_sync_test 1.8.3 { INSERT INTO t1 VALUES(15, 16) } 0 + +# One sync on wal file, one on the db file. +do_execsql_sync_test 1.9 { PRAGMA wal_checkpoint } {2 0 3 3} + +# No syncs. +do_execsql_test 1.10.1 { PRAGMA main.synchronous = OFF } +do_execsql_test 1.10.2 { PRAGMA main.synchronous } 0 +do_execsql_sync_test 1.10.3 { INSERT INTO t1 VALUES(17, 18) } 0 + +#----------------------------------------------------------------------- +# Tests for the compile time settings SQLITE_DEFAULT_SYNCHRONOUS and +# SQLITE_DEFAULT_WAL_SYNCHRONOUS. These tests only run if the former +# is set to "2" and the latter to "1". This is not the default, but +# it is currently the recommended configuration. +# +# https://sqlite.org/compile.html#recommended_compile_time_options +# +if {$SQLITE_DEFAULT_SYNCHRONOUS==2 && $SQLITE_DEFAULT_WAL_SYNCHRONOUS==1} { + + db close + sqlite3 db test.db + + # Wal mode, sync=normal. The first transaction does one sync on directory, + # one on the wal file. The second does no syncs. + do_execsql_sync_test 1.11.1 { INSERT INTO t1 VALUES(19, 20) } 2 + do_execsql_sync_test 1.11.2 { INSERT INTO t1 VALUES(21, 22) } 0 + do_execsql_test 1.11.3 { PRAGMA main.synchronous } 1 + + # One sync on wal file, one on the db file. + do_execsql_sync_test 1.12 { PRAGMA wal_checkpoint } {2 0 2 2} + + # First transaction syncs the wal file once, the second not at all. + # one on the wal file. The second does no syncs. + do_execsql_sync_test 1.13.1 { INSERT INTO t1 VALUES(22, 23) } 1 + do_execsql_sync_test 1.13.2 { INSERT INTO t1 VALUES(24, 25) } 0 + + do_execsql_test 1.14 { PRAGMA journal_mode = delete } {delete} + + # Delete mode, sync=full. The first transaction does one sync on + # directory, two on the journal file, one on the db. The second does + # the same. + do_execsql_sync_test 1.15.1 { INSERT INTO t1 VALUES(26, 27) } 4 + do_execsql_sync_test 1.15.2 { INSERT INTO t1 VALUES(28, 29) } 4 + do_execsql_test 1.15.3 { PRAGMA main.synchronous } 2 + + # Switch back to wal mode. + do_execsql_test 1.16 { PRAGMA journal_mode = wal } {wal} + + do_execsql_sync_test 1.17.1 { INSERT INTO t1 VALUES(30, 31) } 2 + do_execsql_sync_test 1.17.2 { INSERT INTO t1 VALUES(32, 33) } 0 + do_execsql_test 1.17.3 { PRAGMA main.synchronous } 1 + + # Now set synchronous=off, then switch back to delete mode. Check + # that the db handle is still using synchronous=off. + do_execsql_test 1.18.3 { PRAGMA main.synchronous=off } + do_execsql_test 1.18 { PRAGMA journal_mode = delete } {delete} + + do_execsql_sync_test 1.19.1 { INSERT INTO t1 VALUES(34, 35) } 0 + do_execsql_sync_test 1.19.2 { INSERT INTO t1 VALUES(36, 37) } 0 + do_execsql_test 1.19.3 { PRAGMA main.synchronous } 0 + + # Close and reopen the db. Back to synchronous=normal. + db close + sqlite3 db test.db + do_execsql_sync_test 1.20.1 { INSERT INTO t1 VALUES(38, 39) } 4 + do_execsql_sync_test 1.20.2 { INSERT INTO t1 VALUES(40, 41) } 4 + do_execsql_test 1.20.3 { PRAGMA main.synchronous } 2 +} + +finish_test diff --git a/test/tabfunc01.test b/test/tabfunc01.test index f25d070847..dcaafa420c 100644 --- a/test/tabfunc01.test +++ b/test/tabfunc01.test @@ -148,60 +148,68 @@ do_execsql_test tabfunc01-600 { do_test tabfunc01-700 { - set PTR [intarray_addr 5 7 13 17 23] + set PTR1 [intarray_addr 5 7 13 17 23] db eval { - SELECT b FROM t600, carray($PTR,5) WHERE a=value; + SELECT b FROM t600, carray($PTR1,5) WHERE a=value; } } {(005) (007) (013) (017) (023)} do_test tabfunc01-701 { db eval { - SELECT b FROM t600 WHERE a IN carray($PTR,5,'int32'); + SELECT b FROM t600 WHERE a IN carray($PTR1,5,'int32'); } } {(005) (007) (013) (017) (023)} do_test tabfunc01-702 { db eval { - SELECT b FROM t600 WHERE a IN carray($PTR,4,'int32'); + SELECT b FROM t600 WHERE a IN carray($PTR1,4,'int32'); } } {(005) (007) (013) (017)} do_catchsql_test tabfunc01-710 { - SELECT b FROM t600 WHERE a IN carray($PTR,5,'int33'); + SELECT b FROM t600 WHERE a IN carray($PTR1,5,'int33'); } {1 {unknown datatype: 'int33'}} do_test tabfunc01-720 { - set PTR [int64array_addr 5 7 13 17 23] + set PTR2 [int64array_addr 5 7 13 17 23] db eval { - SELECT b FROM t600, carray($PTR,5,'int64') WHERE a=value; + SELECT b FROM t600, carray($PTR2,5,'int64') WHERE a=value; } } {(005) (007) (013) (017) (023)} do_test tabfunc01-721 { db eval { - SELECT remember(123,$PTR); - SELECT value FROM carray($PTR,5,'int64'); + SELECT remember(123,$PTR2); + SELECT value FROM carray($PTR2,5,'int64'); } } {123 123 7 13 17 23} do_test tabfunc01-722 { - set PTR2 [expr {$PTR+16}] + set PTR3 [expr {$PTR2+16}] db eval { - SELECT remember(987,$PTR2); - SELECT value FROM carray($PTR,5,'int64'); + SELECT remember(987,$PTR3); + SELECT value FROM carray($PTR2,5,'int64'); } } {987 123 7 987 17 23} do_test tabfunc01-730 { - set PTR [doublearray_addr 5.0 7.0 13.0 17.0 23.0] + set PTR4 [doublearray_addr 5.0 7.0 13.0 17.0 23.0] db eval { - SELECT b FROM t600, carray($PTR,5,'double') WHERE a=value; + SELECT b FROM t600, carray($PTR4,5,'double') WHERE a=value; } } {(005) (007) (013) (017) (023)} do_test tabfunc01-740 { - set PTR [textarray_addr 5 7 13 17 23] + set PTR5 [textarray_addr x5 x7 x13 x17 x23] db eval { - SELECT b FROM t600, carray($PTR,5,'char*') WHERE a=value; + SELECT b FROM t600, carray($PTR5,5,'char*') WHERE a=trim(value,'x'); } } {(005) (007) (013) (017) (023)} +do_test tabfunc01-750 { + db eval { + SELECT aa.value, bb.value, '|' + FROM carray($PTR4,5,'double') AS aa + JOIN carray($PTR5,5,'char*') AS bb ON aa.rowid=bb.rowid; + } +} {5.0 x5 | 7.0 x7 | 13.0 x13 | 17.0 x17 | 23.0 x23 |} +# Free up memory allocations intarray_addr int64array_addr doublearray_addr diff --git a/test/tester.tcl b/test/tester.tcl index 814788ba45..dc6547d033 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -461,7 +461,7 @@ if {[info exists cmdlinearg]==0} { } {^-+backtrace=.+$} { foreach {dummy cmdlinearg(backtrace)} [split $a =] break - sqlite3_memdebug_backtrace $value + sqlite3_memdebug_backtrace $cmdlinearg(backtrace) } {^-+binarylog=.+$} { foreach {dummy cmdlinearg(binarylog)} [split $a =] break @@ -919,10 +919,37 @@ proc normalize_list {L} { set L2 } -proc do_execsql_test {testname sql {result {}}} { +# Either: +# +# do_execsql_test TESTNAME SQL ?RES? +# do_execsql_test -db DB TESTNAME SQL ?RES? +# +proc do_execsql_test {args} { + set db db + if {[lindex $args 0]=="-db"} { + set db [lindex $args 1] + set args [lrange $args 2 end] + } + + if {[llength $args]==2} { + foreach {testname sql} $args {} + set result "" + } elseif {[llength $args]==3} { + foreach {testname sql result} $args {} + } else { + error [string trim { + wrong # args: should be "do_execsql_test ?-db DB? testname sql ?result?" + }] + } + fix_testname testname - uplevel do_test [list $testname] [list "execsql {$sql}"] [list [list {*}$result]] + + uplevel do_test \ + [list $testname] \ + [list "execsql {$sql} $db"] \ + [list [list {*}$result]] } + proc do_catchsql_test {testname sql result} { fix_testname testname uplevel do_test [list $testname] [list "catchsql {$sql}"] [list $result] @@ -1506,6 +1533,7 @@ proc crashsql {args} { set tclbody {} set crashfile "" set dc "" + set dfltvfs 0 set sql [lindex $args end] for {set ii 0} {$ii < [llength $args]-1} {incr ii 2} { @@ -1519,7 +1547,8 @@ proc crashsql {args} { elseif {$n>1 && [string first $z -file]==0} {set crashfile $z2} \ elseif {$n>1 && [string first $z -tclbody]==0} {set tclbody $z2} \ elseif {$n>1 && [string first $z -blocksize]==0} {set blocksize "-s $z2" } \ - elseif {$n>1 && [string first $z -characteristics]==0} {set dc "-c {$z2}" } \ + elseif {$n>1 && [string first $z -characteristics]==0} {set dc "-c {$z2}" }\ + elseif {$n>1 && [string first $z -dfltvfs]==0} {set dfltvfs $z2 }\ else { error "Unrecognized option: $z" } } @@ -1533,7 +1562,7 @@ proc crashsql {args} { set cfile [string map {\\ \\\\} [file nativename [file join [get_pwd] $crashfile]]] set f [open crash.tcl w] - puts $f "sqlite3_crash_enable 1" + puts $f "sqlite3_crash_enable 1 $dfltvfs" puts $f "sqlite3_crashparams $blocksize $dc $crashdelay $cfile" puts $f "sqlite3_test_control_pending_byte $::sqlite_pending_byte" diff --git a/test/tkt-78e04e52ea.test b/test/tkt-78e04e52ea.test index 975e5b3d1c..963ecf18d1 100644 --- a/test/tkt-78e04e52ea.test +++ b/test/tkt-78e04e52ea.test @@ -42,7 +42,7 @@ do_test tkt-78e04-1.3 { } {} do_test tkt-78e04-1.4 { execsql { - EXPLAIN QUERY PLAN SELECT "" FROM "" WHERE "" LIKE 'abc%'; + EXPLAIN QUERY PLAN SELECT "" FROM "" WHERE "" LIKE '1abc%'; } } {0 0 0 {SCAN TABLE USING COVERING INDEX i1}} do_test tkt-78e04-1.5 { diff --git a/test/update2.test b/test/update2.test new file mode 100644 index 0000000000..6a359d11a9 --- /dev/null +++ b/test/update2.test @@ -0,0 +1,205 @@ +# 2017 January 9 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix update2 + +db func repeat [list string repeat] + +#------------------------------------------------------------------------- +# 1.1.* A one-pass UPDATE that does balance() operations on the IPK index +# that it is scanning. +# +# 1.2.* Same again, but with a WITHOUT ROWID table. +# +set nrow [expr 10] +do_execsql_test 1.1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + CREATE TABLE t2(a INTEGER PRIMARY KEY, b); + WITH s(i) AS ( SELECT 0 UNION ALL SELECT i+1 FROM s WHERE i<$nrow ) + INSERT INTO t1(b) SELECT char((i % 26) + 65) FROM s; + INSERT INTO t2 SELECT * FROM t1; +} + +do_execsql_test 1.1.1 { + UPDATE t1 SET b = repeat(b, 100) +} + +do_execsql_test 1.1.2 { + SELECT * FROM t1; +} [db eval { SELECT a, repeat(b, 100) FROM t2 }] + +do_execsql_test 1.2.0 { + DROP TABLE t1; + CREATE TABLE t1(a INT PRIMARY KEY, b) WITHOUT ROWID; + WITH s(i) AS ( SELECT 0 UNION ALL SELECT i+1 FROM s WHERE i<$nrow ) + INSERT INTO t1(a, b) SELECT i+1, char((i % 26) + 65) FROM s; +} + +#explain_i { UPDATE t1 SET b = repeat(b, 100) } +do_execsql_test 1.2.1 { + UPDATE t1 SET b = repeat(b, 100) +} + +do_execsql_test 1.2.2 { + SELECT * FROM t1; +} [db eval { SELECT a, repeat(b, 100) FROM t2 }] + + +#------------------------------------------------------------------------- +# A one-pass UPDATE that does balance() operations on the IPK index +# that it is scanning. +# +do_execsql_test 2.1 { + CREATE TABLE t3(a PRIMARY KEY, b, c); + CREATE INDEX t3i ON t3(b); +} {} +do_execsql_test 2.2 { UPDATE t3 SET c=1 WHERE b=? } {} +do_execsql_test 2.3 { UPDATE t3 SET c=1 WHERE rowid=? } {} + +#------------------------------------------------------------------------- +# +do_execsql_test 3.0 { + CREATE TABLE t4(a PRIMARY KEY, b, c) WITHOUT ROWID; + CREATE INDEX t4c ON t4(c); + INSERT INTO t4 VALUES(1, 2, 3); + INSERT INTO t4 VALUES(2, 3, 4); +} + +do_execsql_test 3.1 { + UPDATE t4 SET c=c+2 WHERE c>2; + SELECT a, c FROM t4 ORDER BY a; +} {1 5 2 6} + +#------------------------------------------------------------------------- +# +foreach {tn sql} { + 1 { + CREATE TABLE b1(a INTEGER PRIMARY KEY, b, c); + CREATE TABLE c1(a INTEGER PRIMARY KEY, b, c, d) + } + 2 { + CREATE TABLE b1(a INT PRIMARY KEY, b, c) WITHOUT ROWID; + CREATE TABLE c1(a INT PRIMARY KEY, b, c, d) WITHOUT ROWID; + } +} { + execsql { DROP TABLE IF EXISTS b1; DROP TABLE IF EXISTS c1; } + execsql $sql + + do_execsql_test 4.$tn.0 { + CREATE UNIQUE INDEX b1c ON b1(c); + INSERT INTO b1 VALUES(1, 'a', 1); + INSERT INTO b1 VALUES(2, 'b', 15); + INSERT INTO b1 VALUES(3, 'c', 3); + INSERT INTO b1 VALUES(4, 'd', 4); + INSERT INTO b1 VALUES(5, 'e', 5); + INSERT INTO b1 VALUES(6, 'f', 6); + INSERT INTO b1 VALUES(7, 'g', 7); + } + + do_execsql_test 4.$tn.1 { + UPDATE OR REPLACE b1 SET c=c+10 WHERE a BETWEEN 4 AND 7; + SELECT * FROM b1 ORDER BY a; + } { + 1 a 1 + 3 c 3 + 4 d 14 + 5 e 15 + 6 f 16 + 7 g 17 + } + + do_execsql_test 4.$tn.2 { + CREATE INDEX c1d ON c1(d, b); + CREATE UNIQUE INDEX c1c ON c1(c, b); + + INSERT INTO c1 VALUES(1, 'a', 1, 1); + INSERT INTO c1 VALUES(2, 'a', 15, 2); + INSERT INTO c1 VALUES(3, 'a', 3, 3); + INSERT INTO c1 VALUES(4, 'a', 4, 4); + INSERT INTO c1 VALUES(5, 'a', 5, 5); + INSERT INTO c1 VALUES(6, 'a', 6, 6); + INSERT INTO c1 VALUES(7, 'a', 7, 7); + } + + do_execsql_test 4.$tn.3 { + UPDATE OR REPLACE c1 SET c=c+10 WHERE d BETWEEN 4 AND 7; + SELECT * FROM c1 ORDER BY a; + } { + 1 a 1 1 + 3 a 3 3 + 4 a 14 4 + 5 a 15 5 + 6 a 16 6 + 7 a 17 7 + } + + do_execsql_test 4.$tn.4 { PRAGMA integrity_check } ok + + do_execsql_test 4.$tn.5 { + DROP INDEX c1d; + DROP INDEX c1c; + DELETE FROM c1; + + INSERT INTO c1 VALUES(1, 'a', 1, 1); + INSERT INTO c1 VALUES(2, 'a', 15, 2); + INSERT INTO c1 VALUES(3, 'a', 3, 3); + INSERT INTO c1 VALUES(4, 'a', 4, 4); + INSERT INTO c1 VALUES(5, 'a', 5, 5); + INSERT INTO c1 VALUES(6, 'a', 6, 6); + INSERT INTO c1 VALUES(7, 'a', 7, 7); + + CREATE INDEX c1d ON c1(d); + CREATE UNIQUE INDEX c1c ON c1(c); + } + + do_execsql_test 4.$tn.6 { + UPDATE OR REPLACE c1 SET c=c+10 WHERE d BETWEEN 4 AND 7; + SELECT * FROM c1 ORDER BY a; + } { + 1 a 1 1 + 3 a 3 3 + 4 a 14 4 + 5 a 15 5 + 6 a 16 6 + 7 a 17 7 + } +} + +#------------------------------------------------------------------------- +# +do_execsql_test 5.0 { + CREATE TABLE x1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX x1c ON x1(b, c); + INSERT INTO x1 VALUES(1, 'a', 1); + INSERT INTO x1 VALUES(2, 'a', 2); + INSERT INTO x1 VALUES(3, 'a', 3); +} + +do_execsql_test 5.1.1 { + UPDATE x1 SET c=c+1 WHERE b='a'; +} + +do_execsql_test 5.1.2 { + SELECT * FROM x1; +} {1 a 2 2 a 3 3 a 4} + +do_test 5.2 { + catch { array unset A } + db eval { EXPLAIN UPDATE x1 SET c=c+1 WHERE b='a' } { incr A($opcode) } + set A(NotExists) +} {1} + + +finish_test + diff --git a/test/vtab1.test b/test/vtab1.test index 6b6a0e2683..59ab959746 100644 --- a/test/vtab1.test +++ b/test/vtab1.test @@ -1295,25 +1295,25 @@ do_execsql_test 18.1.0 { CREATE INDEX i6 ON t6(b, a); INSERT INTO t6 VALUES(1, 'Peter'); INSERT INTO t6 VALUES(2, 'Andrew'); - INSERT INTO t6 VALUES(3, 'James'); - INSERT INTO t6 VALUES(4, 'John'); + INSERT INTO t6 VALUES(3, '8James'); + INSERT INTO t6 VALUES(4, '8John'); INSERT INTO t6 VALUES(5, 'Phillip'); INSERT INTO t6 VALUES(6, 'Bartholomew'); CREATE VIRTUAL TABLE e6 USING echo(t6); } foreach {tn sql res filter} { - 1.1 "SELECT a FROM e6 WHERE b>'James'" {4 1 5} - {xFilter {SELECT rowid, a, b FROM 't6' WHERE b > ?} James} + 1.1 "SELECT a FROM e6 WHERE b>'8James'" {4 2 6 1 5} + {xFilter {SELECT rowid, a, b FROM 't6' WHERE b > ?} 8James} - 1.2 "SELECT a FROM e6 WHERE b>='J' AND b<'K'" {3 4} - {xFilter {SELECT rowid, a, b FROM 't6' WHERE b >= ? AND b < ?} J K} + 1.2 "SELECT a FROM e6 WHERE b>='8' AND b<'9'" {3 4} + {xFilter {SELECT rowid, a, b FROM 't6' WHERE b >= ? AND b < ?} 8 9} - 1.3 "SELECT a FROM e6 WHERE b LIKE 'J%'" {3 4} - {xFilter {SELECT rowid, a, b FROM 't6' WHERE b like ?} J%} + 1.3 "SELECT a FROM e6 WHERE b LIKE '8J%'" {3 4} + {xFilter {SELECT rowid, a, b FROM 't6' WHERE b like ?} 8J%} - 1.4 "SELECT a FROM e6 WHERE b LIKE 'j%'" {3 4} - {xFilter {SELECT rowid, a, b FROM 't6' WHERE b like ?} j%} + 1.4 "SELECT a FROM e6 WHERE b LIKE '8j%'" {3 4} + {xFilter {SELECT rowid, a, b FROM 't6' WHERE b like ?} 8j%} } { set echo_module {} do_execsql_test 18.$tn.1 $sql $res @@ -1322,11 +1322,11 @@ foreach {tn sql res filter} { do_execsql_test 18.2.0 { PRAGMA case_sensitive_like = ON } foreach {tn sql res filter} { - 2.1 "SELECT a FROM e6 WHERE b LIKE 'J%'" {3 4} - {xFilter {SELECT rowid, a, b FROM 't6' WHERE b like ?} J%} + 2.1 "SELECT a FROM e6 WHERE b LIKE '8J%'" {3 4} + {xFilter {SELECT rowid, a, b FROM 't6' WHERE b like ?} 8J%} - 2.2 "SELECT a FROM e6 WHERE b LIKE 'j%'" {} - {xFilter {SELECT rowid, a, b FROM 't6' WHERE b like ?} j%} + 2.2 "SELECT a FROM e6 WHERE b LIKE '8j%'" {} + {xFilter {SELECT rowid, a, b FROM 't6' WHERE b like ?} 8j%} } { set echo_module {} do_execsql_test 18.$tn.1 $sql $res diff --git a/test/vtabH.test b/test/vtabH.test index f2a116f350..c5684ff516 100644 --- a/test/vtabH.test +++ b/test/vtabH.test @@ -31,14 +31,14 @@ do_execsql_test 1.0 { } foreach {tn sql expect} { - 1 "SELECT * FROM e6 WHERE b LIKE 'abc'" { + 1 "SELECT * FROM e6 WHERE b LIKE '8abc'" { xBestIndex {SELECT rowid, a, b FROM 't6' WHERE b like ?} - xFilter {SELECT rowid, a, b FROM 't6' WHERE b like ?} abc + xFilter {SELECT rowid, a, b FROM 't6' WHERE b like ?} 8abc } - 2 "SELECT * FROM e6 WHERE b GLOB 'abc'" { + 2 "SELECT * FROM e6 WHERE b GLOB '8abc'" { xBestIndex {SELECT rowid, a, b FROM 't6' WHERE b glob ?} - xFilter {SELECT rowid, a, b FROM 't6' WHERE b glob ?} abc + xFilter {SELECT rowid, a, b FROM 't6' WHERE b glob ?} 8abc } } { do_test 1.$tn { @@ -121,29 +121,43 @@ if {$tcl_platform(platform)!="windows" || \ SELECT name FROM fsdir WHERE dir = '.' AND name = '.' } {test.db .} + proc sort_files { names {nocase false} } { + if {$nocase && $::tcl_platform(platform) eq "windows"} { + return [lsort -nocase $names] + } else { + return [lsort $names] + } + } + proc list_root_files {} { if {$::tcl_platform(platform) eq "windows"} { - set res [list] - foreach name [glob -directory $::env(fstreeDrive)/ -- *] { + set res [list]; set dir $::env(fstreeDrive)/; set names [list] + eval lappend names [glob -nocomplain -directory $dir -- *] + foreach name $names { if {[string index [file tail $name] 0] eq "."} continue + if {[file attributes $name -hidden]} continue + if {[file attributes $name -system]} continue lappend res $name } - return $res + return [sort_files $res true] } else { - return [string map {/ {}} [glob /*]] + return [sort_files [string map {/ {}} [glob -nocomplain -- /*]]] } } proc list_files { pattern } { if {$::tcl_platform(platform) eq "windows"} { - set res [list] - foreach name [glob -nocomplain $pattern] { + set res [list]; set names [list] + eval lappend names [glob -nocomplain -- $pattern] + foreach name $names { if {[string index [file tail $name] 0] eq "."} continue + if {[file attributes $name -hidden]} continue + if {[file attributes $name -system]} continue lappend res $name } - return $res + return [sort_files $res] } else { - return [glob -nocomplain $pattern] + return [sort_files [glob -nocomplain -- $pattern]] } } @@ -153,18 +167,19 @@ if {$tcl_platform(platform)!="windows" || \ # set res [list] set root_files [list_root_files] - set num_root_files [llength $root_files] - set lim_root_files [expr {$num_root_files > 5 ? 5 : $num_root_files}] - foreach p [lrange $root_files 0 [expr {$lim_root_files - 1}]] { + foreach p $root_files { if {$::tcl_platform(platform) eq "windows"} { - if {[regexp {\$} $p]} {incr lim_root_files -1} else {lappend res $p} + if {![regexp {\$} $p]} {lappend res $p} } else { lappend res "/$p" } } - do_execsql_test 3.1 [subst { - SELECT path FROM fstree WHERE path NOT GLOB '*\$*' LIMIT $lim_root_files; - }] $res + set num_root_files [llength $root_files] + do_test 3.1 { + sort_files [execsql { + SELECT path FROM fstree WHERE path NOT GLOB '*\$*' LIMIT $num_root_files; + }] true + } [sort_files $res true] # Read all entries in the current directory. # @@ -182,7 +197,7 @@ if {$tcl_platform(platform)!="windows" || \ set res [contents $pwd] do_execsql_test 3.2 { SELECT path FROM fstree WHERE path GLOB $pwd ORDER BY 1 - } [lsort $res] + } [sort_files $res] # Add some sub-directories and files to the current directory. # diff --git a/tool/GetTclKit.bat b/tool/GetTclKit.bat index 8764b195dd..1c9f92bc12 100644 --- a/tool/GetTclKit.bat +++ b/tool/GetTclKit.bat @@ -124,7 +124,7 @@ IF NOT EXIST "%FRAMEWORKDIR%\csc.exe" ( GOTO errors ) -SET PATH=%FRAMEWORKDIR%;%PATH% +CALL :fn_PrependToPath FRAMEWORKDIR :skip_addToPath @@ -246,6 +246,18 @@ GOTO no_errors ENDLOCAL && SET %1=%VALUE% GOTO :EOF +:fn_PrependToPath + IF NOT DEFINED %1 GOTO :EOF + SETLOCAL + SET __ECHO_CMD=ECHO %%%1%% + FOR /F "delims=" %%V IN ('%__ECHO_CMD%') DO ( + SET VALUE=%%V + ) + SET VALUE=%VALUE:"=% + REM " + ENDLOCAL && SET PATH=%VALUE%;%PATH% + GOTO :EOF + :fn_ResetErrorLevel VERIFY > NUL GOTO :EOF diff --git a/tool/build-all-msvc.bat b/tool/build-all-msvc.bat index 497b05e316..aaeb67bdfb 100755 --- a/tool/build-all-msvc.bat +++ b/tool/build-all-msvc.bat @@ -460,7 +460,7 @@ FOR %%P IN (%PLATFORMS%) DO ( REM REM NOTE: Reset the PATH here to the absolute bare minimum required. REM - SET PATH=%TOOLPATH%;%SystemRoot%\System32;%SystemRoot% + CALL :fn_ResetPath REM REM NOTE: This is the inner loop. There are normally two iterations, one @@ -818,6 +818,10 @@ GOTO no_errors CALL :fn_ResetErrorLevel GOTO :EOF +:fn_ResetPath + SET PATH=%TOOLPATH%;%SystemRoot%\System32;%SystemRoot% + GOTO :EOF + :fn_AppendVariable SET __ECHO_CMD=ECHO %%%1%% IF DEFINED %1 ( diff --git a/tool/kvtest-speed.sh b/tool/kvtest-speed.sh new file mode 100644 index 0000000000..5f2c8345be --- /dev/null +++ b/tool/kvtest-speed.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# +# A script for running speed tests using kvtest. +# +# The test database must be set up first. Recommended +# command-line: +# +# ./kvtest init kvtest.db --count 100K --size 12K --variance 5K + +if test "$1" = "" +then + echo "Usage: $0 OUTPUTFILE [OPTIONS]" + exit +fi +NAME=$1 +shift +OPTS="-DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DIRECT_OVERFLOW_READ -DUSE_PREAD" +KVARGS="--count 100K --stats" +gcc -g -Os -I. $OPTS $* kvtest.c sqlite3.c -o kvtest + +# First run using SQL +rm cachegrind.out.[1-9][0-9]* +valgrind --tool=cachegrind ./kvtest run kvtest.db $KVARGS 2>&1 | tee summary-kvtest-$NAME.txt +mv cachegrind.out.[1-9][0-9]* cachegrind.out.sql-$NAME +cg_anno.tcl cachegrind.out.sql-$NAME >cout-kvtest-sql-$NAME.txt + +# Second run using the sqlite3_blob object +valgrind --tool=cachegrind ./kvtest run kvtest.db $KVARGS --blob-api 2>&1 | tee -a summary-kvtest-$NAME.txt +mv cachegrind.out.[1-9][0-9]* cachegrind.out.$NAME +cg_anno.tcl cachegrind.out.$NAME >cout-kvtest-$NAME.txt + +# Diff the sqlite3_blob API analysis for non-trunk runs. +if test "$NAME" != "trunk"; then + fossil test-diff --tk cout-kvtest-trunk.txt cout-kvtest-$NAME.txt & +fi diff --git a/tool/lempar.c b/tool/lempar.c index dab8343f3a..21ed9d79ab 100644 --- a/tool/lempar.c +++ b/tool/lempar.c @@ -316,6 +316,31 @@ static int yyGrowStack(yyParser *p){ # define YYMALLOCARGTYPE size_t #endif +/* Initialize a new parser that has already been allocated. +*/ +void ParseInit(void *yypParser){ + yyParser *pParser = (yyParser*)yypParser; +#ifdef YYTRACKMAXSTACKDEPTH + pParser->yyhwm = 0; +#endif +#if YYSTACKDEPTH<=0 + pParser->yytos = NULL; + pParser->yystack = NULL; + pParser->yystksz = 0; + if( yyGrowStack(pParser) ){ + pParser->yystack = &pParser->yystk0; + pParser->yystksz = 1; + } +#endif +#ifndef YYNOERRORRECOVERY + pParser->yyerrcnt = -1; +#endif + pParser->yytos = pParser->yystack; + pParser->yystack[0].stateno = 0; + pParser->yystack[0].major = 0; +} + +#ifndef Parse_ENGINEALWAYSONSTACK /* ** This function allocates a new parser. ** The only argument is a pointer to a function which works like @@ -331,28 +356,11 @@ static int yyGrowStack(yyParser *p){ void *ParseAlloc(void *(*mallocProc)(YYMALLOCARGTYPE)){ yyParser *pParser; pParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) ); - if( pParser ){ -#ifdef YYTRACKMAXSTACKDEPTH - pParser->yyhwm = 0; -#endif -#if YYSTACKDEPTH<=0 - pParser->yytos = NULL; - pParser->yystack = NULL; - pParser->yystksz = 0; - if( yyGrowStack(pParser) ){ - pParser->yystack = &pParser->yystk0; - pParser->yystksz = 1; - } -#endif -#ifndef YYNOERRORRECOVERY - pParser->yyerrcnt = -1; -#endif - pParser->yytos = pParser->yystack; - pParser->yystack[0].stateno = 0; - pParser->yystack[0].major = 0; - } + if( pParser ) ParseInit(pParser); return pParser; } +#endif /* Parse_ENGINEALWAYSONSTACK */ + /* The following function deletes the "minor type" or semantic value ** associated with a symbol. The symbol can be either a terminal @@ -406,6 +414,18 @@ static void yy_pop_parser_stack(yyParser *pParser){ yy_destructor(pParser, yytos->major, &yytos->minor); } +/* +** Clear all secondary memory allocations from the parser +*/ +void ParseFinalize(void *p){ + yyParser *pParser = (yyParser*)p; + while( pParser->yytos>pParser->yystack ) yy_pop_parser_stack(pParser); +#if YYSTACKDEPTH<=0 + if( pParser->yystack!=&pParser->yystk0 ) free(pParser->yystack); +#endif +} + +#ifndef Parse_ENGINEALWAYSONSTACK /* ** Deallocate and destroy a parser. Destructors are called for ** all stack elements before shutting the parser down. @@ -418,16 +438,13 @@ void ParseFree( void *p, /* The parser to be deleted */ void (*freeProc)(void*) /* Function used to reclaim memory */ ){ - yyParser *pParser = (yyParser*)p; #ifndef YYPARSEFREENEVERNULL - if( pParser==0 ) return; + if( p==0 ) return; #endif - while( pParser->yytos>pParser->yystack ) yy_pop_parser_stack(pParser); -#if YYSTACKDEPTH<=0 - if( pParser->yystack!=&pParser->yystk0 ) free(pParser->yystack); -#endif - (*freeProc)((void*)pParser); + ParseFinalize(p); + (*freeProc)(p); } +#endif /* Parse_ENGINEALWAYSONSTACK */ /* ** Return the peak depth of the stack for a parser. diff --git a/tool/max-limits.c b/tool/max-limits.c new file mode 100644 index 0000000000..d019974426 --- /dev/null +++ b/tool/max-limits.c @@ -0,0 +1,41 @@ +/* +** Link this program against an SQLite library of unknown provenance in order +** to display the compile-time maximum values for various settings. +*/ +#include "sqlite3.h" +#include + +static const struct { + int eCode; + char *zName; +} aLimit[] = { + { SQLITE_LIMIT_LENGTH, "SQLITE_MAX_LENGTH" }, + { SQLITE_LIMIT_SQL_LENGTH, "SQLITE_MAX_SQL_LENGTH" }, + { SQLITE_LIMIT_COLUMN, "SQLITE_MAX_COLUMN" }, + { SQLITE_LIMIT_EXPR_DEPTH, "SQLITE_MAX_EXPR_DEPTH" }, + { SQLITE_LIMIT_COMPOUND_SELECT, "SQLITE_MAX_COMPOUND_SELECT" }, + { SQLITE_LIMIT_VDBE_OP, "SQLITE_MAX_VDBE_OP" }, + { SQLITE_LIMIT_FUNCTION_ARG, "SQLITE_MAX_FUNCTION_ARG" }, + { SQLITE_LIMIT_ATTACHED, "SQLITE_MAX_ATTACHED" }, + { SQLITE_LIMIT_LIKE_PATTERN_LENGTH, "SQLITE_MAX_LIKE_PATTERN_LENGTH" }, + { SQLITE_LIMIT_VARIABLE_NUMBER, "SQLITE_MAX_VARIABLE_NUMBER" }, + { SQLITE_LIMIT_TRIGGER_DEPTH, "SQLITE_MAX_TRIGGER_DEPTH" }, + { SQLITE_LIMIT_WORKER_THREADS, "SQLITE_MAX_WORKER_THREADS" }, +}; + +static int maxLimit(sqlite3 *db, int eCode){ + int iOrig = sqlite3_limit(db, eCode, 0x7fffffff); + return sqlite3_limit(db, eCode, iOrig); +} + +int main(int argc, char **argv){ + sqlite3 *db; + int j, rc; + rc = sqlite3_open(":memory:", &db); + if( rc==SQLITE_OK ){ + for(j=0; j sqlite3.def dumpbin /all $(LIBOBJ) \\ - | .\Replace.exe "^\s+/EXPORT:_?(sqlite3_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \\ + | .\Replace.exe "^\s+/EXPORT:_?(sqlite3(?:session|changeset)?_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \\ | sort >> sqlite3.def }]] diff --git a/tool/mkpragmatab.tcl b/tool/mkpragmatab.tcl index c22f72d491..59b245cc76 100644 --- a/tool/mkpragmatab.tcl +++ b/tool/mkpragmatab.tcl @@ -225,8 +225,8 @@ set pragma_def { NAME: stats FLAG: NeedSchema Result0 SchemaReq - COLS: table index width height - IF: !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) + COLS: tbl idx wdth hght flgs + IF: !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) && defined(SQLITE_DEBUG) NAME: index_info TYPE: INDEX_INFO @@ -361,6 +361,9 @@ set pragma_def { NAME: threads FLAG: Result0 + + NAME: optimize + FLAG: Result1 } # Open the output file diff --git a/tool/mksqlite3h.tcl b/tool/mksqlite3h.tcl index 4f8a1fd497..9d307d1b1e 100644 --- a/tool/mksqlite3h.tcl +++ b/tool/mksqlite3h.tcl @@ -73,7 +73,13 @@ close $in # Set up patterns for recognizing API declarations. # set varpattern {^[a-zA-Z][a-zA-Z_0-9 *]+sqlite3_[_a-zA-Z0-9]+(\[|;| =)} -set declpattern {^ *([a-zA-Z][a-zA-Z_0-9 ]+ \**)(sqlite3_[_a-zA-Z0-9]+)(\(.*)$} +set declpattern1 {^ *([a-zA-Z][a-zA-Z_0-9 ]+ \**)(sqlite3_[_a-zA-Z0-9]+)(\(.*)$} + +set declpattern2 \ + {^ *([a-zA-Z][a-zA-Z_0-9 ]+ \**)(sqlite3session_[_a-zA-Z0-9]+)(\(.*)$} + +set declpattern3 \ + {^ *([a-zA-Z][a-zA-Z_0-9 ]+ \**)(sqlite3changeset_[_a-zA-Z0-9]+)(\(.*)$} # Force the output to use unix line endings, even on Windows. fconfigure stdout -translation lf @@ -121,7 +127,9 @@ foreach file $filelist { if {[regexp $varpattern $line] && ![regexp {^ *typedef} $line]} { set line "SQLITE_API $line" } else { - if {[regexp $declpattern $line all rettype funcname rest]} { + if {[regexp $declpattern1 $line all rettype funcname rest] || \ + [regexp $declpattern2 $line all rettype funcname rest] || \ + [regexp $declpattern3 $line all rettype funcname rest]} { set line SQLITE_API append line " " [string trim $rettype] if {[string index $rettype end] ne "*"} { diff --git a/tool/showdb.c b/tool/showdb.c index d51a2fd83d..ba7a362258 100644 --- a/tool/showdb.c +++ b/tool/showdb.c @@ -535,7 +535,7 @@ static void decodeCell( j = i; nCol = 0; k = nHdr; - while( x+jexplain-$NAME.txt fi +if test "$NAME" != "trunk"; then + fossil test-diff --tk cout-trunk.txt cout-$NAME.txt +fi