diff --git a/Makefile.in b/Makefile.in index c6b090ba43..fe4dc197b9 100644 --- a/Makefile.in +++ b/Makefile.in @@ -54,6 +54,7 @@ LIBTCL = @TCL_LIB_SPEC@ # Compiler options needed for programs that use the readline() library. # READLINE_FLAGS = -DHAVE_READLINE=@TARGET_HAVE_READLINE@ @TARGET_READLINE_INC@ +READLINE_FLAGS += -DHAVE_EDITLINE=@TARGET_HAVE_EDITLINE@ # The library that programs using readline() must link against. # diff --git a/Makefile.msc b/Makefile.msc index 63fbfad6ca..408073c651 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1084,11 +1084,9 @@ TESTEXT = \ $(TOP)\ext\misc\eval.c \ $(TOP)\ext\misc\fileio.c \ $(TOP)\ext\misc\fuzzer.c \ - fts5.c \ $(TOP)\ext\fts5\fts5_tcl.c \ $(TOP)\ext\fts5\fts5_test_mi.c \ $(TOP)\ext\misc\ieee754.c \ - $(TOP)\ext\misc\json1.c \ $(TOP)\ext\misc\nextchar.c \ $(TOP)\ext\misc\percentile.c \ $(TOP)\ext\misc\regexp.c \ diff --git a/VERSION b/VERSION index 6bd10744ae..30291cba22 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.9.1 +3.10.0 diff --git a/autoconf/configure.ac b/autoconf/configure.ac index 888b924e40..7812927557 100644 --- a/autoconf/configure.ac +++ b/autoconf/configure.ac @@ -18,7 +18,6 @@ AC_SYS_LARGEFILE # Check for required programs. AC_PROG_CC -AC_PROG_RANLIB AC_PROG_LIBTOOL AC_PROG_MKDIR_P @@ -30,12 +29,27 @@ AC_CONFIG_FILES([Makefile sqlite3.pc]) AC_SUBST(BUILD_CFLAGS) #----------------------------------------------------------------------- +# --enable-editline # --enable-readline # +AC_ARG_ENABLE(editline, [AS_HELP_STRING( + [--enable-editline], + [use BSD libedit])], + [], [enable_editline=yes]) AC_ARG_ENABLE(readline, [AS_HELP_STRING( [--enable-readline], - [use readline in shell tool (yes, no) [default=yes]])], - [], [enable_readline=yes]) + [use readline])], + [], [enable_readline=no]) +if test x"$enable_editline" != xno ; then + sLIBS=$LIBS + LIBS="" + AC_SEARCH_LIBS([readline],[edit],[enable_readline=no],[enable_editline=no]) + READLINE_LIBS=$LIBS + if test x"$LIBS" != "x"; then + AC_DEFINE([HAVE_EDITLINE],1,Define to use BSD editline) + fi + LIBS=$sLIBS +fi if test x"$enable_readline" != xno ; then sLIBS=$LIBS LIBS="" diff --git a/autoconf/tea/win/nmakehlp.c b/autoconf/tea/win/nmakehlp.c index 2868857efd..e00f1b4996 100644 --- a/autoconf/tea/win/nmakehlp.c +++ b/autoconf/tea/win/nmakehlp.c @@ -603,8 +603,8 @@ SubstituteFile( sp = fopen(substitutions, "rt"); if (sp != NULL) { while (fgets(szBuffer, cbBuffer, sp) != NULL) { - char *ks, *ke, *vs, *ve; - ks = szBuffer; + unsigned char *ks, *ke, *vs, *ve; + ks = (unsigned char*)szBuffer; while (ks && *ks && isspace(*ks)) ++ks; ke = ks; while (ke && *ke && !isspace(*ke)) ++ke; @@ -613,7 +613,7 @@ SubstituteFile( ve = vs; while (ve && *ve && !(*ve == '\r' || *ve == '\n')) ++ve; *ke = 0, *ve = 0; - list_insert(&substPtr, ks, vs); + list_insert(&substPtr, (char*)ks, (char*)vs); } fclose(sp); } diff --git a/configure b/configure index 65591fa0c8..174f41ae24 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for sqlite 3.9.1. +# Generated by GNU Autoconf 2.69 for sqlite 3.10.0. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. @@ -726,8 +726,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.9.1' -PACKAGE_STRING='sqlite 3.9.1' +PACKAGE_VERSION='3.10.0' +PACKAGE_STRING='sqlite 3.10.0' PACKAGE_BUGREPORT='' PACKAGE_URL='' @@ -774,6 +774,7 @@ USE_GCOV OPT_FEATURE_FLAGS USE_AMALGAMATION TARGET_DEBUG +TARGET_HAVE_EDITLINE TARGET_HAVE_READLINE TARGET_READLINE_INC TARGET_READLINE_LIBS @@ -895,6 +896,7 @@ enable_releasemode enable_tempstore enable_tcl with_tcl +enable_editline enable_readline with_readline_lib with_readline_inc @@ -1458,7 +1460,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.9.1 to adapt to many kinds of systems. +\`configure' configures sqlite 3.10.0 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1523,7 +1525,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.9.1:";; + short | recursive ) echo "Configuration of sqlite 3.10.0:";; esac cat <<\_ACEOF @@ -1542,7 +1544,8 @@ Optional Features: --enable-tempstore Use an in-ram database for temporary tables (never,no,yes,always) --disable-tcl do not build TCL extension - --disable-readline disable readline support [default=detect] + --enable-editline enable BSD editline support + --disable-readline disable readline support --enable-debug enable debugging & verbose explain --disable-amalgamation Disable the amalgamation and instead build all files separately @@ -1643,7 +1646,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.9.1 +sqlite configure 3.10.0 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -2062,7 +2065,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.9.1, which was +It was created by sqlite $as_me 3.10.0, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -3920,13 +3923,13 @@ if ${lt_cv_nm_interface+:} false; then : else lt_cv_nm_interface="BSD nm" echo "int some_variable = 0;" > conftest.$ac_ext - (eval echo "\"\$as_me:3923: $ac_compile\"" >&5) + (eval echo "\"\$as_me:3926: $ac_compile\"" >&5) (eval "$ac_compile" 2>conftest.err) cat conftest.err >&5 - (eval echo "\"\$as_me:3926: $NM \\\"conftest.$ac_objext\\\"\"" >&5) + (eval echo "\"\$as_me:3929: $NM \\\"conftest.$ac_objext\\\"\"" >&5) (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out) cat conftest.err >&5 - (eval echo "\"\$as_me:3929: output\"" >&5) + (eval echo "\"\$as_me:3932: output\"" >&5) cat conftest.out >&5 if $GREP 'External.*some_variable' conftest.out > /dev/null; then lt_cv_nm_interface="MS dumpbin" @@ -5132,7 +5135,7 @@ ia64-*-hpux*) ;; *-*-irix6*) # Find out which ABI we are using. - echo '#line 5135 "configure"' > conftest.$ac_ext + echo '#line 5138 "configure"' > conftest.$ac_ext if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5 (eval $ac_compile) 2>&5 ac_status=$? @@ -6657,11 +6660,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:6660: $lt_compile\"" >&5) + (eval echo "\"\$as_me:6663: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:6664: \$? = $ac_status" >&5 + echo "$as_me:6667: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -6996,11 +6999,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:6999: $lt_compile\"" >&5) + (eval echo "\"\$as_me:7002: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:7003: \$? = $ac_status" >&5 + echo "$as_me:7006: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -7101,11 +7104,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:7104: $lt_compile\"" >&5) + (eval echo "\"\$as_me:7107: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:7108: \$? = $ac_status" >&5 + echo "$as_me:7111: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -7156,11 +7159,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:7159: $lt_compile\"" >&5) + (eval echo "\"\$as_me:7162: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:7163: \$? = $ac_status" >&5 + echo "$as_me:7166: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -9536,7 +9539,7 @@ else lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 lt_status=$lt_dlunknown cat > conftest.$ac_ext <<_LT_EOF -#line 9539 "configure" +#line 9542 "configure" #include "confdefs.h" #if HAVE_DLFCN_H @@ -9632,7 +9635,7 @@ else lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 lt_status=$lt_dlunknown cat > conftest.$ac_ext <<_LT_EOF -#line 9635 "configure" +#line 9638 "configure" #include "confdefs.h" #if HAVE_DLFCN_H @@ -10832,6 +10835,14 @@ fi TARGET_READLINE_LIBS="" TARGET_READLINE_INC="" TARGET_HAVE_READLINE=0 +TARGET_HAVE_EDITLINE=0 +# Check whether --enable-editline was given. +if test "${enable_editline+set}" = set; then : + enableval=$enable_editline; with_editline=$enableval +else + with_editline=auto +fi + # Check whether --enable-readline was given. if test "${enable_readline+set}" = set; then : enableval=$enable_readline; with_readline=$enableval @@ -10840,6 +10851,71 @@ else fi +if test x"$with_editline" != xno; then + sLIBS=$LIBS + LIBS="" + TARGET_HAVE_EDITLINE=1 + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing readline" >&5 +$as_echo_n "checking for library containing readline... " >&6; } +if ${ac_cv_search_readline+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_func_search_save_LIBS=$LIBS +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char readline (); +int +main () +{ +return readline (); + ; + return 0; +} +_ACEOF +for ac_lib in '' edit; do + if test -z "$ac_lib"; then + ac_res="none required" + else + ac_res=-l$ac_lib + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + fi + if ac_fn_c_try_link "$LINENO"; then : + ac_cv_search_readline=$ac_res +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext + if ${ac_cv_search_readline+:} false; then : + break +fi +done +if ${ac_cv_search_readline+:} false; then : + +else + ac_cv_search_readline=no +fi +rm conftest.$ac_ext +LIBS=$ac_func_search_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_readline" >&5 +$as_echo "$ac_cv_search_readline" >&6; } +ac_res=$ac_cv_search_readline +if test "$ac_res" != no; then : + test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" + with_readline=no +else + TARGET_HAVE_EDITLINE=0 +fi + + TARGET_READLINE_LIBS=$LIBS + LIBS=$sLIBS +fi if test x"$with_readline" != xno; then found="yes" @@ -11028,6 +11104,7 @@ fi + ########## # Figure out what C libraries are required to compile programs # that use "fdatasync()" function. @@ -11946,7 +12023,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.9.1, which was +This file was extended by sqlite $as_me 3.10.0, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -12012,7 +12089,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -sqlite config.status 3.9.1 +sqlite config.status 3.10.0 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 11ab81d342..0b94d33930 100644 --- a/configure.ac +++ b/configure.ac @@ -460,11 +460,24 @@ AC_SUBST(HAVE_TCL) TARGET_READLINE_LIBS="" TARGET_READLINE_INC="" TARGET_HAVE_READLINE=0 +TARGET_HAVE_EDITLINE=0 +AC_ARG_ENABLE([editline], + [AC_HELP_STRING([--enable-editline],[enable BSD editline support])], + [with_editline=$enableval], + [with_editline=auto]) AC_ARG_ENABLE([readline], - [AC_HELP_STRING([--disable-readline],[disable readline support [default=detect]])], + [AC_HELP_STRING([--disable-readline],[disable readline support])], [with_readline=$enableval], [with_readline=auto]) +if test x"$with_editline" != xno; then + sLIBS=$LIBS + LIBS="" + TARGET_HAVE_EDITLINE=1 + AC_SEARCH_LIBS(readline,edit,[with_readline=no],[TARGET_HAVE_EDITLINE=0]) + TARGET_READLINE_LIBS=$LIBS + LIBS=$sLIBS +fi if test x"$with_readline" != xno; then found="yes" @@ -519,6 +532,7 @@ fi AC_SUBST(TARGET_READLINE_LIBS) AC_SUBST(TARGET_READLINE_INC) AC_SUBST(TARGET_HAVE_READLINE) +AC_SUBST(TARGET_HAVE_EDITLINE) ########## # Figure out what C libraries are required to compile programs diff --git a/ext/fts1/fts1.c b/ext/fts1/fts1.c index 482cf759ba..77fa9e23f5 100644 --- a/ext/fts1/fts1.c +++ b/ext/fts1/fts1.c @@ -205,13 +205,13 @@ static int getVarint32(const char *p, int *pi){ */ /* TODO(shess) Is __isascii() a portable version of (c&0x80)==0? */ static int safe_isspace(char c){ - return (c&0x80)==0 ? isspace(c) : 0; + return (c&0x80)==0 ? isspace((unsigned char)c) : 0; } static int safe_tolower(char c){ - return (c&0x80)==0 ? tolower(c) : c; + return (c&0x80)==0 ? tolower((unsigned char)c) : c; } static int safe_isalnum(char c){ - return (c&0x80)==0 ? isalnum(c) : 0; + return (c&0x80)==0 ? isalnum((unsigned char)c) : 0; } typedef enum DocListType { diff --git a/ext/fts1/simple_tokenizer.c b/ext/fts1/simple_tokenizer.c index d00a77089d..0ddc7055af 100644 --- a/ext/fts1/simple_tokenizer.c +++ b/ext/fts1/simple_tokenizer.c @@ -138,7 +138,7 @@ static int simpleNext( ** case-insensitivity. */ char ch = c->pCurrent[ii]; - c->zToken[ii] = (unsigned char)ch<0x80 ? tolower(ch) : ch; + c->zToken[ii] = (unsigned char)ch<0x80 ? tolower((unsigned char)ch):ch; } c->zToken[n] = '\0'; *ppToken = c->zToken; diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 3671da1f12..8626962067 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -224,7 +224,7 @@ struct Fts5Buffer { int nSpace; }; -int sqlite3Fts5BufferGrow(int*, Fts5Buffer*, int); +int sqlite3Fts5BufferSize(int*, Fts5Buffer*, int); void sqlite3Fts5BufferAppendVarint(int*, Fts5Buffer*, i64); void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, int, const u8*); void sqlite3Fts5BufferAppendString(int *, Fts5Buffer*, const char*); @@ -232,17 +232,19 @@ void sqlite3Fts5BufferFree(Fts5Buffer*); void sqlite3Fts5BufferZero(Fts5Buffer*); void sqlite3Fts5BufferSet(int*, Fts5Buffer*, int, const u8*); void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...); -void sqlite3Fts5BufferAppend32(int*, Fts5Buffer*, int); char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...); #define fts5BufferZero(x) sqlite3Fts5BufferZero(x) -#define fts5BufferGrow(a,b,c) sqlite3Fts5BufferGrow(a,b,c) #define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,c) #define fts5BufferFree(a) sqlite3Fts5BufferFree(a) #define fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d) #define fts5BufferSet(a,b,c,d) sqlite3Fts5BufferSet(a,b,c,d) -#define fts5BufferAppend32(a,b,c) sqlite3Fts5BufferAppend32(a,b,c) + +#define fts5BufferGrow(pRc,pBuf,nn) ( \ + (pBuf)->n + (nn) <= (pBuf)->nSpace ? 0 : \ + sqlite3Fts5BufferSize((pRc),(pBuf),(nn)+(pBuf)->n) \ +) /* Write and decode big-endian 32-bit integer values */ void sqlite3Fts5Put32(u8*, int); @@ -475,7 +477,7 @@ int sqlite3Fts5GetTokenizer( char **pzErr ); -Fts5Index *sqlite3Fts5IndexFromCsrid(Fts5Global*, i64, int*); +Fts5Index *sqlite3Fts5IndexFromCsrid(Fts5Global*, i64, Fts5Config **); /* ** End of interface to code in fts5.c. diff --git a/ext/fts5/fts5_buffer.c b/ext/fts5/fts5_buffer.c index 07d8516ce4..e9aab4622a 100644 --- a/ext/fts5/fts5_buffer.c +++ b/ext/fts5/fts5_buffer.c @@ -15,36 +15,30 @@ #include "fts5Int.h" -int sqlite3Fts5BufferGrow(int *pRc, Fts5Buffer *pBuf, int nByte){ - - if( (pBuf->n + nByte) > pBuf->nSpace ){ - u8 *pNew; - int nNew = pBuf->nSpace ? pBuf->nSpace*2 : 64; - - /* A no-op if an error has already occurred */ - if( *pRc ) return 1; - - while( nNew<(pBuf->n + nByte) ){ - nNew = nNew * 2; - } - pNew = sqlite3_realloc(pBuf->p, nNew); - if( pNew==0 ){ - *pRc = SQLITE_NOMEM; - return 1; - }else{ - pBuf->nSpace = nNew; - pBuf->p = pNew; - } +int sqlite3Fts5BufferSize(int *pRc, Fts5Buffer *pBuf, int nByte){ + int nNew = pBuf->nSpace ? pBuf->nSpace*2 : 64; + u8 *pNew; + while( nNewp, nNew); + if( pNew==0 ){ + *pRc = SQLITE_NOMEM; + return 1; + }else{ + pBuf->nSpace = nNew; + pBuf->p = pNew; } return 0; } + /* ** Encode value iVal as an SQLite varint and append it to the buffer object ** pBuf. If an OOM error occurs, set the error code in p. */ void sqlite3Fts5BufferAppendVarint(int *pRc, Fts5Buffer *pBuf, i64 iVal){ - if( sqlite3Fts5BufferGrow(pRc, pBuf, 9) ) return; + if( fts5BufferGrow(pRc, pBuf, 9) ) return; pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iVal); } @@ -59,12 +53,6 @@ int sqlite3Fts5Get32(const u8 *aBuf){ return (aBuf[0] << 24) + (aBuf[1] << 16) + (aBuf[2] << 8) + aBuf[3]; } -void sqlite3Fts5BufferAppend32(int *pRc, Fts5Buffer *pBuf, int iVal){ - if( sqlite3Fts5BufferGrow(pRc, pBuf, 4) ) return; - sqlite3Fts5Put32(&pBuf->p[pBuf->n], iVal); - pBuf->n += 4; -} - /* ** Append buffer nData/pData to buffer pBuf. If an OOM error occurs, set ** the error code in p. If an error has already occurred when this function @@ -77,7 +65,7 @@ void sqlite3Fts5BufferAppendBlob( const u8 *pData ){ assert( *pRc || nData>=0 ); - if( sqlite3Fts5BufferGrow(pRc, pBuf, nData) ) return; + if( fts5BufferGrow(pRc, pBuf, nData) ) return; memcpy(&pBuf->p[pBuf->n], pData, nData); pBuf->n += nData; } @@ -227,7 +215,7 @@ int sqlite3Fts5PoslistWriterAppend( ){ static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32; int rc = SQLITE_OK; - if( 0==sqlite3Fts5BufferGrow(&rc, pBuf, 5+5+5) ){ + if( 0==fts5BufferGrow(&rc, pBuf, 5+5+5) ){ if( (iPos & colmask) != (pWriter->iPrev & colmask) ){ pBuf->p[pBuf->n++] = 1; pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32)); diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 5f3316f623..b3f4372965 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -374,26 +374,6 @@ struct Fts5SegWriter { int iBtPage; /* Page number corresponding to btterm */ }; -/* -** Object for iterating through the merged results of one or more segments, -** visiting each term/rowid pair in the merged data. -** -** nSeg is always a power of two greater than or equal to the number of -** segments that this object is merging data from. Both the aSeg[] and -** aFirst[] arrays are sized at nSeg entries. The aSeg[] array is padded -** with zeroed objects - these are handled as if they were iterators opened -** on empty segments. -** -** The results of comparing segments aSeg[N] and aSeg[N+1], where N is an -** even number, is stored in aFirst[(nSeg+N)/2]. The "result" of the -** comparison in this context is the index of the iterator that currently -** points to the smaller term/rowid combination. Iterators at EOF are -** considered to be greater than all other iterators. -** -** aFirst[1] contains the index in aSeg[] of the iterator that points to -** the smallest key overall. aFirst[0] is unused. -*/ - typedef struct Fts5CResult Fts5CResult; struct Fts5CResult { u16 iFirst; /* aSeg[] index of firstest iterator */ @@ -500,6 +480,24 @@ struct Fts5SegIter { #define fts5LeafFirstRowidOff(x) (fts5GetU16((x)->p)) /* +** Object for iterating through the merged results of one or more segments, +** visiting each term/rowid pair in the merged data. +** +** nSeg is always a power of two greater than or equal to the number of +** segments that this object is merging data from. Both the aSeg[] and +** aFirst[] arrays are sized at nSeg entries. The aSeg[] array is padded +** with zeroed objects - these are handled as if they were iterators opened +** on empty segments. +** +** The results of comparing segments aSeg[N] and aSeg[N+1], where N is an +** even number, is stored in aFirst[(nSeg+N)/2]. The "result" of the +** comparison in this context is the index of the iterator that currently +** points to the smaller term/rowid combination. Iterators at EOF are +** considered to be greater than all other iterators. +** +** aFirst[1] contains the index in aSeg[] of the iterator that points to +** the smallest key overall. aFirst[0] is unused. +** ** poslist: ** Used by sqlite3Fts5IterPoslist() when the poslist needs to be buffered. ** There is no way to tell if this is populated or not. @@ -1008,6 +1006,18 @@ static int fts5StructureCountSegments(Fts5Structure *pStruct){ } #endif +#define fts5BufferSafeAppendBlob(pBuf, pBlob, nBlob) { \ + assert( (pBuf)->nSpace>=((pBuf)->n+nBlob) ); \ + memcpy(&(pBuf)->p[(pBuf)->n], pBlob, nBlob); \ + (pBuf)->n += nBlob; \ +} + +#define fts5BufferSafeAppendVarint(pBuf, iVal) { \ + (pBuf)->n += sqlite3Fts5PutVarint(&(pBuf)->p[(pBuf)->n], (iVal)); \ + assert( (pBuf)->nSpace>=(pBuf)->n ); \ +} + + /* ** Serialize and store the "structure" record. ** @@ -1026,11 +1036,14 @@ static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){ /* Append the current configuration cookie */ iCookie = p->pConfig->iCookie; if( iCookie<0 ) iCookie = 0; - fts5BufferAppend32(&p->rc, &buf, iCookie); - fts5BufferAppendVarint(&p->rc, &buf, pStruct->nLevel); - fts5BufferAppendVarint(&p->rc, &buf, pStruct->nSegment); - fts5BufferAppendVarint(&p->rc, &buf, (i64)pStruct->nWriteCounter); + if( 0==sqlite3Fts5BufferSize(&p->rc, &buf, 4+9+9+9) ){ + sqlite3Fts5Put32(buf.p, iCookie); + buf.n = 4; + fts5BufferSafeAppendVarint(&buf, pStruct->nLevel); + fts5BufferSafeAppendVarint(&buf, pStruct->nSegment); + fts5BufferSafeAppendVarint(&buf, (i64)pStruct->nWriteCounter); + } for(iLvl=0; iLvlnLevel; iLvl++){ int iSeg; /* Used to iterate through segments */ @@ -1486,6 +1499,7 @@ static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){ pIter->bDel = (nSz & 0x0001); pIter->nPos = nSz>>1; pIter->iLeafOffset = iOff; + assert_nc( pIter->nPos>=0 ); } } @@ -1659,12 +1673,13 @@ static void fts5SegIterReverseNewPage(Fts5Index *p, Fts5SegIter *pIter){ if( pNew ){ /* iTermLeafOffset may be equal to szLeaf if the term is the last ** thing on the page - i.e. the first rowid is on the following page. - ** In this case leaf pIter->pLeaf==0, this iterator is at EOF. */ - if( pIter->iLeafPgno==pIter->iTermLeafPgno - && pIter->iTermLeafOffsetszLeaf - ){ - pIter->pLeaf = pNew; - pIter->iLeafOffset = pIter->iTermLeafOffset; + ** In this case leave pIter->pLeaf==0, this iterator is at EOF. */ + if( pIter->iLeafPgno==pIter->iTermLeafPgno ){ + assert( pIter->pLeaf==0 ); + if( pIter->iTermLeafOffsetszLeaf ){ + pIter->pLeaf = pNew; + pIter->iLeafOffset = pIter->iTermLeafOffset; + } }else{ int iRowidOff; iRowidOff = fts5LeafFirstRowidOff(pNew); @@ -1830,7 +1845,15 @@ static void fts5SegIterNext( if( pbNewTerm ) *pbNewTerm = 1; } }else{ - fts5SegIterLoadNPos(p, pIter); + /* The following could be done by calling fts5SegIterLoadNPos(). But + ** this block is particularly performance critical, so equivalent + ** code is inlined. */ + int nSz; + assert( p->rc==SQLITE_OK ); + fts5FastGetVarint32(pIter->pLeaf->p, pIter->iLeafOffset, nSz); + pIter->bDel = (nSz & 0x0001); + pIter->nPos = nSz>>1; + assert_nc( pIter->nPos>=0 ); } } } @@ -2039,11 +2062,14 @@ static void fts5LeafSeek( if( pIter->pLeaf==0 ) return; a = pIter->pLeaf->p; if( fts5LeafIsTermless(pIter->pLeaf)==0 ){ - fts5GetVarint32(&pIter->pLeaf->p[pIter->pLeaf->szLeaf], iOff); + iPgidx = pIter->pLeaf->szLeaf; + iPgidx += fts5GetVarint32(&pIter->pLeaf->p[iPgidx], iOff); if( iOff<4 || iOff>=pIter->pLeaf->szLeaf ){ p->rc = FTS5_CORRUPT; }else{ nKeep = 0; + iTermOff = iOff; + n = pIter->pLeaf->nn; iOff += fts5GetVarint32(&a[iOff], nNew); break; } @@ -2593,6 +2619,35 @@ static void fts5MultiIterNext( } } +static void fts5MultiIterNext2( + Fts5Index *p, + Fts5IndexIter *pIter, + int *pbNewTerm /* OUT: True if *might* be new term */ +){ + assert( pIter->bSkipEmpty ); + if( p->rc==SQLITE_OK ){ + do { + int iFirst = pIter->aFirst[1].iFirst; + Fts5SegIter *pSeg = &pIter->aSeg[iFirst]; + int bNewTerm = 0; + + fts5SegIterNext(p, pSeg, &bNewTerm); + if( pSeg->pLeaf==0 || bNewTerm + || fts5MultiIterAdvanceRowid(p, pIter, iFirst) + ){ + fts5MultiIterAdvanced(p, pIter, iFirst, 1); + fts5MultiIterSetEof(pIter); + *pbNewTerm = 1; + }else{ + *pbNewTerm = 0; + } + fts5AssertMultiIterSetup(p, pIter); + + }while( fts5MultiIterIsEmpty(p, pIter) ); + } +} + + static Fts5IndexIter *fts5MultiIterAlloc( Fts5Index *p, /* FTS5 backend to iterate within */ int nSeg @@ -3346,9 +3401,12 @@ static void fts5WriteInit( pWriter->bFirstTermInPage = 1; pWriter->iBtPage = 1; + assert( pWriter->writer.buf.n==0 ); + assert( pWriter->writer.pgidx.n==0 ); + /* Grow the two buffers to pgsz + padding bytes in size. */ - fts5BufferGrow(&p->rc, &pWriter->writer.pgidx, nBuffer); - fts5BufferGrow(&p->rc, &pWriter->writer.buf, nBuffer); + sqlite3Fts5BufferSize(&p->rc, &pWriter->writer.pgidx, nBuffer); + sqlite3Fts5BufferSize(&p->rc, &pWriter->writer.buf, nBuffer); if( p->pIdxWriter==0 ){ Fts5Config *pConfig = p->pConfig; @@ -3701,17 +3759,6 @@ static int fts5PoslistPrefix(const u8 *aBuf, int nMax){ return ret; } -#define fts5BufferSafeAppendBlob(pBuf, pBlob, nBlob) { \ - assert( (pBuf)->nSpace>=((pBuf)->n+nBlob) ); \ - memcpy(&(pBuf)->p[(pBuf)->n], pBlob, nBlob); \ - (pBuf)->n += nBlob; \ -} - -#define fts5BufferSafeAppendVarint(pBuf, iVal) { \ - (pBuf)->n += sqlite3Fts5PutVarint(&(pBuf)->p[(pBuf)->n], (iVal)); \ - assert( (pBuf)->nSpace>=(pBuf)->n ); \ -} - /* ** Flush the contents of in-memory hash table iHash to a new level-0 ** segment on disk. Also update the corresponding structure record. @@ -4060,7 +4107,7 @@ static int fts5IndexExtractCol( const u8 *pEnd = &p[n]; /* One byte past end of position list */ u8 prev = 0; - while( iCol!=iCurrent ){ + while( iCol>iCurrent ){ /* Advance pointer p until it points to pEnd or an 0x01 byte that is ** not part of a varint */ while( (prev & 0x80) || *p!=0x01 ){ @@ -4070,6 +4117,7 @@ static int fts5IndexExtractCol( *pa = p++; p += fts5GetVarint32(p, iCurrent); } + if( iCol!=iCurrent ) return 0; /* Advance pointer p until it points to pEnd or an 0x01 byte that is ** not part of a varint */ @@ -4107,18 +4155,6 @@ static int fts5AppendPoslist( assert( fts5MultiIterEof(p, pMulti)==0 ); assert( pSeg->nPos>0 ); if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos+9+9) ){ - int iSv1; - int iSv2; - int iData; - - /* Append iDelta */ - iSv1 = pBuf->n; - fts5BufferSafeAppendVarint(pBuf, iDelta); - - /* WRITEPOSLISTSIZE */ - iSv2 = pBuf->n; - fts5BufferSafeAppendVarint(pBuf, pSeg->nPos*2); - iData = pBuf->n; if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf && (pColset==0 || pColset->nCol==1) @@ -4127,27 +4163,45 @@ static int fts5AppendPoslist( int nPos; if( pColset ){ nPos = fts5IndexExtractCol(&pPos, pSeg->nPos, pColset->aiCol[0]); + if( nPos==0 ) return 1; }else{ nPos = pSeg->nPos; } + assert( nPos>0 ); + fts5BufferSafeAppendVarint(pBuf, iDelta); + fts5BufferSafeAppendVarint(pBuf, nPos*2); fts5BufferSafeAppendBlob(pBuf, pPos, nPos); }else{ - fts5SegiterPoslist(p, pSeg, pColset, pBuf); - } + int iSv1; + int iSv2; + int iData; - if( pColset ){ - int nActual = pBuf->n - iData; - if( nActual!=pSeg->nPos ){ - if( nActual==0 ){ - pBuf->n = iSv1; - return 1; - }else{ - int nReq = sqlite3Fts5GetVarintLen((u32)(nActual*2)); - while( iSv2<(iData-nReq) ){ pBuf->p[iSv2++] = 0x80; } - sqlite3Fts5PutVarint(&pBuf->p[iSv2], nActual*2); + /* Append iDelta */ + iSv1 = pBuf->n; + fts5BufferSafeAppendVarint(pBuf, iDelta); + + /* WRITEPOSLISTSIZE */ + iSv2 = pBuf->n; + fts5BufferSafeAppendVarint(pBuf, pSeg->nPos*2); + iData = pBuf->n; + + fts5SegiterPoslist(p, pSeg, pColset, pBuf); + + if( pColset ){ + int nActual = pBuf->n - iData; + if( nActual!=pSeg->nPos ){ + if( nActual==0 ){ + pBuf->n = iSv1; + return 1; + }else{ + int nReq = sqlite3Fts5GetVarintLen((u32)(nActual*2)); + while( iSv2<(iData-nReq) ){ pBuf->p[iSv2++] = 0x80; } + sqlite3Fts5PutVarint(&pBuf->p[iSv2], nActual*2); + } } } } + } } @@ -4236,7 +4290,7 @@ static void fts5MergePrefixLists( memset(&out, 0, sizeof(out)); memset(&tmp, 0, sizeof(tmp)); - sqlite3Fts5BufferGrow(&p->rc, &out, p1->n + p2->n); + sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n); fts5DoclistIterInit(p1, &i1); fts5DoclistIterInit(p2, &i2); while( p->rc==SQLITE_OK && (i1.aPoslist!=0 || i2.aPoslist!=0) ){ @@ -4327,17 +4381,20 @@ static void fts5SetupPrefixIter( Fts5IndexIter *p1 = 0; /* Iterator used to gather data from index */ Fts5Data *pData; Fts5Buffer doclist; + int bNewTerm = 1; memset(&doclist, 0, sizeof(doclist)); for(fts5MultiIterNew(p, pStruct, 1, flags, pToken, nToken, -1, 0, &p1); fts5MultiIterEof(p, p1)==0; - fts5MultiIterNext(p, p1, 0, 0) + fts5MultiIterNext2(p, p1, &bNewTerm) ){ i64 iRowid = fts5MultiIterRowid(p1); int nTerm; const u8 *pTerm = fts5MultiIterTerm(p1, &nTerm); assert_nc( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 ); - if( nTerm0 && iRowid<=iLastRowid ){ for(i=0; p->rc==SQLITE_OK && doclist.n; i++){ @@ -4597,11 +4654,9 @@ int sqlite3Fts5IndexQuery( Fts5Buffer buf = {0, 0, 0}; /* If the QUERY_SCAN flag is set, all other flags must be clear. */ - assert( (flags & FTS5INDEX_QUERY_SCAN)==0 - || (flags & FTS5INDEX_QUERY_SCAN)==FTS5INDEX_QUERY_SCAN - ); + assert( (flags & FTS5INDEX_QUERY_SCAN)==0 || flags==FTS5INDEX_QUERY_SCAN ); - if( sqlite3Fts5BufferGrow(&p->rc, &buf, nToken+1)==0 ){ + if( sqlite3Fts5BufferSize(&p->rc, &buf, nToken+1)==0 ){ memcpy(&buf.p[1], pToken, nToken); #ifdef SQLITE_DEBUG @@ -5286,9 +5341,11 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){ Fts5IndexIter *pIter; /* Used to iterate through entire index */ Fts5Structure *pStruct; /* Index structure */ +#ifdef SQLITE_DEBUG /* Used by extra internal tests only run if NDEBUG is not defined */ u64 cksum3 = 0; /* Checksum based on contents of indexes */ Fts5Buffer term = {0,0,0}; /* Buffer used to hold most recent term */ +#endif /* Load the FTS index structure */ pStruct = fts5StructureRead(p); @@ -5344,7 +5401,9 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){ if( p->rc==SQLITE_OK && cksum!=cksum2 ) p->rc = FTS5_CORRUPT; fts5StructureRelease(pStruct); +#ifdef SQLITE_DEBUG fts5BufferFree(&term); +#endif fts5BufferFree(&poslist); return fts5IndexReturn(p); } @@ -5529,8 +5588,9 @@ static int fts5DecodeDoclist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){ } while( iOffdb); + int eConflict = SQLITE_ABORT; + if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ + eConflict = sqlite3_vtab_on_conflict(pConfig->db); + } assert( eType0==SQLITE_INTEGER || eType0==SQLITE_NULL ); assert( nArg!=1 || eType0==SQLITE_INTEGER ); @@ -2021,20 +2024,20 @@ static void fts5ApiCallback( ** Given cursor id iId, return a pointer to the corresponding Fts5Index ** object. Or NULL If the cursor id does not exist. ** -** If successful, set *pnCol to the number of indexed columns in the -** table before returning. +** If successful, set *ppConfig to point to the associated config object +** before returning. */ Fts5Index *sqlite3Fts5IndexFromCsrid( - Fts5Global *pGlobal, - i64 iCsrId, - int *pnCol + Fts5Global *pGlobal, /* FTS5 global context for db handle */ + i64 iCsrId, /* Id of cursor to find */ + Fts5Config **ppConfig /* OUT: Configuration object */ ){ Fts5Cursor *pCsr; Fts5Table *pTab; pCsr = fts5CursorFromCsrid(pGlobal, iCsrId); pTab = (Fts5Table*)pCsr->base.pVtab; - *pnCol = pTab->pConfig->nCol; + *ppConfig = pTab->pConfig; return pTab->pIndex; } diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index ce38762cad..5fd5dc51a2 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -513,13 +513,15 @@ int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel){ } /* Delete the %_content record */ - if( rc==SQLITE_OK ){ - rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_CONTENT, &pDel, 0); - } - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pDel, 1, iDel); - sqlite3_step(pDel); - rc = sqlite3_reset(pDel); + if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ + if( rc==SQLITE_OK ){ + rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_CONTENT, &pDel, 0); + } + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDel, 1, iDel); + sqlite3_step(pDel); + rc = sqlite3_reset(pDel); + } } /* Write the averages record */ @@ -914,12 +916,12 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){ /* Check that the %_docsize and %_content tables contain the expected ** number of rows. */ if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){ - i64 nRow; + i64 nRow = 0; rc = fts5StorageCount(p, "content", &nRow); if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT; } if( rc==SQLITE_OK && pConfig->bColumnsize ){ - i64 nRow; + i64 nRow = 0; rc = fts5StorageCount(p, "docsize", &nRow); if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT; } diff --git a/ext/fts5/fts5_vocab.c b/ext/fts5/fts5_vocab.c index adbe526b7b..860cfedb9b 100644 --- a/ext/fts5/fts5_vocab.c +++ b/ext/fts5/fts5_vocab.c @@ -55,16 +55,18 @@ struct Fts5VocabCursor { int bEof; /* True if this cursor is at EOF */ Fts5IndexIter *pIter; /* Term/rowid iterator object */ + int nLeTerm; /* Size of zLeTerm in bytes */ + char *zLeTerm; /* (term <= $zLeTerm) paramater, or NULL */ + /* These are used by 'col' tables only */ - int nCol; + Fts5Config *pConfig; /* Fts5 table configuration */ int iCol; i64 *aCnt; i64 *aDoc; - /* Output values */ + /* Output values used by 'row' and 'col' tables */ i64 rowid; /* This table's current rowid value */ Fts5Buffer term; /* Current value of 'term' column */ - i64 aVal[3]; /* Up to three columns left of 'term' */ }; #define FTS5_VOCAB_COL 0 @@ -73,6 +75,14 @@ struct Fts5VocabCursor { #define FTS5_VOCAB_COL_SCHEMA "term, col, doc, cnt" #define FTS5_VOCAB_ROW_SCHEMA "term, doc, cnt" +/* +** Bits for the mask used as the idxNum value by xBestIndex/xFilter. +*/ +#define FTS5_VOCAB_TERM_EQ 0x01 +#define FTS5_VOCAB_TERM_GE 0x02 +#define FTS5_VOCAB_TERM_LE 0x04 + + /* ** Translate a string containing an fts5vocab table type to an ** FTS5_VOCAB_XXX constant. If successful, set *peType to the output @@ -170,7 +180,7 @@ static int fts5VocabInitVtab( const char *zType = bDb ? argv[5] : argv[4]; int nDb = (int)strlen(zDb)+1; int nTab = (int)strlen(zTab)+1; - int eType; + int eType = 0; rc = fts5VocabTableType(zType, pzErr, &eType); if( rc==SQLITE_OK ){ @@ -230,6 +240,45 @@ static int fts5VocabBestIndexMethod( sqlite3_vtab *pVTab, sqlite3_index_info *pInfo ){ + int i; + int iTermEq = -1; + int iTermGe = -1; + int iTermLe = -1; + int idxNum = 0; + int nArg = 0; + + for(i=0; inConstraint; i++){ + struct sqlite3_index_constraint *p = &pInfo->aConstraint[i]; + if( p->usable==0 ) continue; + if( p->iColumn==0 ){ /* term column */ + if( p->op==SQLITE_INDEX_CONSTRAINT_EQ ) iTermEq = i; + if( p->op==SQLITE_INDEX_CONSTRAINT_LE ) iTermLe = i; + if( p->op==SQLITE_INDEX_CONSTRAINT_LT ) iTermLe = i; + if( p->op==SQLITE_INDEX_CONSTRAINT_GE ) iTermGe = i; + if( p->op==SQLITE_INDEX_CONSTRAINT_GT ) iTermGe = i; + } + } + + if( iTermEq>=0 ){ + idxNum |= FTS5_VOCAB_TERM_EQ; + pInfo->aConstraintUsage[iTermEq].argvIndex = ++nArg; + pInfo->estimatedCost = 100; + }else{ + pInfo->estimatedCost = 1000000; + if( iTermGe>=0 ){ + idxNum |= FTS5_VOCAB_TERM_GE; + pInfo->aConstraintUsage[iTermGe].argvIndex = ++nArg; + pInfo->estimatedCost = pInfo->estimatedCost / 2; + } + if( iTermLe>=0 ){ + idxNum |= FTS5_VOCAB_TERM_LE; + pInfo->aConstraintUsage[iTermLe].argvIndex = ++nArg; + pInfo->estimatedCost = pInfo->estimatedCost / 2; + } + } + + pInfo->idxNum = idxNum; + return SQLITE_OK; } @@ -242,12 +291,11 @@ static int fts5VocabOpenMethod( ){ Fts5VocabTable *pTab = (Fts5VocabTable*)pVTab; Fts5Index *pIndex = 0; - int nCol = 0; + Fts5Config *pConfig = 0; Fts5VocabCursor *pCsr = 0; int rc = SQLITE_OK; sqlite3_stmt *pStmt = 0; char *zSql = 0; - int nByte; zSql = sqlite3Fts5Mprintf(&rc, "SELECT t.%Q FROM %Q.%Q AS t WHERE t.%Q MATCH '*id'", @@ -262,7 +310,7 @@ static int fts5VocabOpenMethod( if( pStmt && sqlite3_step(pStmt)==SQLITE_ROW ){ i64 iId = sqlite3_column_int64(pStmt, 0); - pIndex = sqlite3Fts5IndexFromCsrid(pTab->pGlobal, iId, &nCol); + pIndex = sqlite3Fts5IndexFromCsrid(pTab->pGlobal, iId, &pConfig); } if( rc==SQLITE_OK && pIndex==0 ){ @@ -276,14 +324,17 @@ static int fts5VocabOpenMethod( } } - nByte = nCol * sizeof(i64) * 2 + sizeof(Fts5VocabCursor); - pCsr = (Fts5VocabCursor*)sqlite3Fts5MallocZero(&rc, nByte); + if( rc==SQLITE_OK ){ + int nByte = pConfig->nCol * sizeof(i64) * 2 + sizeof(Fts5VocabCursor); + pCsr = (Fts5VocabCursor*)sqlite3Fts5MallocZero(&rc, nByte); + } + if( pCsr ){ pCsr->pIndex = pIndex; pCsr->pStmt = pStmt; - pCsr->nCol = nCol; + pCsr->pConfig = pConfig; pCsr->aCnt = (i64*)&pCsr[1]; - pCsr->aDoc = &pCsr->aCnt[nCol]; + pCsr->aDoc = &pCsr->aCnt[pConfig->nCol]; }else{ sqlite3_finalize(pStmt); } @@ -296,6 +347,9 @@ static void fts5VocabResetCursor(Fts5VocabCursor *pCsr){ pCsr->rowid = 0; sqlite3Fts5IterClose(pCsr->pIter); pCsr->pIter = 0; + sqlite3_free(pCsr->zLeTerm); + pCsr->nLeTerm = -1; + pCsr->zLeTerm = 0; } /* @@ -319,16 +373,17 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){ Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; Fts5VocabTable *pTab = (Fts5VocabTable*)pCursor->pVtab; int rc = SQLITE_OK; + int nCol = pCsr->pConfig->nCol; pCsr->rowid++; if( pTab->eType==FTS5_VOCAB_COL ){ - for(pCsr->iCol++; pCsr->iColnCol; pCsr->iCol++){ + for(pCsr->iCol++; pCsr->iColiCol++){ if( pCsr->aCnt[pCsr->iCol] ) break; } } - if( pTab->eType==FTS5_VOCAB_ROW || pCsr->iCol>=pCsr->nCol ){ + if( pTab->eType==FTS5_VOCAB_ROW || pCsr->iCol>=nCol ){ if( sqlite3Fts5IterEof(pCsr->pIter) ){ pCsr->bEof = 1; }else{ @@ -336,10 +391,18 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){ int nTerm; zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm); + if( pCsr->nLeTerm>=0 ){ + int nCmp = MIN(nTerm, pCsr->nLeTerm); + int bCmp = memcmp(pCsr->zLeTerm, zTerm, nCmp); + if( bCmp<0 || (bCmp==0 && pCsr->nLeTermbEof = 1; + return SQLITE_OK; + } + } + sqlite3Fts5BufferSet(&rc, &pCsr->term, nTerm, (const u8*)zTerm); - memset(pCsr->aVal, 0, sizeof(pCsr->aVal)); - memset(pCsr->aCnt, 0, pCsr->nCol * sizeof(i64)); - memset(pCsr->aDoc, 0, pCsr->nCol * sizeof(i64)); + memset(pCsr->aCnt, 0, nCol * sizeof(i64)); + memset(pCsr->aDoc, 0, nCol * sizeof(i64)); pCsr->iCol = 0; assert( pTab->eType==FTS5_VOCAB_COL || pTab->eType==FTS5_VOCAB_ROW ); @@ -353,9 +416,9 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){ if( rc==SQLITE_OK ){ if( pTab->eType==FTS5_VOCAB_ROW ){ while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ - pCsr->aVal[1]++; + pCsr->aCnt[0]++; } - pCsr->aVal[0]++; + pCsr->aDoc[0]++; }else{ int iCol = -1; while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ @@ -369,9 +432,12 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){ } rc = sqlite3Fts5IterNextScan(pCsr->pIter); } + if( rc==SQLITE_OK ){ zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm); - if( nTerm!=pCsr->term.n || memcmp(zTerm, pCsr->term.p, nTerm) ) break; + if( nTerm!=pCsr->term.n || memcmp(zTerm, pCsr->term.p, nTerm) ){ + break; + } if( sqlite3Fts5IterEof(pCsr->pIter) ) break; } } @@ -380,9 +446,7 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){ if( pCsr->bEof==0 && pTab->eType==FTS5_VOCAB_COL ){ while( pCsr->aCnt[pCsr->iCol]==0 ) pCsr->iCol++; - pCsr->aVal[0] = pCsr->iCol; - pCsr->aVal[1] = pCsr->aDoc[pCsr->iCol]; - pCsr->aVal[2] = pCsr->aCnt[pCsr->iCol]; + assert( pCsr->iColpConfig->nCol ); } return rc; } @@ -398,11 +462,47 @@ static int fts5VocabFilterMethod( sqlite3_value **apVal /* Arguments for the indexing scheme */ ){ Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; - int rc; - const int flags = FTS5INDEX_QUERY_SCAN; + int rc = SQLITE_OK; + + int iVal = 0; + int f = FTS5INDEX_QUERY_SCAN; + const char *zTerm = 0; + int nTerm = 0; + + sqlite3_value *pEq = 0; + sqlite3_value *pGe = 0; + sqlite3_value *pLe = 0; fts5VocabResetCursor(pCsr); - rc = sqlite3Fts5IndexQuery(pCsr->pIndex, 0, 0, flags, 0, &pCsr->pIter); + if( idxNum & FTS5_VOCAB_TERM_EQ ) pEq = apVal[iVal++]; + if( idxNum & FTS5_VOCAB_TERM_GE ) pGe = apVal[iVal++]; + if( idxNum & FTS5_VOCAB_TERM_LE ) pLe = apVal[iVal++]; + + if( pEq ){ + zTerm = (const char *)sqlite3_value_text(pEq); + nTerm = sqlite3_value_bytes(pEq); + f = 0; + }else{ + if( pGe ){ + zTerm = (const char *)sqlite3_value_text(pGe); + nTerm = sqlite3_value_bytes(pGe); + } + if( pLe ){ + const char *zCopy = (const char *)sqlite3_value_text(pLe); + pCsr->nLeTerm = sqlite3_value_bytes(pLe); + pCsr->zLeTerm = sqlite3_malloc(pCsr->nLeTerm+1); + if( pCsr->zLeTerm==0 ){ + rc = SQLITE_NOMEM; + }else{ + memcpy(pCsr->zLeTerm, zCopy, pCsr->nLeTerm+1); + } + } + } + + + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IndexQuery(pCsr->pIndex, zTerm, nTerm, f, 0, &pCsr->pIter); + } if( rc==SQLITE_OK ){ rc = fts5VocabNextMethod(pCursor); } @@ -425,17 +525,29 @@ static int fts5VocabColumnMethod( int iCol /* Index of column to read value from */ ){ Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; - switch( iCol ){ - case 0: /* term */ - sqlite3_result_text( - pCtx, (const char*)pCsr->term.p, pCsr->term.n, SQLITE_TRANSIENT - ); - break; - default: - assert( iCol<4 && iCol>0 ); - sqlite3_result_int64(pCtx, pCsr->aVal[iCol-1]); - break; + if( iCol==0 ){ + sqlite3_result_text( + pCtx, (const char*)pCsr->term.p, pCsr->term.n, SQLITE_TRANSIENT + ); + } + else if( ((Fts5VocabTable*)(pCursor->pVtab))->eType==FTS5_VOCAB_COL ){ + assert( iCol==1 || iCol==2 || iCol==3 ); + if( iCol==1 ){ + const char *z = pCsr->pConfig->azCol[pCsr->iCol]; + sqlite3_result_text(pCtx, z, -1, SQLITE_STATIC); + }else if( iCol==2 ){ + sqlite3_result_int64(pCtx, pCsr->aDoc[pCsr->iCol]); + }else{ + sqlite3_result_int64(pCtx, pCsr->aCnt[pCsr->iCol]); + } + }else{ + assert( iCol==1 || iCol==2 ); + if( iCol==1 ){ + sqlite3_result_int64(pCtx, pCsr->aDoc[0]); + }else{ + sqlite3_result_int64(pCtx, pCsr->aCnt[0]); + } } return SQLITE_OK; } diff --git a/ext/fts5/test/fts5aa.test b/ext/fts5/test/fts5aa.test index f8ea6a8f71..59b8ab86d1 100644 --- a/ext/fts5/test/fts5aa.test +++ b/ext/fts5/test/fts5aa.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5aa -# If SQLITE_ENABLE_FTS3 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5conflict.test b/ext/fts5/test/fts5conflict.test new file mode 100644 index 0000000000..5c1e593249 --- /dev/null +++ b/ext/fts5/test/fts5conflict.test @@ -0,0 +1,70 @@ +# 2015 October 27 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5conflict + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE TABLE t1(x INTEGER PRIMARY KEY, a, b); + CREATE VIRTUAL TABLE ft USING fts5(a, b, content=t1, content_rowid=x); +} + +do_execsql_test 1.1 { + REPLACE INTO ft(rowid, a, b) VALUES(1, 'a b c', 'a b c'); + REPLACE INTO t1 VALUES(1, 'a b c', 'a b c'); +} + +do_execsql_test 1.2 { + INSERT INTO ft(ft) VALUES('integrity-check'); +} + +do_execsql_test 2.0 { + CREATE TABLE tbl(a INTEGER PRIMARY KEY, b, c); + CREATE VIRTUAL TABLE fts_idx USING fts5(b, c, content=tbl, content_rowid=a); + CREATE TRIGGER tbl_ai AFTER INSERT ON tbl BEGIN + INSERT INTO fts_idx(rowid, b, c) VALUES (new.a, new.b, new.c); + END; + CREATE TRIGGER tbl_ad AFTER DELETE ON tbl BEGIN + INSERT INTO fts_idx(fts_idx, rowid, b, c) + VALUES('delete', old.a, old.b, old.c); + END; + CREATE TRIGGER tbl_au AFTER UPDATE ON tbl BEGIN + INSERT INTO fts_idx(fts_idx, rowid, b, c) + VALUES('delete', old.a, old.b, old.c); + INSERT INTO fts_idx(rowid, b, c) VALUES (new.a, new.b, new.c); + END; +} + +do_execsql_test 2.1 { + PRAGMA recursive_triggers = 1; + INSERT INTO tbl VALUES(1, 'x y z', '1 2 3'); + INSERT INTO tbl VALUES(10, 'x y z', '1 2 3'); + INSERT INTO tbl VALUES(100, 'x 1 z', '1 y 3'); + + UPDATE tbl SET b = '1 2 x' WHERE rowid=10; + REPLACE INTO tbl VALUES(1, '4 5 6', '3 2 1'); + DELETE FROM tbl WHERE a=100; + + INSERT INTO fts_idx(fts_idx) VALUES('integrity-check'); +} + +finish_test + + diff --git a/ext/fts5/test/fts5fault5.test b/ext/fts5/test/fts5fault5.test index c14f394eb6..41de5208b2 100644 --- a/ext/fts5/test/fts5fault5.test +++ b/ext/fts5/test/fts5fault5.test @@ -90,6 +90,14 @@ do_faultsim_test 3.1 -faults oom-t* -body { faultsim_test_result {0 {0 1 10 11 12 13 14 15 16 17 18 19 2 3 4 5 6 7 8 9}} } +do_faultsim_test 3.2 -faults oom-t* -body { + db eval { + SELECT term FROM tv WHERE term BETWEEN '1' AND '2'; + } +} -test { + faultsim_test_result {0 {1 10 11 12 13 14 15 16 17 18 19 2}} +} + finish_test diff --git a/ext/fts5/test/fts5integrity.test b/ext/fts5/test/fts5integrity.test index 478e790d6b..84490e0cb4 100644 --- a/ext/fts5/test/fts5integrity.test +++ b/ext/fts5/test/fts5integrity.test @@ -9,7 +9,7 @@ # #*********************************************************************** # -# This file containst tests focused on the integrity-check procedure. +# This file contains tests focused on the integrity-check procedure. # source [file join [file dirname [info script]] fts5_common.tcl] @@ -102,6 +102,53 @@ do_catchsql_test 4.5 { #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM zz_data} {puts $r} #exit +execsql { ROLLBACK } + + +#------------------------------------------------------------------------- +# Test that integrity-check works on a reasonably large db with many +# different terms. + +# Document generator command. +proc rnddoc {n} { + set doc [list] + for {set i 0} {$i<$n} {incr i} { + lappend doc [format %.5d [expr int(rand()*10000)]] + } + return $doc +} +db func rnddoc rnddoc + +expr srand(0) +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE gg USING fts5(a, prefix="1,2,3"); + INSERT INTO gg(gg, rank) VALUES('pgsz', 256); + INSERT INTO gg VALUES(rnddoc(20)); + INSERT INTO gg SELECT rnddoc(20) FROM gg; + INSERT INTO gg SELECT rnddoc(20) FROM gg; + INSERT INTO gg SELECT rnddoc(20) FROM gg; + INSERT INTO gg SELECT rnddoc(20) FROM gg; + INSERT INTO gg SELECT rnddoc(20) FROM gg; + INSERT INTO gg SELECT rnddoc(20) FROM gg; + INSERT INTO gg SELECT rnddoc(20) FROM gg; + INSERT INTO gg SELECT rnddoc(20) FROM gg; + INSERT INTO gg SELECT rnddoc(20) FROM gg; + INSERT INTO gg SELECT rnddoc(20) FROM gg; + INSERT INTO gg SELECT rnddoc(20) FROM gg; +} + +do_execsql_test 5.1 { + INSERT INTO gg(gg) VALUES('integrity-check'); +} + +do_execsql_test 5.2 { + INSERT INTO gg(gg) VALUES('optimize'); +} + +breakpoint +do_execsql_test 5.3 { + INSERT INTO gg(gg) VALUES('integrity-check'); +} finish_test diff --git a/ext/fts5/test/fts5query.test b/ext/fts5/test/fts5query.test new file mode 100644 index 0000000000..570abf46ba --- /dev/null +++ b/ext/fts5/test/fts5query.test @@ -0,0 +1,83 @@ +# 2015 October 27 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5query + +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +for {set tn 1 ; set pgsz 64} {$tn<32} {incr tn; incr pgsz 16} { + reset_db + do_test 1.$tn.1 { + execsql { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(t1, rank) VALUES('pgsz', $pgsz); + BEGIN; + } + foreach x [list aaa bbb ccc ddd eee fff ggg hhh iii jjj] { + set doc [string repeat "$x " 30] + execsql { INSERT INTO t1 VALUES($doc) } + } + execsql COMMIT + } {} + + do_execsql_test 1.$tn.2 { + INSERT INTO t1(t1) VALUES('integrity-check'); + } + + set ret 1 + foreach x [list a b c d e f g h i j] { + do_execsql_test 1.$tn.3.$ret { + SELECT rowid FROM t1 WHERE t1 MATCH $x || '*'; + } $ret + incr ret + } +} + +for {set tn 1 ; set pgsz 64} {$tn<32} {incr tn; incr pgsz 16} { + reset_db + do_test 2.$tn.1 { + execsql { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(t1, rank) VALUES('pgsz', $pgsz); + BEGIN; + } + foreach x [list bbb ddd fff hhh jjj lll nnn ppp rrr ttt] { + set doc [string repeat "$x " 30] + execsql { INSERT INTO t1 VALUES($doc) } + } + execsql COMMIT + } {} + + do_execsql_test 1.$tn.2 { + INSERT INTO t1(t1) VALUES('integrity-check'); + } + + set ret 1 + foreach x [list a c e g i k m o q s u] { + do_execsql_test 2.$tn.3.$ret { + SELECT rowid FROM t1 WHERE t1 MATCH $x || '*'; + } {} + incr ret + } +} + + +finish_test + + diff --git a/ext/fts5/test/fts5vocab.test b/ext/fts5/test/fts5vocab.test index c95f50b835..dc5099c6e3 100644 --- a/ext/fts5/test/fts5vocab.test +++ b/ext/fts5/test/fts5vocab.test @@ -56,7 +56,7 @@ do_execsql_test 1.4.1 { do_execsql_test 1.4.2 { SELECT * FROM v2; -} {x 0 2 4 y 0 1 1 z 0 1 1} +} {x one 2 4 y one 1 1 z one 1 1} do_execsql_test 1.5.1 { BEGIN; @@ -67,7 +67,7 @@ do_execsql_test 1.5.1 { do_execsql_test 1.5.2 { SELECT * FROM v2 WHERE term<'d'; COMMIT; -} {a 0 1 1 b 0 1 1 c 0 1 1} +} {a one 1 1 b one 1 1 c one 1 1} do_execsql_test 1.6 { DELETE FROM t1 WHERE one = 'a b c'; @@ -91,14 +91,14 @@ do_execsql_test 2.0 { } set res_col { - a 0 6 11 a 1 7 9 - b 0 6 7 b 1 7 7 - c 0 6 12 c 1 5 8 - d 0 4 6 d 1 9 13 - e 0 6 7 e 1 6 6 - f 0 9 10 f 1 7 10 - g 0 5 7 g 1 5 7 - x 0 1 1 y 1 1 1 + a a 6 11 a b 7 9 + b a 6 7 b b 7 7 + c a 6 12 c b 5 8 + d a 4 6 d b 9 13 + e a 6 7 e b 6 6 + f a 9 10 f b 7 10 + g a 5 7 g b 5 7 + x a 1 1 y b 1 1 } set res_row { a 10 20 b 9 14 c 9 20 d 9 19 @@ -213,5 +213,137 @@ do_catchsql_test 6.2 { SELECT * FROM vocab3; } {1 {no such fts5 table: main.lll}} +#------------------------------------------------------------------------- +# Test single term queries on fts5vocab tables (i.e. those with term=? +# constraints in the WHERE clause). +# +do_execsql_test 7.0 { + CREATE VIRTUAL TABLE tx USING fts5(one, two); + INSERT INTO tx VALUES('g a ggg g a b eee', 'cc d aa ff g ee'); + INSERT INTO tx VALUES('dd fff i a i jjj', 'f fff hh jj e f'); + INSERT INTO tx VALUES('ggg a f f fff dd aa', 'd ggg f f j gg ddd'); + INSERT INTO tx VALUES('e bb h jjj ii gg', 'e aa e f c fff'); + INSERT INTO tx VALUES('j ff aa a h', 'h a j bbb bb'); + INSERT INTO tx VALUES('cc i ff c d f', 'dd ii fff f c cc d'); + INSERT INTO tx VALUES('jjj g i bb cc eee', 'hhh iii aaa b bbb aaa'); + INSERT INTO tx VALUES('hhh hhh hhh bb fff f', 'fff gg aa ii h a'); + INSERT INTO tx VALUES('b c cc aaa iii ggg f', 'iii ff ee a ff c cc'); + INSERT INTO tx VALUES('hhh b hhh aaa j i i', 'dd ee ee aa bbb iii'); + INSERT INTO tx VALUES('hh dd h b g ff i', 'ccc bb cc ccc f a d'); + INSERT INTO tx VALUES('g d b ggg jj', 'fff jj ff jj g gg ee'); + INSERT INTO tx VALUES('g ee ggg ggg cc bb eee', 'aa j jjj bbb dd eee ff'); + INSERT INTO tx VALUES('c jjj hh ddd dd h', 'e aaa h jjj gg'); + + CREATE VIRTUAL TABLE txr USING fts5vocab(tx, row); + CREATE VIRTUAL TABLE txc USING fts5vocab(tx, col); +} + +proc cont {L elem} { + set n 0 + foreach e $L { if {$elem==$e} {incr n} } + set n +} +db func cont cont + +foreach {term} { + a aa aaa + b bb bbb + c cc ccc + d dd ddd + e ee eee + f ff fff + g gg ggg + h hh hhh + i ii iii + j jj jjj +} { + set resr [db eval { + SELECT $term, + sum(cont(one || ' ' || two, $term) > 0), + sum(cont(one || ' ' || two, $term)) + FROM tx + }] + if {[lindex $resr 1]==0} {set resr [list]} + + set r1 [db eval { + SELECT $term, 'one', sum(cont(one, $term)>0), sum(cont(one, $term)) FROM tx + }] + if {[lindex $r1 2]==0} {set r1 [list]} + + set r2 [db eval { + SELECT $term, 'two', sum(cont(two, $term)>0), sum(cont(two, $term)) FROM tx + }] + if {[lindex $r2 2]==0} {set r2 [list]} + + set resc [concat $r1 $r2] + do_execsql_test 7.$term.1 {SELECT * FROM txc WHERE term=$term} $resc + do_execsql_test 7.$term.2 {SELECT * FROM txr WHERE term=$term} $resr +} + +do_execsql_test 7.1 { + CREATE TABLE txr_c AS SELECT * FROM txr; + CREATE TABLE txc_c AS SELECT * FROM txc; +} + +# Test range queries on the fts5vocab tables created above. +# +foreach {tn a b} { + 1 a jjj + 2 bb j + 3 ccc ddd + 4 dd xyz + 5 xzy dd + 6 h hh +} { + do_execsql_test 7.2.$tn.1 { + SELECT * FROM txr WHERE term>=$a + } [db eval {SELECT * FROM txr_c WHERE term>=$a}] + do_execsql_test 7.2.$tn.2 { + SELECT * FROM txr WHERE term<=$b + } [db eval {SELECT * FROM txr_c WHERE term <=$b}] + do_execsql_test 7.2.$tn.3 { + SELECT * FROM txr WHERE term>=$a AND term<=$b + } [db eval {SELECT * FROM txr_c WHERE term>=$a AND term <=$b}] + + do_execsql_test 7.2.$tn.4 { + SELECT * FROM txc WHERE term>=$a + } [db eval {SELECT * FROM txc_c WHERE term>=$a}] + do_execsql_test 7.2.$tn.5 { + SELECT * FROM txc WHERE term<=$b + } [db eval {SELECT * FROM txc_c WHERE term <=$b}] + do_execsql_test 7.2.$tn.6 { + SELECT * FROM txc WHERE term>=$a AND term<=$b + } [db eval {SELECT * FROM txc_c WHERE term>=$a AND term <=$b}] + + do_execsql_test 7.2.$tn.7 { + SELECT * FROM txr WHERE term>$a + } [db eval {SELECT * FROM txr_c WHERE term>$a}] + do_execsql_test 7.2.$tn.8 { + SELECT * FROM txr WHERE term<$b + } [db eval {SELECT * FROM txr_c WHERE term<$b}] + do_execsql_test 7.2.$tn.9 { + SELECT * FROM txr WHERE term>$a AND term<$b + } [db eval {SELECT * FROM txr_c WHERE term>$a AND term <$b}] + + do_execsql_test 7.2.$tn.10 { + SELECT * FROM txc WHERE term>$a + } [db eval {SELECT * FROM txc_c WHERE term>$a}] + do_execsql_test 7.2.$tn.11 { + SELECT * FROM txc WHERE term<$b + } [db eval {SELECT * FROM txc_c WHERE term<$b}] + do_execsql_test 7.2.$tn.12 { + SELECT * FROM txc WHERE term>$a AND term<$b + } [db eval {SELECT * FROM txc_c WHERE term>$a AND term <$b}] +} + +do_execsql_test 7.3.1 { + SELECT count(*) FROM txr, txr_c WHERE txr.term = txr_c.term; +} {30} + +do_execsql_test 7.3.2 { + SELECT count(*) FROM txc, txc_c + WHERE txc.term = txc_c.term AND txc.col=txc_c.col; +} {57} + finish_test diff --git a/ext/fts5/tool/fts5txt2db.tcl b/ext/fts5/tool/fts5txt2db.tcl index 23f607a801..c22c5dae72 100644 --- a/ext/fts5/tool/fts5txt2db.tcl +++ b/ext/fts5/tool/fts5txt2db.tcl @@ -25,7 +25,7 @@ is exhausted. exit -1 } -set O(aColsize) [list 10 10 10] +set O(aColSize) [list 10 10 10] set O(tblname) t1 set O(fts) fts5 @@ -61,22 +61,7 @@ if {$i > [llength $argv]-2} usage set O(db) [lindex $argv $i] set O(files) [lrange $argv [expr $i+1] end] -foreach {k v} [lrange $argv 0 end-2] { - switch -- $k { - -colsize { - set O(aColSize) $v - } - - -colsize { - set O(aColSize) $v - } - } - -} - sqlite3 db $O(db) -load_static_extension db fts5 - # Create the FTS table in the db. Return a list of the table columns. # @@ -84,7 +69,7 @@ proc create_table {} { global O set cols [list a b c d e f g h i j k l m n o p q r s t u v w x y z] - set nCol [llength $O(aColsize)] + set nCol [llength $O(aColSize)] set cols [lrange $cols 0 [expr $nCol-1]] set sql "CREATE VIRTUAL TABLE IF NOT EXISTS $O(tblname) USING $O(fts) (" @@ -117,14 +102,14 @@ set i 0 set cols [create_table] set sql "INSERT INTO $O(tblname) VALUES(\$[lindex $cols 0]" foreach c [lrange $cols 1 end] { - append sql ", \$$c" + append sql ", \$A($c)" } append sql ")" db eval BEGIN while {$i < $N} { - foreach c $cols s $O(aColsize) { - set $c [lrange $tokens $i [expr $i+$s-1]] + foreach c $cols s $O(aColSize) { + set A($c) [lrange $tokens $i [expr $i+$s-1]] incr i $s } db eval $sql diff --git a/ext/fts5/tool/mkfts5c.tcl b/ext/fts5/tool/mkfts5c.tcl index 535506de1b..797811d46e 100644 --- a/ext/fts5/tool/mkfts5c.tcl +++ b/ext/fts5/tool/mkfts5c.tcl @@ -78,7 +78,7 @@ proc fts5c_printfile {zIn} { global G set data [readfile $zIn] set zTail [file tail $zIn] - puts $G(fd) "#line 2 \"$zTail\"" + puts $G(fd) "#line 1 \"$zTail\"" set sub_map [list --FTS5-SOURCE-ID-- [fts5_source_id $::srcdir]] if {$zTail=="fts5parse.c"} { @@ -86,8 +86,10 @@ proc fts5c_printfile {zIn} { } foreach line [split $data "\n"] { - if {[regexp {^#include.*fts5} $line]} continue - if { ![regexp { sqlite3Fts5Init\(} $line] + if {[regexp {^#include.*fts5} $line]} { + set line "/* $line */" + } elseif { + ![regexp { sqlite3Fts5Init\(} $line] && [regexp {^(const )?[a-zA-Z][a-zA-Z0-9]* [*]?sqlite3Fts5} $line] } { set line "static $line" diff --git a/ext/fts5/tool/showfts5.tcl b/ext/fts5/tool/showfts5.tcl index d9af5f38eb..75ac0f1c8f 100644 --- a/ext/fts5/tool/showfts5.tcl +++ b/ext/fts5/tool/showfts5.tcl @@ -8,11 +8,13 @@ proc usage {} { puts stderr "usage: $::argv0 ?OPTIONS? database table" puts stderr "" puts stderr " -nterm (count number of terms in each segment)" + puts stderr " -segments (output segment contents)" puts stderr "" exit 1 } set O(nterm) 0 +set O(segments) 0 if {[llength $argv]<2} usage foreach a [lrange $argv 0 end-2] { @@ -21,6 +23,10 @@ foreach a [lrange $argv 0 end-2] { set O(nterm) 1 } + -segments { + set O(segments) 1 + } + default { usage } @@ -78,6 +84,13 @@ db eval "SELECT fts5_decode(rowid, block) AS d FROM ${tbl}_data WHERE id=10" { } } +if {$O(segments)} { + puts "" + db eval "SELECT fts5_decode(rowid, block) AS d FROM ${tbl}_data WHERE id>10" { + puts $d + } +} + diff --git a/ext/misc/amatch.c b/ext/misc/amatch.c index d08ef57aad..852491988a 100644 --- a/ext/misc/amatch.c +++ b/ext/misc/amatch.c @@ -816,10 +816,10 @@ static const char *amatchValueOfKey(const char *zKey, const char *zStr){ int i; if( nStr #include +#define ISSPACE(X) isspace((unsigned char)(X)) +#define ISDIGIT(X) isdigit((unsigned char)(X)) + /* The suffix to append to the child command lines, if any */ #if defined(_WIN32) # define GETPID (int)GetCurrentProcessId @@ -187,10 +190,10 @@ int strglob(const char *zGlob, const char *z){ } if( c2==0 || (seen ^ invert)==0 ) return 0; }else if( c=='#' ){ - if( (z[0]=='-' || z[0]=='+') && isdigit(z[1]) ) z++; - if( !isdigit(z[0]) ) return 0; + if( (z[0]=='-' || z[0]=='+') && ISDIGIT(z[1]) ) z++; + if( !ISDIGIT(z[0]) ) return 0; z++; - while( isdigit(z[0]) ){ z++; } + while( ISDIGIT(z[0]) ){ z++; } }else{ if( c!=(*(z++)) ) return 0; } @@ -289,7 +292,7 @@ static void logMessage(const char *zFormat, ...){ */ static int clipLength(const char *z){ int n = (int)strlen(z); - while( n>0 && isspace(z[n-1]) ){ n--; } + while( n>0 && ISSPACE(z[n-1]) ){ n--; } return n; } @@ -444,7 +447,7 @@ static void stringAppendTerm(String *p, const char *z){ stringAppend(p, "nil", 3); return; } - for(i=0; z[i] && !isspace(z[i]); i++){} + for(i=0; z[i] && !ISSPACE(z[i]); i++){} if( i>0 && z[i]==0 ){ stringAppend(p, z, i); return; @@ -699,7 +702,7 @@ static char *readFile(const char *zFilename){ */ static int tokenLength(const char *z, int *pnLine){ int n = 0; - if( isspace(z[0]) || (z[0]=='/' && z[1]=='*') ){ + if( ISSPACE(z[0]) || (z[0]=='/' && z[1]=='*') ){ int inC = 0; int c; if( z[0]=='/' ){ @@ -708,7 +711,7 @@ static int tokenLength(const char *z, int *pnLine){ } while( (c = z[n++])!=0 ){ if( c=='\n' ) (*pnLine)++; - if( isspace(c) ) continue; + if( ISSPACE(c) ) continue; if( inC && c=='*' && z[n]=='/' ){ n++; inC = 0; @@ -734,7 +737,7 @@ static int tokenLength(const char *z, int *pnLine){ } }else{ int c; - for(n=1; (c = z[n])!=0 && !isspace(c) && c!='"' && c!='\'' && c!=';'; n++){} + for(n=1; (c = z[n])!=0 && !ISSPACE(c) && c!='"' && c!='\'' && c!=';'; n++){} } return n; } @@ -748,7 +751,7 @@ static int extractToken(const char *zIn, int nIn, char *zOut, int nOut){ zOut[0] = 0; return 0; } - for(i=0; i=2 ) logMessage("%.*s", len, zScript+ii); n = extractToken(zScript+ii+2, len-2, zCmd, sizeof(zCmd)); for(nArg=0; n=len-2 ) break; n += extractToken(zScript+ii+2+n, len-2-n, azArg[nArg], sizeof(azArg[nArg])); @@ -976,7 +979,7 @@ static void runScript( if( strcmp(zCmd, "match")==0 ){ int jj; char *zAns = zScript+ii; - for(jj=7; jjhints = x; +} + + #ifndef SQLITE_OMIT_AUTOVACUUM /* ** Given a page number of a regular database page, return the page @@ -4451,13 +4471,16 @@ static int btreeCursor( BtCursor *pX; /* Looping over other all cursors */ assert( sqlite3BtreeHoldsMutex(p) ); - assert( wrFlag==0 || wrFlag==1 ); + assert( wrFlag==0 + || wrFlag==BTREE_WRCSR + || wrFlag==(BTREE_WRCSR|BTREE_FORDELETE) + ); /* The following assert statements verify that if this is a sharable ** b-tree database, the connection is holding the required table locks, ** and that no other connection has any open cursor that conflicts with ** this lock. */ - assert( hasSharedCacheTableLock(p, iTable, pKeyInfo!=0, wrFlag+1) ); + assert( hasSharedCacheTableLock(p, iTable, pKeyInfo!=0, (wrFlag?2:1)) ); assert( wrFlag==0 || !hasReadConflicts(p, iTable) ); /* Assert that the caller has opened the required transaction. */ @@ -4482,8 +4505,7 @@ static int btreeCursor( pCur->pKeyInfo = pKeyInfo; pCur->pBtree = p; pCur->pBt = pBt; - assert( wrFlag==0 || wrFlag==BTCF_WriteFlag ); - pCur->curFlags = wrFlag; + pCur->curFlags = wrFlag ? BTCF_WriteFlag : 0; pCur->curPagerFlags = wrFlag ? 0 : PAGER_GET_READONLY; /* If there are two or more cursors on the same btree, then all such ** cursors *must* have the BTCF_Multiple flag set. */ @@ -4895,7 +4917,9 @@ static int accessPayload( /* If required, populate the overflow page-list cache. */ if( (pCur->curFlags & BTCF_ValidOvfl)!=0 ){ - assert(!pCur->aOverflow[iIdx] || pCur->aOverflow[iIdx]==nextPage); + assert( pCur->aOverflow[iIdx]==0 + || pCur->aOverflow[iIdx]==nextPage + || CORRUPT_DB ); pCur->aOverflow[iIdx] = nextPage; } @@ -10028,14 +10052,6 @@ int sqlite3BtreeSetVersion(Btree *pBtree, int iVersion){ return rc; } -/* -** set the mask of hint flags for cursor pCsr. -*/ -void sqlite3BtreeCursorHints(BtCursor *pCsr, unsigned int mask){ - assert( mask==BTREE_BULKLOAD || mask==BTREE_SEEK_EQ || mask==0 ); - pCsr->hints = mask; -} - #ifdef SQLITE_DEBUG /* ** Return true if the cursor has a hint specified. This routine is diff --git a/src/btree.h b/src/btree.h index 93e2e9ce8d..c9b494795f 100644 --- a/src/btree.h +++ b/src/btree.h @@ -150,8 +150,37 @@ int sqlite3BtreeNewDb(Btree *p); #define BTREE_DATA_VERSION 15 /* A virtual meta-value */ /* -** Values that may be OR'd together to form the second argument of an -** sqlite3BtreeCursorHints() call. +** Kinds of hints that can be passed into the sqlite3BtreeCursorHint() +** interface. +** +** BTREE_HINT_RANGE (arguments: Expr*, Mem*) +** +** The first argument is an Expr* (which is guaranteed to be constant for +** the lifetime of the cursor) that defines constraints on which rows +** might be fetched with this cursor. The Expr* tree may contain +** TK_REGISTER nodes that refer to values stored in the array of registers +** passed as the second parameter. In other words, if Expr.op==TK_REGISTER +** then the value of the node is the value in Mem[pExpr.iTable]. Any +** TK_COLUMN node in the expression tree refers to the Expr.iColumn-th +** column of the b-tree of the cursor. The Expr tree will not contain +** any function calls nor subqueries nor references to b-trees other than +** the cursor being hinted. +** +** The design of the _RANGE hint is aid b-tree implementations that try +** to prefetch content from remote machines - to provide those +** implementations with limits on what needs to be prefetched and thereby +** reduce network bandwidth. +** +** Note that BTREE_HINT_FLAGS with BTREE_BULKLOAD is the only hint used by +** standard SQLite. The other hints are provided for extentions that use +** the SQLite parser and code generator but substitute their own storage +** engine. +*/ +#define BTREE_HINT_RANGE 0 /* Range constraints on queries */ + +/* +** Values that may be OR'd together to form the argument to the +** BTREE_HINT_FLAGS hint for sqlite3BtreeCursorHint(): ** ** The BTREE_BULKLOAD flag is set on index cursors when the index is going ** to be filled with content that is already in sorted order. @@ -165,6 +194,22 @@ int sqlite3BtreeNewDb(Btree *p); #define BTREE_BULKLOAD 0x00000001 /* Used to full index in sorted order */ #define BTREE_SEEK_EQ 0x00000002 /* EQ seeks only - no range seeks */ +/* +** Flags passed as the third argument to sqlite3BtreeCursor(). +** +** For read-only cursors the wrFlag argument is always zero. For read-write +** cursors it may be set to either (BTREE_WRCSR|BTREE_FORDELETE) or +** (BTREE_WRCSR). If the BTREE_FORDELETE flag is set, then the cursor will +** only be used by SQLite for the following: +** +** * to seek to and delete specific entries, and/or +** +** * to read values that will be used to create keys that other +** BTREE_FORDELETE cursors will seek to and delete. +*/ +#define BTREE_WRCSR 0x00000004 /* read-write cursor */ +#define BTREE_FORDELETE 0x00000008 /* Cursor is for seek/delete only */ + int sqlite3BtreeCursor( Btree*, /* BTree containing table to open */ int iTable, /* Index of root page */ @@ -174,6 +219,10 @@ int sqlite3BtreeCursor( ); int sqlite3BtreeCursorSize(void); void sqlite3BtreeCursorZero(BtCursor*); +void sqlite3BtreeCursorHintFlags(BtCursor*, unsigned); +#ifdef SQLITE_ENABLE_CURSOR_HINTS +void sqlite3BtreeCursorHint(BtCursor*, int, ...); +#endif int sqlite3BtreeCloseCursor(BtCursor*); int sqlite3BtreeMovetoUnpacked( @@ -208,7 +257,6 @@ int sqlite3BtreePutData(BtCursor*, u32 offset, u32 amt, void*); void sqlite3BtreeIncrblobCursor(BtCursor *); void sqlite3BtreeClearCursor(BtCursor *); int sqlite3BtreeSetVersion(Btree *pBt, int iVersion); -void sqlite3BtreeCursorHints(BtCursor *, unsigned int mask); #ifdef SQLITE_DEBUG int sqlite3BtreeCursorHasHint(BtCursor*, unsigned int mask); #endif diff --git a/src/delete.c b/src/delete.c index faef3a814e..cd683e37d8 100644 --- a/src/delete.c +++ b/src/delete.c @@ -478,12 +478,13 @@ void sqlite3DeleteFrom( */ if( !isView ){ int iAddrOnce = 0; + u8 p5 = (eOnePass==ONEPASS_OFF ? 0 : OPFLAG_FORDELETE); if( eOnePass==ONEPASS_MULTI ){ iAddrOnce = sqlite3CodeOnce(pParse); VdbeCoverage(v); } testcase( IsVirtual(pTab) ); - sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, iTabCur, aToOpen, - &iDataCur, &iIdxCur); + sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, p5, iTabCur, + aToOpen, &iDataCur, &iIdxCur); assert( pPk || IsVirtual(pTab) || iDataCur==iTabCur ); assert( pPk || IsVirtual(pTab) || iIdxCur==iDataCur+1 ); if( eOnePass==ONEPASS_MULTI ) sqlite3VdbeJumpHere(v, iAddrOnce); diff --git a/src/expr.c b/src/expr.c index a93bab991d..6371e15ca7 100644 --- a/src/expr.c +++ b/src/expr.c @@ -548,11 +548,11 @@ Expr *sqlite3PExpr( const Token *pToken /* Argument token */ ){ Expr *p; - if( op==TK_AND && pLeft && pRight && pParse->nErr==0 ){ + if( op==TK_AND && pParse->nErr==0 ){ /* Take advantage of short-circuit false optimization for AND */ p = sqlite3ExprAnd(pParse->db, pLeft, pRight); }else{ - p = sqlite3ExprAlloc(pParse->db, op, pToken, 1); + p = sqlite3ExprAlloc(pParse->db, op & TKFLG_MASK, pToken, 1); sqlite3ExprAttachSubtrees(pParse->db, p, pLeft, pRight); } if( p ) { @@ -1415,6 +1415,22 @@ int sqlite3ExprIsConstantOrFunction(Expr *p, u8 isInit){ return exprIsConst(p, 4+isInit, 0); } +#ifdef SQLITE_ENABLE_CURSOR_HINTS +/* +** Walk an expression tree. Return 1 if the expression contains a +** subquery of some kind. Return 0 if there are no subqueries. +*/ +int sqlite3ExprContainsSubquery(Expr *p){ + Walker w; + memset(&w, 0, sizeof(w)); + w.eCode = 1; + w.xExprCallback = sqlite3ExprWalkNoop; + w.xSelectCallback = selectNodeIsConstant; + sqlite3WalkExpr(&w, p); + return w.eCode==0; +} +#endif + /* ** If the expression p codes a constant integer that is small enough ** to fit in a 32-bit integer, return 1 and put the value of the integer @@ -3356,6 +3372,10 @@ void sqlite3ExprCodeAndCache(Parse *pParse, Expr *pExpr, int target){ ** ** The SQLITE_ECEL_FACTOR argument allows constant arguments to be ** factored out into initialization code. +** +** The SQLITE_ECEL_REF flag means that expressions in the list with +** ExprList.a[].u.x.iOrderByCol>0 have already been evaluated and stored +** in registers at srcReg, and so the value can be copied from there. */ int sqlite3ExprCodeExprList( Parse *pParse, /* Parsing context */ diff --git a/src/insert.c b/src/insert.c index 9274b0771d..3d213a8d37 100644 --- a/src/insert.c +++ b/src/insert.c @@ -762,7 +762,7 @@ void sqlite3Insert( /* If this is not a view, open the table and and all indices */ if( !isView ){ int nIdx; - nIdx = sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, -1, 0, + nIdx = sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, 0, -1, 0, &iDataCur, &iIdxCur); aRegIdx = sqlite3DbMallocRaw(db, sizeof(int)*(nIdx+1)); if( aRegIdx==0 ){ @@ -1650,6 +1650,7 @@ int sqlite3OpenTableAndIndices( Parse *pParse, /* Parsing context */ Table *pTab, /* Table to be opened */ int op, /* OP_OpenRead or OP_OpenWrite */ + u8 p5, /* P5 value for OP_Open* instructions */ int iBase, /* Use this for the table cursor, if there is one */ u8 *aToOpen, /* If not NULL: boolean for each table and index */ int *piDataCur, /* Write the database source cursor number here */ @@ -1662,6 +1663,7 @@ int sqlite3OpenTableAndIndices( Vdbe *v; assert( op==OP_OpenRead || op==OP_OpenWrite ); + assert( op==OP_OpenWrite || p5==0 ); if( IsVirtual(pTab) ){ /* This routine is a no-op for virtual tables. Leave the output ** variables *piDataCur and *piIdxCur uninitialized so that valgrind @@ -1689,6 +1691,7 @@ int sqlite3OpenTableAndIndices( if( aToOpen==0 || aToOpen[i+1] ){ sqlite3VdbeAddOp3(v, op, iIdxCur, pIdx->tnum, iDb); sqlite3VdbeSetP4KeyInfo(pParse, pIdx); + sqlite3VdbeChangeP5(v, p5); VdbeComment((v, "%s", pIdx->zName)); } } diff --git a/src/main.c b/src/main.c index ef2fa66ec3..56d4588494 100644 --- a/src/main.c +++ b/src/main.c @@ -740,6 +740,36 @@ int sqlite3_db_release_memory(sqlite3 *db){ return SQLITE_OK; } +/* +** Flush any dirty pages in the pager-cache for any attached database +** to disk. +*/ +int sqlite3_db_cacheflush(sqlite3 *db){ + int i; + int rc = SQLITE_OK; + int bSeenBusy = 0; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + sqlite3BtreeEnterAll(db); + for(i=0; rc==SQLITE_OK && inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( pBt && sqlite3BtreeIsInTrans(pBt) ){ + Pager *pPager = sqlite3BtreePager(pBt); + rc = sqlite3PagerFlush(pPager); + if( rc==SQLITE_BUSY ){ + bSeenBusy = 1; + rc = SQLITE_OK; + } + } + } + sqlite3BtreeLeaveAll(db); + sqlite3_mutex_leave(db->mutex); + return ((rc==SQLITE_OK && bSeenBusy) ? SQLITE_BUSY : rc); +} + /* ** Configuration settings for an individual database connection */ @@ -2954,6 +2984,21 @@ opendb_out: void *pArg = sqlite3GlobalConfig.pSqllogArg; sqlite3GlobalConfig.xSqllog(pArg, db, zFilename, 0); } +#endif +#if defined(SQLITE_HAS_CODEC) + if( rc==SQLITE_OK ){ + const char *zHexKey = sqlite3_uri_parameter(zOpen, "hexkey"); + if( zHexKey && zHexKey[0] ){ + u8 iByte; + int i; + char zKey[40]; + for(i=0, iByte=0; ipnBytesFreed. */ static SQLITE_NOINLINE void measureAllocationSize(sqlite3 *db, void *p){ - if( p ) *db->pnBytesFreed += sqlite3DbMallocSize(db,p); + *db->pnBytesFreed += sqlite3DbMallocSize(db,p); } /* diff --git a/src/mem1.c b/src/mem1.c index da6ae33e4c..b960ccfd47 100644 --- a/src/mem1.c +++ b/src/mem1.c @@ -171,11 +171,12 @@ static void sqlite3MemFree(void *pPrior){ ** or xRealloc(). */ static int sqlite3MemSize(void *pPrior){ - assert( pPrior!=0 ); #ifdef SQLITE_MALLOCSIZE + assert( pPrior!=0 ); return (int)SQLITE_MALLOCSIZE(pPrior); #else sqlite3_int64 *p; + assert( pPrior!=0 ); p = (sqlite3_int64*)pPrior; p--; return (int)p[0]; diff --git a/src/pager.c b/src/pager.c index 3169b396e4..7e12799c84 100644 --- a/src/pager.c +++ b/src/pager.c @@ -4538,6 +4538,25 @@ static int pagerStress(void *p, PgHdr *pPg){ return pager_error(pPager, rc); } +/* +** Flush all unreferenced dirty pages to disk. +*/ +int sqlite3PagerFlush(Pager *pPager){ + int rc = pPager->errCode; + if( !MEMDB ){ + PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache); + assert( assert_pager_state(pPager) ); + while( rc==SQLITE_OK && pList ){ + PgHdr *pNext = pList->pDirty; + if( pList->nRef==0 ){ + rc = pagerStress((void*)pPager, pList); + } + pList = pNext; + } + } + + return rc; +} /* ** Allocate and initialize a new Pager object and put a pointer to it @@ -5974,9 +5993,10 @@ int sqlite3PagerWrite(PgHdr *pPg){ Pager *pPager = pPg->pPager; assert( (pPg->flags & PGHDR_MMAP)==0 ); assert( pPager->eState>=PAGER_WRITER_LOCKED ); - assert( pPager->eState!=PAGER_ERROR ); assert( assert_pager_state(pPager) ); - if( (pPg->flags & PGHDR_WRITEABLE)!=0 && pPager->dbSize>=pPg->pgno ){ + if( pPager->errCode ){ + return pPager->errCode; + }else if( (pPg->flags & PGHDR_WRITEABLE)!=0 && pPager->dbSize>=pPg->pgno ){ if( pPager->nSavepoint ) return subjournalPageIfRequired(pPg); return SQLITE_OK; }else if( pPager->sectorSize > (u32)pPager->pageSize ){ @@ -6163,30 +6183,33 @@ int sqlite3PagerSync(Pager *pPager, const char *zMaster){ ** and SQLite error code is returned. */ int sqlite3PagerExclusiveLock(Pager *pPager, PgHdr *pPage1){ - int rc = SQLITE_OK; - assert( pPager->eState==PAGER_WRITER_CACHEMOD - || pPager->eState==PAGER_WRITER_DBMOD - || pPager->eState==PAGER_WRITER_LOCKED - ); + int rc = pPager->errCode; assert( assert_pager_state(pPager) ); - if( 0==pagerUseWal(pPager) ){ - rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); - } -#ifndef SQLITE_OMIT_CONCURRENT - else{ - if( pPager->pAllRead ){ - /* This is an CONCURRENT transaction. Attempt to lock the wal database - ** here. If SQLITE_BUSY (but not SQLITE_BUSY_SNAPSHOT) is returned, - ** invoke the busy-handler and try again for as long as it returns - ** non-zero. */ - do { - rc = sqlite3WalLockForCommit(pPager->pWal, pPage1, pPager->pAllRead); - }while( rc==SQLITE_BUSY - && pPager->xBusyHandler(pPager->pBusyHandlerArg) - ); + if( rc==SQLITE_OK ){ + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + || pPager->eState==PAGER_WRITER_LOCKED + ); + assert( assert_pager_state(pPager) ); + if( 0==pagerUseWal(pPager) ){ + rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); + } +#ifndef SQLITE_OMIT_CONCURRENT + else{ + if( pPager->pAllRead ){ + /* This is an CONCURRENT transaction. Attempt to lock the wal database + ** here. If SQLITE_BUSY (but not SQLITE_BUSY_SNAPSHOT) is returned, + ** invoke the busy-handler and try again for as long as it returns + ** non-zero. */ + do { + rc = sqlite3WalLockForCommit(pPager->pWal, pPage1, pPager->pAllRead); + }while( rc==SQLITE_BUSY + && pPager->xBusyHandler(pPager->pBusyHandlerArg) + ); + } } - } #endif /* SQLITE_OMIT_CONCURRENT */ + } return rc; } diff --git a/src/pager.h b/src/pager.h index e3c38faf49..e99b46fafa 100644 --- a/src/pager.h +++ b/src/pager.h @@ -132,6 +132,7 @@ int sqlite3PagerGetJournalMode(Pager*); int sqlite3PagerOkToChangeJournalMode(Pager*); i64 sqlite3PagerJournalSizeLimit(Pager *, i64); sqlite3_backup **sqlite3PagerBackupPtr(Pager*); +int sqlite3PagerFlush(Pager*); /* Functions used to obtain and release page references. */ int sqlite3PagerAcquire(Pager *pPager, Pgno pgno, DbPage **ppPage, int clrFlag); diff --git a/src/parse.y b/src/parse.y index 63bf5e4715..6b4bf94240 100644 --- a/src/parse.y +++ b/src/parse.y @@ -946,6 +946,13 @@ term(A) ::= CTIME_KW(OP). { pOut->zStart = pLeft->zStart; pOut->zEnd = pRight->zEnd; } + + /* If doNot is true, then add a TK_NOT Expr-node wrapper around the + ** outside of *ppExpr. + */ + static void exprNot(Parse *pParse, int doNot, Expr **ppExpr){ + if( doNot ) *ppExpr = sqlite3PExpr(pParse, TK_NOT, *ppExpr, 0, 0); + } } expr(A) ::= expr(X) AND(OP) expr(Y). {spanBinaryExpr(&A,pParse,@OP,&X,&Y);} @@ -968,7 +975,7 @@ expr(A) ::= expr(X) likeop(OP) expr(Y). [LIKE_KW] { pList = sqlite3ExprListAppend(pParse,0, Y.pExpr); pList = sqlite3ExprListAppend(pParse,pList, X.pExpr); A.pExpr = sqlite3ExprFunction(pParse, pList, &OP.eOperator); - if( OP.bNot ) A.pExpr = sqlite3PExpr(pParse, TK_NOT, A.pExpr, 0, 0); + exprNot(pParse, OP.bNot, &A.pExpr); A.zStart = X.zStart; A.zEnd = Y.zEnd; if( A.pExpr ) A.pExpr->flags |= EP_InfixFunc; @@ -979,7 +986,7 @@ expr(A) ::= expr(X) likeop(OP) expr(Y) ESCAPE expr(E). [LIKE_KW] { pList = sqlite3ExprListAppend(pParse,pList, X.pExpr); pList = sqlite3ExprListAppend(pParse,pList, E.pExpr); A.pExpr = sqlite3ExprFunction(pParse, pList, &OP.eOperator); - if( OP.bNot ) A.pExpr = sqlite3PExpr(pParse, TK_NOT, A.pExpr, 0, 0); + exprNot(pParse, OP.bNot, &A.pExpr); A.zStart = X.zStart; A.zEnd = E.zEnd; if( A.pExpr ) A.pExpr->flags |= EP_InfixFunc; @@ -1069,7 +1076,7 @@ expr(A) ::= expr(W) between_op(N) expr(X) AND expr(Y). [BETWEEN] { }else{ sqlite3ExprListDelete(pParse->db, pList); } - if( N ) A.pExpr = sqlite3PExpr(pParse, TK_NOT, A.pExpr, 0, 0); + exprNot(pParse, N, &A.pExpr); A.zStart = W.zStart; A.zEnd = Y.zEnd; } @@ -1124,7 +1131,7 @@ expr(A) ::= expr(W) between_op(N) expr(X) AND expr(Y). [BETWEEN] { }else{ sqlite3ExprListDelete(pParse->db, Y); } - if( N ) A.pExpr = sqlite3PExpr(pParse, TK_NOT, A.pExpr, 0, 0); + exprNot(pParse, N, &A.pExpr); } A.zStart = X.zStart; A.zEnd = &E.z[E.n]; @@ -1150,7 +1157,7 @@ expr(A) ::= expr(W) between_op(N) expr(X) AND expr(Y). [BETWEEN] { }else{ sqlite3SelectDelete(pParse->db, Y); } - if( N ) A.pExpr = sqlite3PExpr(pParse, TK_NOT, A.pExpr, 0, 0); + exprNot(pParse, N, &A.pExpr); A.zStart = X.zStart; A.zEnd = &E.z[E.n]; } @@ -1164,7 +1171,7 @@ expr(A) ::= expr(W) between_op(N) expr(X) AND expr(Y). [BETWEEN] { }else{ sqlite3SrcListDelete(pParse->db, pSrc); } - if( N ) A.pExpr = sqlite3PExpr(pParse, TK_NOT, A.pExpr, 0, 0); + exprNot(pParse, N, &A.pExpr); A.zStart = X.zStart; A.zEnd = Z.z ? &Z.z[Z.n] : &Y.z[Y.n]; } diff --git a/src/pragma.c b/src/pragma.c index 64614a7ebc..14c4811746 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1465,7 +1465,7 @@ void sqlite3Pragma( sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); sqlite3VdbeJumpHere(v, addr); sqlite3ExprCacheClear(pParse); - sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenRead, + sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenRead, 0, 1, 0, &iDataCur, &iIdxCur); sqlite3VdbeAddOp2(v, OP_Integer, 0, 7); for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ diff --git a/src/select.c b/src/select.c index 67e25f0b29..8db983891f 100644 --- a/src/select.c +++ b/src/select.c @@ -4452,7 +4452,7 @@ static int selectExpander(Walker *pWalker, Select *p){ ** Walker.xSelectCallback is set to do something useful for every ** subquery in the parser tree. */ -static int exprWalkNoop(Walker *NotUsed, Expr *NotUsed2){ +int sqlite3ExprWalkNoop(Walker *NotUsed, Expr *NotUsed2){ UNUSED_PARAMETER2(NotUsed, NotUsed2); return WRC_Continue; } @@ -4473,7 +4473,7 @@ static int exprWalkNoop(Walker *NotUsed, Expr *NotUsed2){ static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){ Walker w; memset(&w, 0, sizeof(w)); - w.xExprCallback = exprWalkNoop; + w.xExprCallback = sqlite3ExprWalkNoop; w.pParse = pParse; if( pParse->hasCompound ){ w.xSelectCallback = convertCompoundSelectToSubquery; @@ -4540,7 +4540,7 @@ static void sqlite3SelectAddTypeInfo(Parse *pParse, Select *pSelect){ Walker w; memset(&w, 0, sizeof(w)); w.xSelectCallback2 = selectAddSubqueryTypeInfo; - w.xExprCallback = exprWalkNoop; + w.xExprCallback = sqlite3ExprWalkNoop; w.pParse = pParse; sqlite3WalkSelect(&w, pSelect); #endif diff --git a/src/sqlite.h.in b/src/sqlite.h.in index b5d12334d8..961c5169fc 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -478,6 +478,7 @@ int sqlite3_exec( #define SQLITE_IOERR_GETTEMPPATH (SQLITE_IOERR | (25<<8)) #define SQLITE_IOERR_CONVPATH (SQLITE_IOERR | (26<<8)) #define SQLITE_IOERR_VNODE (SQLITE_IOERR | (27<<8)) +#define SQLITE_IOERR_AUTH (SQLITE_IOERR | (28<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) #define SQLITE_BUSY_SNAPSHOT (SQLITE_BUSY | (2<<8)) @@ -7791,6 +7792,35 @@ int sqlite3_stmt_scanstatus( */ void sqlite3_stmt_scanstatus_reset(sqlite3_stmt*); +/* +** CAPI3REF: Flush caches to disk mid-transaction +** +** If a write-transaction is open when this function is called, any dirty +** pages in the pager-cache that are not currently in use are written out +** to disk. A dirty page may be in use if a database cursor created by an +** active SQL statement is reading from it, or if it is page 1 of a database +** file (page 1 is always "in use"). Dirty pages are flushed for all +** databases - "main", "temp" and any attached databases. +** +** If this function needs to obtain extra database locks before dirty pages +** can be flushed to disk, it does so. If said locks cannot be obtained +** immediately and there is a busy-handler callback configured, it is invoked +** in the usual manner. If the required lock still cannot be obtained, then +** the database is skipped and an attempt made to flush any dirty pages +** belonging to the next (if any) database. If any databases are skipped +** because locks cannot be obtained, but no other error occurs, this +** function returns SQLITE_BUSY. +** +** If any other error occurs while flushing dirty pages to disk (for +** example an IO error or out-of-memory condition), then processing is +** abandoned and an SQLite error code returned to the caller immediately. +** +** Otherwise, if no error occurs, SQLITE_OK is returned. +** +** This function does not set the database handle error code or message +** returned by the sqlite3_errcode() and sqlite3_errmsg() functions. +*/ +int sqlite3_db_cacheflush(sqlite3*); /* ** Undo the hack that converts floating point types to integer for diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 91b0312da7..fc15d12db8 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1311,6 +1311,7 @@ struct sqlite3 { #define SQLITE_Transitive 0x0200 /* Transitive constraints */ #define SQLITE_OmitNoopJoin 0x0400 /* Omit unused tables in joins */ #define SQLITE_Stat34 0x0800 /* Use STAT3 or STAT4 data */ +#define SQLITE_CursorHints 0x2000 /* Add OP_CursorHint opcodes */ #define SQLITE_AllOpts 0xffff /* All optimizations */ /* @@ -2795,7 +2796,8 @@ struct AuthContext { #define OPFLAG_TYPEOFARG 0x80 /* OP_Column only used for typeof() */ #define OPFLAG_BULKCSR 0x01 /* OP_Open** used to open bulk cursor */ #define OPFLAG_SEEKEQ 0x02 /* OP_Open** cursor uses EQ seek only */ -#define OPFLAG_P2ISREG 0x04 /* P2 to OP_Open** is a register number */ +#define OPFLAG_FORDELETE 0x08 /* OP_Open is opening for-delete csr */ +#define OPFLAG_P2ISREG 0x10 /* P2 to OP_Open** is a register number */ #define OPFLAG_PERMUTE 0x01 /* OP_Compare: use the permutation */ /* @@ -3024,6 +3026,7 @@ struct Walker { int iCur; /* A cursor number */ SrcList *pSrcList; /* FROM clause */ struct SrcCount *pSrcCount; /* Counting column references */ + struct CCurHint *pCCurHint; /* Used by codeCursorHint() */ } u; }; @@ -3033,6 +3036,7 @@ int sqlite3WalkExprList(Walker*, ExprList*); int sqlite3WalkSelect(Walker*, Select*); int sqlite3WalkSelectExpr(Walker*, Select*); int sqlite3WalkSelectFrom(Walker*, Select*); +int sqlite3ExprWalkNoop(Walker*, Expr*); /* ** Return code from the parse-tree walking primitives and their @@ -3449,6 +3453,9 @@ int sqlite3ExprIsConstant(Expr*); int sqlite3ExprIsConstantNotJoin(Expr*); int sqlite3ExprIsConstantOrFunction(Expr*, u8); int sqlite3ExprIsTableConstant(Expr*,int); +#ifdef SQLITE_ENABLE_CURSOR_HINTS +int sqlite3ExprContainsSubquery(Expr*); +#endif int sqlite3ExprIsInteger(Expr*, int*); int sqlite3ExprCanBeNull(const Expr*); int sqlite3ExprNeedsNoAffinityChange(const Expr*, char); @@ -3461,7 +3468,7 @@ void sqlite3ResolvePartIdxLabel(Parse*,int); void sqlite3GenerateConstraintChecks(Parse*,Table*,int*,int,int,int,int, u8,u8,int,int*); void sqlite3CompleteInsertion(Parse*,Table*,int,int,int,int*,int,int,int); -int sqlite3OpenTableAndIndices(Parse*, Table*, int, int, u8*, int*, int*); +int sqlite3OpenTableAndIndices(Parse*, Table*, int, u8, int, u8*, int*, int*); void sqlite3BeginWriteOperation(Parse*, int, int); void sqlite3MultiWrite(Parse*); void sqlite3MayAbort(Parse*); diff --git a/src/status.c b/src/status.c index a7fddf060f..69f92ff7c6 100644 --- a/src/status.c +++ b/src/status.c @@ -19,15 +19,15 @@ /* ** Variables in which to record status information. */ +#if SQLITE_PTRSIZE>4 +typedef sqlite3_int64 sqlite3StatValueType; +#else +typedef u32 sqlite3StatValueType; +#endif typedef struct sqlite3StatType sqlite3StatType; static SQLITE_WSD struct sqlite3StatType { -#if SQLITE_PTRSIZE>4 - sqlite3_int64 nowValue[10]; /* Current value */ - sqlite3_int64 mxValue[10]; /* Maximum value */ -#else - u32 nowValue[10]; /* Current value */ - u32 mxValue[10]; /* Maximum value */ -#endif + sqlite3StatValueType nowValue[10]; /* Current value */ + sqlite3StatValueType mxValue[10]; /* Maximum value */ } sqlite3Stat = { {0,}, {0,} }; /* @@ -112,7 +112,10 @@ void sqlite3StatusDown(int op, int N){ ** The caller must hold the appropriate mutex. */ void sqlite3StatusHighwater(int op, int X){ + sqlite3StatValueType newValue; wsdStatInit; + assert( X>=0 ); + newValue = (sqlite3StatValueType)X; assert( op>=0 && op=0 && opwsdStat.mxValue[op] ){ - wsdStat.mxValue[op] = X; + if( newValue>wsdStat.mxValue[op] ){ + wsdStat.mxValue[op] = newValue; } } diff --git a/src/test1.c b/src/test1.c index 6c47754bf9..dd190feebc 100644 --- a/src/test1.c +++ b/src/test1.c @@ -4688,6 +4688,34 @@ static int test_db_release_memory( return TCL_OK; } +/* +** Usage: sqlite3_db_cacheflush DB +** +** Attempt to flush any dirty pages to disk. +*/ +static int test_db_cacheflush( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3 *db; + int rc; + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB"); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; + rc = sqlite3_db_cacheflush(db); + if( rc ){ + Tcl_SetResult(interp, (char *)sqlite3ErrStr(rc), TCL_STATIC); + return TCL_ERROR; + } + + Tcl_ResetResult(interp); + return TCL_OK; +} + /* ** Usage: sqlite3_db_filename DB DBNAME ** @@ -6876,6 +6904,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "sqlite3_release_memory", test_release_memory, 0}, { "sqlite3_db_release_memory", test_db_release_memory, 0}, + { "sqlite3_db_cacheflush", test_db_cacheflush, 0}, { "sqlite3_db_filename", test_db_filename, 0}, { "sqlite3_db_readonly", test_db_readonly, 0}, { "sqlite3_soft_heap_limit", test_soft_heap_limit, 0}, diff --git a/src/test3.c b/src/test3.c index 07d12d28c0..212dff5fad 100644 --- a/src/test3.c +++ b/src/test3.c @@ -214,11 +214,12 @@ static int btree_cursor( pBt = sqlite3TestTextToPtr(argv[1]); if( Tcl_GetInt(interp, argv[2], &iTable) ) return TCL_ERROR; if( Tcl_GetBoolean(interp, argv[3], &wrFlag) ) return TCL_ERROR; + if( wrFlag ) wrFlag = BTREE_WRCSR; pCur = (BtCursor *)ckalloc(sqlite3BtreeCursorSize()); memset(pCur, 0, sqlite3BtreeCursorSize()); sqlite3BtreeEnter(pBt); #ifndef SQLITE_OMIT_SHARED_CACHE - rc = sqlite3BtreeLockTable(pBt, iTable, wrFlag); + rc = sqlite3BtreeLockTable(pBt, iTable, !!wrFlag); #endif if( rc==SQLITE_OK ){ rc = sqlite3BtreeCursor(pBt, iTable, wrFlag, 0, pCur); @@ -598,6 +599,48 @@ static int btree_set_cache_size( return TCL_OK; } +/* +** usage: btree_insert CSR ?KEY? VALUE +** +** Set the size of the cache used by btree $ID. +*/ +static int btree_insert( + ClientData clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *const objv[] +){ + BtCursor *pCur; + int rc; + void *pKey = 0; + int nKey = 0; + void *pData = 0; + int nData = 0; + + if( objc!=4 && objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "?-intkey? CSR KEY VALUE"); + return TCL_ERROR; + } + + if( objc==4 ){ + if( Tcl_GetIntFromObj(interp, objv[2], &nKey) ) return TCL_ERROR; + pData = (void*)Tcl_GetByteArrayFromObj(objv[3], &nData); + }else{ + pKey = (void*)Tcl_GetByteArrayFromObj(objv[2], &nKey); + } + pCur = (BtCursor*)sqlite3TestTextToPtr(Tcl_GetString(objv[1])); + + sqlite3BtreeEnter(pCur->pBtree); + rc = sqlite3BtreeInsert(pCur, pKey, nKey, pData, nData, 0, 0, 0); + sqlite3BtreeLeave(pCur->pBtree); + + Tcl_ResetResult(interp); + if( rc ){ + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); + return TCL_ERROR; + } + return TCL_OK; +} /* @@ -629,5 +672,7 @@ int Sqlitetest3_Init(Tcl_Interp *interp){ Tcl_CreateCommand(interp, aCmd[i].zName, aCmd[i].xProc, 0, 0); } + Tcl_CreateObjCommand(interp, "btree_insert", btree_insert, 0, 0); + return TCL_OK; } diff --git a/src/test_config.c b/src/test_config.c index 51e4a6921d..451a411e21 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -119,6 +119,12 @@ static void set_options(Tcl_Interp *interp){ Tcl_SetVar2(interp, "sqlite_options", "8_3_names", "0", TCL_GLOBAL_ONLY); #endif +#ifdef SQLITE_ENABLE_CURSOR_HINTS + Tcl_SetVar2(interp, "sqlite_options", "cursorhints", "1", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "cursorhints", "0", TCL_GLOBAL_ONLY); +#endif + #ifdef SQLITE_ENABLE_MEMSYS3 Tcl_SetVar2(interp, "sqlite_options", "mem3", "1", TCL_GLOBAL_ONLY); #else diff --git a/src/update.c b/src/update.c index 5cb5574909..1335c269ed 100644 --- a/src/update.c +++ b/src/update.c @@ -429,7 +429,7 @@ void sqlite3Update( if( aiCurOnePass[0]>=0 ) aToOpen[aiCurOnePass[0]-iBaseCur] = 0; if( aiCurOnePass[1]>=0 ) aToOpen[aiCurOnePass[1]-iBaseCur] = 0; } - sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, iBaseCur, aToOpen, + sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, 0, iBaseCur, aToOpen, 0, 0); } diff --git a/src/vdbe.c b/src/vdbe.c index a08d12a911..eb605acc94 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3356,7 +3356,6 @@ case OP_ReopenIdx: { case OP_OpenRead: case OP_OpenWrite: - assert( (pOp->p5&(OPFLAG_P2ISREG|OPFLAG_BULKCSR|OPFLAG_SEEKEQ))==pOp->p5 ); assert( pOp->opcode==OP_OpenWrite || pOp->p5==0 || pOp->p5==OPFLAG_SEEKEQ ); assert( p->bIsReader ); assert( pOp->opcode==OP_OpenRead || pOp->opcode==OP_ReopenIdx @@ -3377,7 +3376,8 @@ case OP_OpenWrite: pX = pDb->pBt; assert( pX!=0 ); if( pOp->opcode==OP_OpenWrite ){ - wrFlag = 1; + assert( OPFLAG_FORDELETE==BTREE_FORDELETE ); + wrFlag = BTREE_WRCSR | (pOp->p5 & OPFLAG_FORDELETE); assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( pDb->pSchema->file_format < p->minWriteFileFormat ){ p->minWriteFileFormat = pDb->pSchema->file_format; @@ -3429,8 +3429,12 @@ case OP_OpenWrite: open_cursor_set_hints: assert( OPFLAG_BULKCSR==BTREE_BULKLOAD ); assert( OPFLAG_SEEKEQ==BTREE_SEEK_EQ ); - sqlite3BtreeCursorHints(pCur->pCursor, - (pOp->p5 & (OPFLAG_BULKCSR|OPFLAG_SEEKEQ))); + testcase( pOp->p5 & OPFLAG_BULKCSR ); +#ifdef SQLITE_ENABLE_CURSOR_HINT + testcase( pOp->p2 & OPFLAG_SEEKEQ ); +#endif + sqlite3BtreeCursorHintFlags(pCur->pCursor, + (pOp->p5 & (OPFLAG_BULKCSR|OPFLAG_SEEKEQ))); break; } @@ -3497,11 +3501,11 @@ case OP_OpenEphemeral: { assert( pKeyInfo->db==db ); assert( pKeyInfo->enc==ENC(db) ); pCx->pKeyInfo = pKeyInfo; - rc = sqlite3BtreeCursor(pCx->pBt, pgno, 1, pKeyInfo, pCx->pCursor); + rc = sqlite3BtreeCursor(pCx->pBt, pgno, BTREE_WRCSR, pKeyInfo, pCx->pCursor); } pCx->isTable = 0; }else{ - rc = sqlite3BtreeCursor(pCx->pBt, MASTER_ROOT, 1, 0, pCx->pCursor); + rc = sqlite3BtreeCursor(pCx->pBt, MASTER_ROOT, BTREE_WRCSR, 0, pCx->pCursor); pCx->isTable = 1; } } @@ -6627,6 +6631,26 @@ case OP_Init: { /* jump */ break; } +#ifdef SQLITE_ENABLE_CURSOR_HINTS +/* Opcode: CursorHint P1 * * P4 * +** +** Provide a hint to cursor P1 that it only needs to return rows that +** satisfy the Expr in P4. TK_REGISTER terms in the P4 expression refer +** to values currently held in registers. TK_COLUMN terms in the P4 +** expression refer to columns in the b-tree to which cursor P1 is pointing. +*/ +case OP_CursorHint: { + VdbeCursor *pC; + + assert( pOp->p1>=0 && pOp->p1nCursor ); + assert( pOp->p4type==P4_EXPR ); + pC = p->apCsr[pOp->p1]; + if( pC ){ + sqlite3BtreeCursorHint(pC->pCursor, BTREE_HINT_RANGE, pOp->p4.pExpr, aMem); + } + break; +} +#endif /* SQLITE_ENABLE_CURSOR_HINTS */ /* Opcode: Noop * * * * * ** diff --git a/src/vdbe.h b/src/vdbe.h index 01d902ab87..68a4f6b2a8 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -60,6 +60,9 @@ struct VdbeOp { KeyInfo *pKeyInfo; /* Used when p4type is P4_KEYINFO */ int *ai; /* Used when p4type is P4_INTARRAY */ SubProgram *pProgram; /* Used when p4type is P4_SUBPROGRAM */ +#ifdef SQLITE_ENABLE_CURSOR_HINTS + Expr *pExpr; /* Used when p4type is P4_EXPR */ +#endif int (*xAdvance)(BtCursor *, int *); } p4; #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS @@ -110,6 +113,7 @@ typedef struct VdbeOpList VdbeOpList; #define P4_COLLSEQ (-4) /* P4 is a pointer to a CollSeq structure */ #define P4_FUNCDEF (-5) /* P4 is a pointer to a FuncDef structure */ #define P4_KEYINFO (-6) /* P4 is a pointer to a KeyInfo structure */ +#define P4_EXPR (-7) /* P4 is a pointer to an Expr tree */ #define P4_MEM (-8) /* P4 is a pointer to a Mem* structure */ #define P4_TRANSIENT 0 /* P4 is a pointer to a transient string */ #define P4_VTAB (-10) /* P4 is a pointer to an sqlite3_vtab structure */ diff --git a/src/vdbeInt.h b/src/vdbeInt.h index e92239f512..729304bd8d 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -26,6 +26,17 @@ # define SQLITE_MAX_SCHEMA_RETRY 50 #endif +/* +** VDBE_DISPLAY_P4 is true or false depending on whether or not the +** "explain" P4 display logic is enabled. +*/ +#if !defined(SQLITE_OMIT_EXPLAIN) || !defined(NDEBUG) \ + || defined(VDBE_PROFILE) || defined(SQLITE_DEBUG) +# define VDBE_DISPLAY_P4 1 +#else +# define VDBE_DISPLAY_P4 0 +#endif + /* ** SQL is translated into a sequence of instructions to be ** executed by a virtual machine. Each instruction is an instance diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 23394d2b9b..8e15ef6b0f 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -763,6 +763,12 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){ if( db->pnBytesFreed==0 ) sqlite3KeyInfoUnref((KeyInfo*)p4); break; } +#ifdef SQLITE_ENABLE_CURSOR_HINTS + case P4_EXPR: { + sqlite3ExprDelete(db, (Expr*)p4); + break; + } +#endif case P4_MPRINTF: { if( db->pnBytesFreed==0 ) sqlite3_free(p4); break; @@ -895,6 +901,15 @@ void sqlite3VdbeChangeP4(Vdbe *p, int addr, const char *zP4, int n){ }else if( n==P4_KEYINFO ){ pOp->p4.p = (void*)zP4; pOp->p4type = P4_KEYINFO; +#ifdef SQLITE_ENABLE_CURSOR_HINTS + }else if( n==P4_EXPR ){ + /* Responsibility for deleting the Expr tree is handed over to the + ** VDBE by this operation. The caller should have already invoked + ** sqlite3ExprDup() or whatever other routine is needed to make a + ** private copy of the tree. */ + pOp->p4.pExpr = (Expr*)zP4; + pOp->p4type = P4_EXPR; +#endif }else if( n==P4_VTAB ){ pOp->p4.p = (void*)zP4; pOp->p4type = P4_VTAB; @@ -1085,9 +1100,84 @@ static int displayComment( } #endif /* SQLITE_DEBUG */ +#if VDBE_DISPLAY_P4 && defined(SQLITE_ENABLE_CURSOR_HINTS) +/* +** Translate the P4.pExpr value for an OP_CursorHint opcode into text +** that can be displayed in the P4 column of EXPLAIN output. +*/ +static int displayP4Expr(int nTemp, char *zTemp, Expr *pExpr){ + const char *zOp = 0; + int n; + switch( pExpr->op ){ + case TK_STRING: + sqlite3_snprintf(nTemp, zTemp, "%Q", pExpr->u.zToken); + break; + case TK_INTEGER: + sqlite3_snprintf(nTemp, zTemp, "%d", pExpr->u.iValue); + break; + case TK_NULL: + sqlite3_snprintf(nTemp, zTemp, "NULL"); + break; + case TK_REGISTER: { + sqlite3_snprintf(nTemp, zTemp, "r[%d]", pExpr->iTable); + break; + } + case TK_COLUMN: { + if( pExpr->iColumn<0 ){ + sqlite3_snprintf(nTemp, zTemp, "rowid"); + }else{ + sqlite3_snprintf(nTemp, zTemp, "c%d", (int)pExpr->iColumn); + } + break; + } + case TK_LT: zOp = "LT"; break; + case TK_LE: zOp = "LE"; break; + case TK_GT: zOp = "GT"; break; + case TK_GE: zOp = "GE"; break; + case TK_NE: zOp = "NE"; break; + case TK_EQ: zOp = "EQ"; break; + case TK_IS: zOp = "IS"; break; + case TK_ISNOT: zOp = "ISNOT"; break; + case TK_AND: zOp = "AND"; break; + case TK_OR: zOp = "OR"; break; + case TK_PLUS: zOp = "ADD"; break; + case TK_STAR: zOp = "MUL"; break; + case TK_MINUS: zOp = "SUB"; break; + case TK_REM: zOp = "REM"; break; + case TK_BITAND: zOp = "BITAND"; break; + case TK_BITOR: zOp = "BITOR"; break; + case TK_SLASH: zOp = "DIV"; break; + case TK_LSHIFT: zOp = "LSHIFT"; break; + case TK_RSHIFT: zOp = "RSHIFT"; break; + case TK_CONCAT: zOp = "CONCAT"; break; + case TK_UMINUS: zOp = "MINUS"; break; + case TK_UPLUS: zOp = "PLUS"; break; + case TK_BITNOT: zOp = "BITNOT"; break; + case TK_NOT: zOp = "NOT"; break; + case TK_ISNULL: zOp = "ISNULL"; break; + case TK_NOTNULL: zOp = "NOTNULL"; break; -#if !defined(SQLITE_OMIT_EXPLAIN) || !defined(NDEBUG) \ - || defined(VDBE_PROFILE) || defined(SQLITE_DEBUG) + default: + sqlite3_snprintf(nTemp, zTemp, "%s", "expr"); + break; + } + + if( zOp ){ + sqlite3_snprintf(nTemp, zTemp, "%s(", zOp); + n = sqlite3Strlen30(zTemp); + n += displayP4Expr(nTemp-n, zTemp+n, pExpr->pLeft); + if( npRight ){ + zTemp[n++] = ','; + n += displayP4Expr(nTemp-n, zTemp+n, pExpr->pRight); + } + sqlite3_snprintf(nTemp-n, zTemp+n, ")"); + } + return sqlite3Strlen30(zTemp); +} +#endif /* VDBE_DISPLAY_P4 && defined(SQLITE_ENABLE_CURSOR_HINTS) */ + + +#if VDBE_DISPLAY_P4 /* ** Compute a string that describes the P4 parameter for an opcode. ** Use zTemp for any required temporary buffer space. @@ -1127,6 +1217,12 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ assert( ip4.pExpr); + break; + } +#endif case P4_COLLSEQ: { CollSeq *pColl = pOp->p4.pColl; sqlite3_snprintf(nTemp, zTemp, "(%.20s)", pColl->zName); @@ -1202,7 +1298,7 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ assert( zP4!=0 ); return zP4; } -#endif +#endif /* VDBE_DISPLAY_P4 */ /* ** Declare to the Vdbe that the BTree object at db->aDb[i] is used. @@ -1516,12 +1612,12 @@ int sqlite3VdbeList( pMem->u.i = pOp->p3; /* P3 */ pMem++; - if( sqlite3VdbeMemClearAndResize(pMem, 32) ){ /* P4 */ + if( sqlite3VdbeMemClearAndResize(pMem, 100) ){ /* P4 */ assert( p->db->mallocFailed ); return SQLITE_ERROR; } pMem->flags = MEM_Str|MEM_Term; - zP4 = displayP4(pOp, pMem->z, 32); + zP4 = displayP4(pOp, pMem->z, pMem->szMalloc); if( zP4!=pMem->z ){ sqlite3VdbeMemSetStr(pMem, zP4, -1, SQLITE_UTF8, 0); }else{ diff --git a/src/where.c b/src/where.c index af8e2f35fd..1c87706ea2 100644 --- a/src/where.c +++ b/src/where.c @@ -485,14 +485,20 @@ static LogEst estLog(LogEst N){ ** Convert OP_Column opcodes to OP_Copy in previously generated code. ** ** This routine runs over generated VDBE code and translates OP_Column -** opcodes into OP_Copy, and OP_Rowid into OP_Null, when the table is being -** accessed via co-routine instead of via table lookup. +** opcodes into OP_Copy when the table is being accessed via co-routine +** instead of via table lookup. +** +** If the bIncrRowid parameter is 0, then any OP_Rowid instructions on +** cursor iTabCur are transformed into OP_Null. Or, if bIncrRowid is non-zero, +** then each OP_Rowid is transformed into an instruction to increment the +** value stored in its output register. */ static void translateColumnToCopy( Vdbe *v, /* The VDBE containing code to translate */ int iStart, /* Translate from this opcode to the end */ int iTabCur, /* OP_Column/OP_Rowid references to this table */ - int iRegister /* The first column is in this register */ + int iRegister, /* The first column is in this register */ + int bIncrRowid /* If non-zero, transform OP_rowid to OP_AddImm(1) */ ){ VdbeOp *pOp = sqlite3VdbeGetOp(v, iStart); int iEnd = sqlite3VdbeCurrentAddr(v); @@ -504,9 +510,16 @@ static void translateColumnToCopy( pOp->p2 = pOp->p3; pOp->p3 = 0; }else if( pOp->opcode==OP_Rowid ){ - pOp->opcode = OP_Null; - pOp->p1 = 0; - pOp->p3 = 0; + if( bIncrRowid ){ + /* Increment the value stored in the P2 operand of the OP_Rowid. */ + pOp->opcode = OP_AddImm; + pOp->p1 = pOp->p2; + pOp->p2 = 1; + }else{ + pOp->opcode = OP_Null; + pOp->p1 = 0; + pOp->p3 = 0; + } } } } @@ -614,6 +627,8 @@ static void constructAutomaticIndex( Expr *pPartial = 0; /* Partial Index Expression */ int iContinue = 0; /* Jump here to skip excluded rows */ struct SrcList_item *pTabItem; /* FROM clause term being indexed */ + int addrCounter = 0; /* Address where integer counter is initialized */ + int regBase; /* Array of registers where record is assembled */ /* Generate code to skip over the creation and initialization of the ** transient index on 2nd and subsequent iterations of the loop. */ @@ -742,6 +757,7 @@ static void constructAutomaticIndex( pTabItem = &pWC->pWInfo->pTabList->a[pLevel->iFrom]; if( pTabItem->fg.viaCoroutine ){ int regYield = pTabItem->regReturn; + addrCounter = sqlite3VdbeAddOp2(v, OP_Integer, 0, 0); sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub); addrTop = sqlite3VdbeAddOp1(v, OP_Yield, regYield); VdbeCoverage(v); @@ -755,12 +771,15 @@ static void constructAutomaticIndex( pLoop->wsFlags |= WHERE_PARTIALIDX; } regRecord = sqlite3GetTempReg(pParse); - sqlite3GenerateIndexKey(pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0); + regBase = sqlite3GenerateIndexKey( + pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0 + ); sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue); if( pTabItem->fg.viaCoroutine ){ - translateColumnToCopy(v, addrTop, pLevel->iTabCur, pTabItem->regResult); + sqlite3VdbeChangeP2(v, addrCounter, regBase+n); + translateColumnToCopy(v, addrTop, pLevel->iTabCur, pTabItem->regResult, 1); sqlite3VdbeGoto(v, addrTop); pTabItem->fg.viaCoroutine = 0; }else{ @@ -3991,6 +4010,7 @@ WhereInfo *sqlite3WhereBegin( int ii; /* Loop counter */ sqlite3 *db; /* Database connection */ int rc; /* Return code */ + u8 bFordelete = 0; assert( (wctrlFlags & WHERE_ONEPASS_MULTIROW)==0 || ( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 @@ -4246,8 +4266,11 @@ WhereInfo *sqlite3WhereBegin( && 0==(wsFlags & WHERE_VIRTUALTABLE) )){ pWInfo->eOnePass = bOnerow ? ONEPASS_SINGLE : ONEPASS_MULTI; - if( HasRowid(pTabList->a[0].pTab) ){ - pWInfo->a[0].pWLoop->wsFlags &= ~WHERE_IDX_ONLY; + if( HasRowid(pTabList->a[0].pTab) && (wsFlags & WHERE_IDX_ONLY) ){ + if( wctrlFlags & WHERE_ONEPASS_MULTIROW ){ + bFordelete = OPFLAG_FORDELETE; + } + pWInfo->a[0].pWLoop->wsFlags = (wsFlags & ~WHERE_IDX_ONLY); } } } @@ -4295,6 +4318,14 @@ WhereInfo *sqlite3WhereBegin( SQLITE_INT_TO_PTR(n), P4_INT32); assert( n<=pTab->nCol ); } +#ifdef SQLITE_ENABLE_CURSOR_HINTS + if( pLoop->u.btree.pIndex!=0 ){ + sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ|bFordelete); + }else +#endif + { + sqlite3VdbeChangeP5(v, bFordelete); + } #ifdef SQLITE_ENABLE_COLUMN_USED_MASK sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, pTabItem->iCursor, 0, 0, (const u8*)&pTabItem->colUsed, P4_INT64); @@ -4509,7 +4540,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ */ if( pTabItem->fg.viaCoroutine && !db->mallocFailed ){ translateColumnToCopy(v, pLevel->addrBody, pLevel->iTabCur, - pTabItem->regResult); + pTabItem->regResult, 0); continue; } diff --git a/src/wherecode.c b/src/wherecode.c index 9e00c2714d..87db0e0a25 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -590,6 +590,154 @@ static void whereLikeOptimizationStringFixup( } } +#ifdef SQLITE_ENABLE_CURSOR_HINTS +/* +** Information is passed from codeCursorHint() down to individual nodes of +** the expression tree (by sqlite3WalkExpr()) using an instance of this +** structure. +*/ +struct CCurHint { + int iTabCur; /* Cursor for the main table */ + int iIdxCur; /* Cursor for the index, if pIdx!=0. Unused otherwise */ + Index *pIdx; /* The index used to access the table */ +}; + +/* +** This function is called for every node of an expression that is a candidate +** for a cursor hint on an index cursor. For TK_COLUMN nodes that reference +** the table CCurHint.iTabCur, verify that the same column can be +** accessed through the index. If it cannot, then set pWalker->eCode to 1. +*/ +static int codeCursorHintCheckExpr(Walker *pWalker, Expr *pExpr){ + struct CCurHint *pHint = pWalker->u.pCCurHint; + assert( pHint->pIdx!=0 ); + if( pExpr->op==TK_COLUMN + && pExpr->iTable==pHint->iTabCur + && sqlite3ColumnOfIndex(pHint->pIdx, pExpr->iColumn)<0 + ){ + pWalker->eCode = 1; + } + return WRC_Continue; +} + + +/* +** This function is called on every node of an expression tree used as an +** argument to the OP_CursorHint instruction. If the node is a TK_COLUMN +** that accesses any table other than the one identified by +** CCurHint.iTabCur, then do the following: +** +** 1) allocate a register and code an OP_Column instruction to read +** the specified column into the new register, and +** +** 2) transform the expression node to a TK_REGISTER node that reads +** from the newly populated register. +** +** Also, if the node is a TK_COLUMN that does access the table idenified +** by pCCurHint.iTabCur, and an index is being used (which we will +** know because CCurHint.pIdx!=0) then transform the TK_COLUMN into +** an access of the index rather than the original table. +*/ +static int codeCursorHintFixExpr(Walker *pWalker, Expr *pExpr){ + int rc = WRC_Continue; + struct CCurHint *pHint = pWalker->u.pCCurHint; + if( pExpr->op==TK_COLUMN ){ + if( pExpr->iTable!=pHint->iTabCur ){ + Vdbe *v = pWalker->pParse->pVdbe; + int reg = ++pWalker->pParse->nMem; /* Register for column value */ + sqlite3ExprCodeGetColumnOfTable( + v, pExpr->pTab, pExpr->iTable, pExpr->iColumn, reg + ); + pExpr->op = TK_REGISTER; + pExpr->iTable = reg; + }else if( pHint->pIdx!=0 ){ + pExpr->iTable = pHint->iIdxCur; + pExpr->iColumn = sqlite3ColumnOfIndex(pHint->pIdx, pExpr->iColumn); + assert( pExpr->iColumn>=0 ); + } + }else if( pExpr->op==TK_AGG_FUNCTION ){ + /* An aggregate function in the WHERE clause of a query means this must + ** be a correlated sub-query, and expression pExpr is an aggregate from + ** the parent context. Do not walk the function arguments in this case. + ** + ** todo: It should be possible to replace this node with a TK_REGISTER + ** expression, as the result of the expression must be stored in a + ** register at this point. The same holds for TK_AGG_COLUMN nodes. */ + rc = WRC_Prune; + } + return rc; +} + +/* +** Insert an OP_CursorHint instruction if it is appropriate to do so. +*/ +static void codeCursorHint( + WhereInfo *pWInfo, /* The where clause */ + WhereLevel *pLevel, /* Which loop to provide hints for */ + WhereTerm *pEndRange /* Hint this end-of-scan boundary term if not NULL */ +){ + Parse *pParse = pWInfo->pParse; + sqlite3 *db = pParse->db; + Vdbe *v = pParse->pVdbe; + Expr *pExpr = 0; + WhereLoop *pLoop = pLevel->pWLoop; + int iCur; + WhereClause *pWC; + WhereTerm *pTerm; + int i, j; + struct CCurHint sHint; + Walker sWalker; + + if( OptimizationDisabled(db, SQLITE_CursorHints) ) return; + iCur = pLevel->iTabCur; + assert( iCur==pWInfo->pTabList->a[pLevel->iFrom].iCursor ); + sHint.iTabCur = iCur; + sHint.iIdxCur = pLevel->iIdxCur; + sHint.pIdx = pLoop->u.btree.pIndex; + memset(&sWalker, 0, sizeof(sWalker)); + sWalker.pParse = pParse; + sWalker.u.pCCurHint = &sHint; + pWC = &pWInfo->sWC; + for(i=0; inTerm; i++){ + pTerm = &pWC->a[i]; + if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; + if( pTerm->prereqAll & pLevel->notReady ) continue; + if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) continue; + + /* All terms in pWLoop->aLTerm[] except pEndRange are used to initialize + ** the cursor. These terms are not needed as hints for a pure range + ** scan (that has no == terms) so omit them. */ + if( pLoop->u.btree.nEq==0 && pTerm!=pEndRange ){ + for(j=0; jnLTerm && pLoop->aLTerm[j]!=pTerm; j++){} + if( jnLTerm ) continue; + } + + /* No subqueries or non-deterministic functions allowed */ + if( sqlite3ExprContainsSubquery(pTerm->pExpr) ) continue; + + /* For an index scan, make sure referenced columns are actually in + ** the index. */ + if( sHint.pIdx!=0 ){ + sWalker.eCode = 0; + sWalker.xExprCallback = codeCursorHintCheckExpr; + sqlite3WalkExpr(&sWalker, pTerm->pExpr); + if( sWalker.eCode ) continue; + } + + /* If we survive all prior tests, that means this term is worth hinting */ + pExpr = sqlite3ExprAnd(db, pExpr, sqlite3ExprDup(db, pTerm->pExpr, 0)); + } + if( pExpr!=0 ){ + sWalker.xExprCallback = codeCursorHintFixExpr; + sqlite3WalkExpr(&sWalker, pExpr); + sqlite3VdbeAddOp4(v, OP_CursorHint, + (sHint.pIdx ? sHint.iIdxCur : sHint.iTabCur), 0, 0, + (const char*)pExpr, P4_EXPR); + } +} +#else +# define codeCursorHint(A,B,C) /* No-op */ +#endif /* SQLITE_ENABLE_CURSOR_HINTS */ /* ** Generate code for the start of the iLevel-th loop in the WHERE clause @@ -754,6 +902,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( pStart = pEnd; pEnd = pTerm; } + codeCursorHint(pWInfo, pLevel, pEnd); if( pStart ){ Expr *pX; /* The expression that defines the start bound */ int r1, rTemp; /* Registers for holding the start boundary */ @@ -947,15 +1096,6 @@ Bitmask sqlite3WhereCodeOneLoopStart( } assert( pRangeEnd==0 || (pRangeEnd->wtFlags & TERM_VNULL)==0 ); - /* Generate code to evaluate all constraint terms using == or IN - ** and store the values of those terms in an array of registers - ** starting at regBase. - */ - regBase = codeAllEqualityTerms(pParse,pLevel,bRev,nExtraReg,&zStartAff); - assert( zStartAff==0 || sqlite3Strlen30(zStartAff)>=nEq ); - if( zStartAff ) cEndAff = zStartAff[nEq]; - addrNxt = pLevel->addrNxt; - /* If we are doing a reverse order scan on an ascending index, or ** a forward order scan on a descending index, interchange the ** start and end terms (pRangeStart and pRangeEnd). @@ -967,6 +1107,16 @@ Bitmask sqlite3WhereCodeOneLoopStart( SWAP(u8, bSeekPastNull, bStopAtNull); } + /* Generate code to evaluate all constraint terms using == or IN + ** and store the values of those terms in an array of registers + ** starting at regBase. + */ + codeCursorHint(pWInfo, pLevel, pRangeEnd); + regBase = codeAllEqualityTerms(pParse,pLevel,bRev,nExtraReg,&zStartAff); + assert( zStartAff==0 || sqlite3Strlen30(zStartAff)>=nEq ); + if( zStartAff ) cEndAff = zStartAff[nEq]; + addrNxt = pLevel->addrNxt; + testcase( pRangeStart && (pRangeStart->eOperator & WO_LE)!=0 ); testcase( pRangeStart && (pRangeStart->eOperator & WO_GE)!=0 ); testcase( pRangeEnd && (pRangeEnd->eOperator & WO_LE)!=0 ); @@ -1249,7 +1399,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( pAndExpr = sqlite3ExprAnd(db, pAndExpr, pExpr); } if( pAndExpr ){ - pAndExpr = sqlite3PExpr(pParse, TK_AND, 0, pAndExpr, 0); + pAndExpr = sqlite3PExpr(pParse, TK_AND|TKFLG_DONTFOLD, 0, pAndExpr, 0); } } @@ -1405,6 +1555,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( ** a pseudo-cursor. No need to Rewind or Next such cursors. */ pLevel->op = OP_Noop; }else{ + codeCursorHint(pWInfo, pLevel, 0); pLevel->op = aStep[bRev]; pLevel->p1 = iCur; pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk); diff --git a/test/autoindex5.test b/test/autoindex5.test index 2d5fad2eda..649ae123a4 100644 --- a/test/autoindex5.test +++ b/test/autoindex5.test @@ -17,6 +17,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +set testprefix autoindex5 # Schema is from the Debian security database # @@ -103,6 +104,26 @@ do_execsql_test autoindex5-1.1 { OR sp.release = 'wheezy' OR sp.release = 'squeeze' ) ORDER BY sp.name, st.bug_name, sp.release, sp.subrelease; } {/SEARCH SUBQUERY 2 USING AUTOMATIC COVERING INDEX .bug_name=/} + +#------------------------------------------------------------------------- +# Test that ticket [8a2adec1] has been fixed. +# +do_execsql_test 2.1 { + CREATE TABLE one(o); + INSERT INTO one DEFAULT VALUES; + + CREATE TABLE t1(x, z); + INSERT INTO t1 VALUES('aaa', 4.0); + INSERT INTO t1 VALUES('aaa', 4.0); + CREATE VIEW vvv AS + SELECT * FROM t1 + UNION ALL + SELECT 0, 0 WHERE 0; + + SELECT ( + SELECT sum(z) FROM vvv WHERE x='aaa' + ) FROM one; +} {8.0} finish_test diff --git a/test/cacheflush.test b/test/cacheflush.test new file mode 100644 index 0000000000..007ee8a065 --- /dev/null +++ b/test/cacheflush.test @@ -0,0 +1,324 @@ +# 2011 November 16 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains test cases for sqlite3_db_cacheflush API. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix cacheflush +test_set_config_pagecache 0 0 + +# Run the supplied SQL on a copy of the database currently stored on +# disk in file $dbfile. +proc diskquery {dbfile sql} { + forcecopy $dbfile dq.db + sqlite3 dq dq.db + set res [execsql $sql dq] + dq close + set res +} + +# Simplest possible test. +# +do_execsql_test 1.1.0 { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + BEGIN; + INSERT INTO t1 VALUES(3, 4); +} +do_test 1.1.1 { + diskquery test.db { SELECT * FROM t1 } +} {1 2} +do_test 1.1.2 { + sqlite3_db_cacheflush db + diskquery test.db { SELECT * FROM t1 } +} {1 2 3 4} + +# Test that multiple pages may be flushed to disk. +# +do_execsql_test 1.2.0 { + COMMIT; + CREATE TABLE t2(a, b); + BEGIN; + INSERT INTO t1 VALUES(5, 6); + INSERT INTO t2 VALUES('a', 'b'); +} +do_test 1.2.1 { + diskquery test.db { + SELECT * FROM t1; + SELECT * FROM t2; + } +} {1 2 3 4} +do_test 1.2.2 { + sqlite3_db_cacheflush db + diskquery test.db { + SELECT * FROM t1; + SELECT * FROM t2; + } +} {1 2 3 4 5 6 a b} + +# Test that pages with nRef!=0 are not flushed to disk. +# +do_execsql_test 1.3.0 { + COMMIT; + CREATE TABLE t3(a, b); + BEGIN; + INSERT INTO t1 VALUES(7, 8); + INSERT INTO t2 VALUES('c', 'd'); + INSERT INTO t3 VALUES('i', 'ii'); +} +do_test 1.3.1 { + diskquery test.db { + SELECT * FROM t1; + SELECT * FROM t2; + SELECT * FROM t3; + } +} {1 2 3 4 5 6 a b} +do_test 1.3.2 { + db eval { SELECT a FROM t1 } { + if {$a==3} { + sqlite3_db_cacheflush db + } + } + diskquery test.db { + SELECT * FROM t1; + SELECT * FROM t2; + SELECT * FROM t3; + } +} {1 2 3 4 5 6 a b c d i ii} +do_test 1.3.2 { + sqlite3_db_cacheflush db + diskquery test.db { + SELECT * FROM t1; + SELECT * FROM t2; + SELECT * FROM t3; + } +} {1 2 3 4 5 6 7 8 a b c d i ii} + +# Check that SQLITE_BUSY is returned if pages cannot be flushed due to +# conflicting read locks. +# +do_execsql_test 1.4.0 { + COMMIT; + BEGIN; + INSERT INTO t1 VALUES(9, 10); +} +do_test 1.4.1 { + sqlite3 db2 test.db + db2 eval { + BEGIN; + SELECT * FROM t1; + } + diskquery test.db { + SELECT * FROM t1; + } +} {1 2 3 4 5 6 7 8} +do_test 1.4.2 { + list [catch { sqlite3_db_cacheflush db } msg] $msg +} {1 {database is locked}} +do_test 1.4.3 { + diskquery test.db { + SELECT * FROM t1; + } +} {1 2 3 4 5 6 7 8} +do_test 1.4.4 { + db2 close + sqlite3_db_cacheflush db + diskquery test.db { + SELECT * FROM t1; + } +} {1 2 3 4 5 6 7 8 9 10} +do_execsql_test 1.4.5 { COMMIT } + +#------------------------------------------------------------------------- +# Test that ATTACHed database caches are also flushed. +# +forcedelete test.db2 +do_execsql_test 2.1.0 { + ATTACH 'test.db2' AS aux; + CREATE TABLE aux.t4(x, y); + INSERT INTO t4 VALUES('A', 'B'); + BEGIN; + INSERT INTO t1 VALUES(11, 12); + INSERT INTO t4 VALUES('C', 'D'); +} +do_test 2.1.1 { + diskquery test.db { SELECT * FROM t1; } +} {1 2 3 4 5 6 7 8 9 10} +do_test 2.1.2 { + diskquery test.db2 { SELECT * FROM t4; } +} {A B} +do_test 2.1.3 { + sqlite3_db_cacheflush db + diskquery test.db { SELECT * FROM t1; } +} {1 2 3 4 5 6 7 8 9 10 11 12} +do_test 2.1.4 { + sqlite3_db_cacheflush db + diskquery test.db2 { SELECT * FROM t4; } +} {A B C D} +do_execsql_test 2.1.5 { COMMIT } + +# And that hitting an SQLITE_BUSY when flushing "main" does not stop +# SQLite from going on to flush "aux". +# +do_execsql_test 2.2.0 { + BEGIN; + INSERT INTO t1 VALUES(13, 14); + INSERT INTO t4 VALUES('E', 'F'); +} +do_test 2.2.1 { + diskquery test.db { SELECT * FROM t1; } +} {1 2 3 4 5 6 7 8 9 10 11 12} +do_test 2.2.2 { + diskquery test.db2 { SELECT * FROM t4; } +} {A B C D} +do_test 2.2.3 { + sqlite3 db2 test.db + execsql { + BEGIN; + SELECT * FROM t1; + } db2 + list [catch { sqlite3_db_cacheflush db } msg] $msg +} {1 {database is locked}} +do_test 2.2.4 { + diskquery test.db { SELECT * FROM t1; } +} {1 2 3 4 5 6 7 8 9 10 11 12} +do_test 2.2.5 { + diskquery test.db2 { SELECT * FROM t4; } +} {A B C D E F} +do_test 2.2.6 { + db2 close + sqlite3_db_cacheflush db + diskquery test.db { SELECT * FROM t1; } +} {1 2 3 4 5 6 7 8 9 10 11 12 13 14} +do_execsql_test 2.2.7 { COMMIT } + +#------------------------------------------------------------------------- +# Test that nothing terrible happens if sqlite3_db_cacheflush() is +# called on an in-memory database. +# +do_test 3.0 { + db close + sqlite3 db :memory: + db eval { + CREATE TABLE t1(x PRIMARY KEY); + CREATE TABLE t2(y PRIMARY KEY); + BEGIN; + INSERT INTO t1 VALUES(randomblob(100)); + INSERT INTO t2 VALUES(randomblob(100)); + INSERT INTO t1 VALUES(randomblob(100)); + INSERT INTO t2 VALUES(randomblob(100)); + } + sqlite3_db_cacheflush db +} {} + +do_execsql_test 3.1 { PRAGMA integrity_check } ok +do_execsql_test 3.2 { COMMIT } +do_execsql_test 3.3 { PRAGMA integrity_check } ok +do_execsql_test 3.4 { + SELECT count(*) FROM t1; + SELECT count(*) FROM t2; +} {2 2} + +#------------------------------------------------------------------------- +# Test that calling sqlite3_db_cacheflush() does not interfere with +# savepoint transactions. +# +do_test 4.0 { + reset_db + execsql { + CREATE TABLE ta(a, aa); + CREATE TABLE tb(b, bb); + INSERT INTO ta VALUES('a', randomblob(500)); + INSERT INTO tb VALUES('b', randomblob(500)); + BEGIN; + UPDATE ta SET a = 'A'; + SAVEPOINT one; + UPDATE tb SET b = 'B'; + } + + sqlite3_db_cacheflush db + diskquery test.db { + SELECT a FROM ta; + SELECT b FROM tb; + } +} {A B} + +do_test 4.1 { + execsql { + ROLLBACK TO one; + } + sqlite3_db_cacheflush db + diskquery test.db { + SELECT a FROM ta; + SELECT b FROM tb; + } +} {A b} + +do_test 4.2 { + execsql { + INSERT INTO tb VALUES('c', randomblob(10)); + INSERT INTO tb VALUES('d', randomblob(10)); + INSERT INTO tb VALUES('e', randomblob(10)); + } + sqlite3_db_cacheflush db + diskquery test.db { + SELECT a FROM ta; + SELECT b FROM tb; + } +} {A b c d e} + +do_test 4.3 { + execsql { + SAVEPOINT two; + UPDATE tb SET b = upper(b); + } + sqlite3_db_cacheflush db + diskquery test.db { + SELECT a FROM ta; + SELECT b FROM tb; + } +} {A B C D E} + +do_test 4.4 { + execsql { + ROLLBACK TO two; + } + sqlite3_db_cacheflush db + diskquery test.db { + SELECT a FROM ta; + SELECT b FROM tb; + } +} {A b c d e} + +do_test 4.4 { + execsql { + ROLLBACK TO one; + } + sqlite3_db_cacheflush db + diskquery test.db { + SELECT a FROM ta; + SELECT b FROM tb; + } +} {A b} + +do_test 4.5 { + execsql { + ROLLBACK; + SELECT a FROM ta; + SELECT b FROM tb; + } +} {a b} + +test_restore_config_pagecache +finish_test + diff --git a/test/cffault.test b/test/cffault.test new file mode 100644 index 0000000000..a89477b617 --- /dev/null +++ b/test/cffault.test @@ -0,0 +1,159 @@ +# 2011 November 16 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains fault-injection test cases for the +# sqlite3_db_cacheflush API. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix cacheflush +source $testdir/malloc_common.tcl + +# Run the supplied SQL on a copy of the database currently stored on +# disk in file $dbfile. +proc diskquery {dbfile sql} { + forcecopy $dbfile dq.db + sqlite3 dq dq.db + set res [execsql $sql dq] + dq close + set res +} + +do_execsql_test 1.0 { + CREATE TABLE t1(a PRIMARY KEY, b); + CREATE INDEX i1 ON t1(b); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t1 VALUES(5, 6); + INSERT INTO t1 VALUES(7, 8); +} +faultsim_save_and_close + +do_faultsim_test 1.1 -prep { + faultsim_restore_and_reopen + db eval { + BEGIN; + UPDATE t1 SET b=b+1; + } +} -body { + sqlite3_db_cacheflush db +} -test { + if {[sqlite3_get_autocommit db]} { error "Transaction rolled back!" } + faultsim_test_result {0 {}} {1 {disk I/O error}} + catch { db eval COMMIT } + faultsim_integrity_check +} + +do_faultsim_test 1.2 -prep { + faultsim_restore_and_reopen + db eval { + BEGIN; + UPDATE t1 SET b=b+1; + } +} -body { + set result [list] + db eval { SELECT * FROM t1 } { + if {$a==5} { catch { sqlite3_db_cacheflush db } } + lappend result $a $b + } + set result +} -test { + faultsim_test_result {0 {1 3 3 5 5 7 7 9}} {1 {disk I/O error}} + catch { db eval COMMIT } + faultsim_integrity_check +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 2.0 { + CREATE TABLE t1(a PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + CREATE INDEX i2 ON t1(c, b); + INSERT INTO t1 VALUES(1, 2, randomblob(600)); + INSERT INTO t1 VALUES(3, 4, randomblob(600)); + INSERT INTO t1 VALUES(5, 6, randomblob(600)); + INSERT INTO t1 VALUES(7, 8, randomblob(600)); + INSERT INTO t1 VALUES(9, 10, randomblob(600)); +} +faultsim_save_and_close + +do_faultsim_test 2.1 -prep { + faultsim_restore_and_reopen + db eval { + BEGIN; + UPDATE t1 SET b=b+1; + } +} -body { + set result [list] + db eval { SELECT * FROM t1 } { + if {$a==5} { catch { sqlite3_db_cacheflush db } } + lappend result $a $b + } + set result +} -test { + faultsim_test_result {0 {1 3 3 5 5 7 7 9 9 11}} {1 {disk I/O error}} + catch { db eval { INSERT INTO t1 VALUES(11, 12, randomblob(600)) } } + catch { db eval COMMIT } + faultsim_integrity_check +} + +do_faultsim_test 2.2 -prep { + faultsim_restore_and_reopen + db eval { + BEGIN; + UPDATE t1 SET b=b+1; + } +} -body { + sqlite3_db_cacheflush db +} -test { + if {[sqlite3_get_autocommit db]} { error "Transaction rolled back!" } + faultsim_test_result {0 {}} {1 {disk I/O error}} + catch { db eval { SELECT * FROM t1 } } + catch { db eval COMMIT } + faultsim_integrity_check +} + +do_faultsim_test 2.3 -prep { + faultsim_restore_and_reopen + db eval { + BEGIN; + UPDATE t1 SET b=b-1; + } +} -body { + sqlite3_db_cacheflush db +} -test { + if {[sqlite3_get_autocommit db]} { error "Transaction rolled back!" } + faultsim_test_result {0 {}} {1 {disk I/O error}} + catch { db eval { INSERT INTO t1 VALUES(11, 12, randomblob(600)) } } + catch { db eval COMMIT } + faultsim_integrity_check +} + +do_faultsim_test 2.4 -prep { + faultsim_restore_and_reopen + db eval { + BEGIN; + UPDATE t1 SET b=b-1; + } +} -body { + catch { sqlite3_db_cacheflush db } + catch { sqlite3_db_release_memory db } + catch { sqlite3_db_cacheflush db } + execsql { SELECT a, b FROM t1 } +} -test { + faultsim_test_result {0 {1 1 3 3 5 5 7 7 9 9}} {1 {disk I/O error}} + catchsql ROLLBACK + faultsim_integrity_check +} + +finish_test + diff --git a/test/cursorhint.test b/test/cursorhint.test new file mode 100644 index 0000000000..69bc248cd7 --- /dev/null +++ b/test/cursorhint.test @@ -0,0 +1,162 @@ +# 2014 July 15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix cursorhint + +ifcapable !cursorhints { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE TABLE t1(a,b,c,d); + CREATE TABLE t2(x,y,z); + INSERT INTO t1(a,b) VALUES(10, 15); + INSERT INTO t1(a,b) VALUES(20, 25); + INSERT INTO t2(x,y) VALUES('ten', 'fifteen'); + INSERT INTO t2(x,y) VALUES('twenty', 'twentyfive'); + CREATE TABLE t3(id TEXT PRIMARY KEY, a, b, c, d) WITHOUT ROWID; + INSERT INTO t3(id,a,b,c,d) SELECT rowid, a, b, c, d FROM t1; + PRAGMA automatic_index = 0; +} + +# Run EXPLAIN on $sql. Return a list of P4 values for all $opcode +# opcodes. +# +proc p4_of_opcode {db opcode sql} { + set res {} + $db eval "EXPLAIN $sql" x { + if {$x(opcode)==$opcode} {lappend res $x(p4)} + } + return $res +} + +# Run EXPLAIN on $sql. Return a list of P5 values for all $opcode +# opcodes that contain regexp $comment in their comment +# +proc p5_of_opcode {db opcode comment sql} { + set res {} + $db eval "EXPLAIN $sql" x { + if {$x(opcode)==$opcode && [regexp $comment $x(comment)]} { + lappend res $x(p5) + } + } + return $res +} + +# Verify that when t1 is in the outer loop and t2 is in the inner loop, +# no cursor hints occur for t1 (since it is a full table scan) but that +# each t2 access has a cursor hint based on the current t1.a value. +# +do_test 1.1 { + p4_of_opcode db CursorHint { + SELECT * FROM t1 CROSS JOIN t2 WHERE a=x + } +} {{EQ(r[1],c0)}} +do_test 1.2 { + p5_of_opcode db OpenRead . { + SELECT * FROM t1 CROSS JOIN t2 WHERE a=x + } +} {00 00} + +# Do the same test the other way around. +# +do_test 2.1 { + p4_of_opcode db CursorHint { + SELECT * FROM t2 CROSS JOIN t1 WHERE a=x + } +} {{EQ(c0,r[1])}} +do_test 2.2 { + p5_of_opcode db OpenRead . { + SELECT * FROM t2 CROSS JOIN t1 WHERE a=x + } +} {00 00} + +# Various expressions captured by CursorHint +# +do_test 3.1 { + p4_of_opcode db CursorHint { + SELECT * FROM t1 WHERE a=15 AND c=22 AND rowid!=98 + } +} {AND(AND(EQ(c0,15),EQ(c2,22)),NE(rowid,98))} +do_test 3.2 { + p4_of_opcode db CursorHint { + SELECT * FROM t3 WHERE a<15 AND b>22 AND id!=98 + } +} {AND(AND(LT(c1,15),GT(c2,22)),NE(c0,98))} + +# Indexed queries +# +do_test 4.1asc { + db eval { + CREATE INDEX t1bc ON t1(b,c); + CREATE INDEX t2yz ON t2(y,z); + } + p4_of_opcode db CursorHint { + SELECT * FROM t1 WHERE b>11 ORDER BY b ASC; + } +} {} +do_test 4.1desc { + p4_of_opcode db CursorHint { + SELECT * FROM t1 WHERE b>11 ORDER BY b DESC; + } +} {GT(c0,11)} +do_test 4.2 { + p5_of_opcode db OpenRead . { + SELECT * FROM t1 WHERE b>11; + } +} {02 00} +do_test 4.3asc { + p4_of_opcode db CursorHint { + SELECT c FROM t1 WHERE b<11 ORDER BY b ASC; + } +} {LT(c0,11)} +do_test 4.3desc { + p4_of_opcode db CursorHint { + SELECT c FROM t1 WHERE b<11 ORDER BY b DESC; + } +} {} +do_test 4.4 { + p5_of_opcode db OpenRead . { + SELECT c FROM t1 WHERE b<11; + } +} {00} + +do_test 4.5asc { + p4_of_opcode db CursorHint { + SELECT c FROM t1 WHERE b>=10 AND b<=20 ORDER BY b ASC; + } +} {LE(c0,20)} +do_test 4.5desc { + p4_of_opcode db CursorHint { + SELECT c FROM t1 WHERE b>=10 AND b<=20 ORDER BY b DESC; + } +} {GE(c0,10)} + +# If there are any equality terms used in the constraint, then all terms +# should be hinted. +# +do_test 4.6asc { + p4_of_opcode db CursorHint { + SELECT rowid FROM t1 WHERE b=22 AND c>=10 AND c<=20 ORDER BY b,c ASC; + } +} {AND(AND(EQ(c0,22),GE(c1,10)),LE(c1,20))} +do_test 4.6desc { + p4_of_opcode db CursorHint { + SELECT rowid FROM t1 WHERE b=22 AND c>=10 AND c<=20 ORDER BY b,c DESC; + } +} {AND(AND(EQ(c0,22),GE(c1,10)),LE(c1,20))} + +finish_test diff --git a/test/fordelete.test b/test/fordelete.test new file mode 100644 index 0000000000..1e860e8867 --- /dev/null +++ b/test/fordelete.test @@ -0,0 +1,130 @@ +# 2001 September 15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the SELECT statement. +# +# $Id: select1.test,v 1.70 2009/05/28 01:00:56 drh Exp $ + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix fordelete + +# This function returns a list of the tables or indexes opened with +# OP_OpenWrite instructions when the SQL statement passed as the only +# argument is executed. If the OPFLAG_FORDELETE flag is specified on +# the OP_OpenWrite, an asterix is appended to the object name. The list +# is sorted in [lsort] order before it is returned. +# +proc analyze_delete_program {sql} { + # Build a map from root page to table/index name. + db eval { + SELECT name, rootpage FROM sqlite_master + } { + set T($rootpage) $name + } + + # Calculate the results. + set res [list] + db eval "EXPLAIN $sql" R { + if {$R(opcode) == "OpenWrite"} { + set obj $T($R(p2)) + if {"0x$R(p5)" & 0x08} { append obj *} + lappend res $obj + } + } + + lsort $res +} + +proc do_adp_test {tn sql res} { + uplevel [list do_test $tn [list analyze_delete_program $sql] [list {*}$res]] +} + +do_execsql_test 1.0 { + CREATE TABLE t1(a PRIMARY KEY, b); +} + +foreach {tn sql res} { + 1 { DELETE FROM t1 WHERE a=?} { sqlite_autoindex_t1_1 t1* } + 2 { DELETE FROM t1 WHERE a=? AND b=? } { sqlite_autoindex_t1_1 t1 } + 3 { DELETE FROM t1 WHERE a>? } { sqlite_autoindex_t1_1 t1* } + 4 { DELETE FROM t1 WHERE rowid=? } { sqlite_autoindex_t1_1* t1 } +} { + do_adp_test 1.$tn $sql $res +} + +do_execsql_test 2.0 { + CREATE TABLE t2(a, b, c); + CREATE INDEX t2a ON t2(a); + CREATE INDEX t2b ON t2(b); + CREATE INDEX t2c ON t2(c); +} +foreach {tn sql res} { + 1 { DELETE FROM t2 WHERE a=?} { t2* t2a t2b* t2c* } + 2 { DELETE FROM t2 WHERE a=? AND +b=?} { t2 t2a t2b* t2c* } + 3 { DELETE FROM t2 WHERE a=? OR b=?} { t2 t2a* t2b* t2c* } + 4 { DELETE FROM t2 WHERE +a=? } { t2 t2a* t2b* t2c* } + 5 { DELETE FROM t2 WHERE rowid=? } { t2 t2a* t2b* t2c* } +} { + do_adp_test 2.$tn $sql $res +} + +#------------------------------------------------------------------------- +# Test that a record that consists of the bytes: +# +# 0x01 0x00 +# +# is interpreted by OP_Column as a vector of NULL values (assuming the +# default column values are NULL). Also test that: +# +# 0x00 +# +# is handled in the same way. +# +do_execsql_test 3.0 { + CREATE TABLE x1(a INTEGER PRIMARY KEY, b, c, d); + CREATE TABLE x2(a INTEGER PRIMARY KEY, b, c, d); +} + +do_test 3.1 { + set root [db one { SELECT rootpage FROM sqlite_master WHERE name = 'x1' }] + db eval { + BEGIN IMMEDIATE; + } + set bt [btree_from_db db] + set csr [btree_cursor $bt $root 1] + btree_insert $csr 5 "\000" + btree_close_cursor $csr + db eval { COMMIT } + + db eval { + SELECT * FROM x1; + } +} {5 {} {} {}} + +do_test 3.2 { + set root [db one { SELECT rootpage FROM sqlite_master WHERE name = 'x2' }] + db eval { + BEGIN IMMEDIATE; + } + set bt [btree_from_db db] + set csr [btree_cursor $bt $root 1] + btree_insert $csr 6 "\000" + btree_close_cursor $csr + db eval { COMMIT } + + db eval { + SELECT * FROM x2; + } +} {6 {} {} {}} + +finish_test + diff --git a/test/fuzzcheck.c b/test/fuzzcheck.c index 15ddd09eed..19995684e4 100644 --- a/test/fuzzcheck.c +++ b/test/fuzzcheck.c @@ -70,6 +70,9 @@ #include #include #include "sqlite3.h" +#define ISSPACE(X) isspace((unsigned char)(X)) +#define ISDIGIT(X) isdigit((unsigned char)(X)) + #ifdef __unix__ # include @@ -633,9 +636,9 @@ static void runSql(sqlite3 *db, const char *zSql, unsigned runFlags){ if( runFlags & SQL_TRACE ){ const char *z = zSql; int n; - while( z0 && isspace(z[n-1]) ) n--; + while( n>0 && ISSPACE(z[n-1]) ) n--; if( n==0 ) break; if( pStmt==0 ){ printf("TRACE: %.*s (error: %s)\n", n, z, sqlite3_errmsg(db)); @@ -757,7 +760,7 @@ static int integerValue(const char *zArg){ zArg++; } }else{ - while( isdigit(zArg[0]) ){ + while( ISDIGIT(zArg[0]) ){ v = v*10 + zArg[0] - '0'; zArg++; } diff --git a/test/speedtest1.c b/test/speedtest1.c index b34dae65ae..b41c732053 100644 --- a/test/speedtest1.c +++ b/test/speedtest1.c @@ -47,6 +47,8 @@ static const char zHelp[] = #include #include #include +#define ISSPACE(X) isspace((unsigned char)(X)) +#define ISDIGIT(X) isdigit((unsigned char)(X)) #if SQLITE_VERSION_NUMBER<3005000 # define sqlite3_int64 sqlite_int64 @@ -315,7 +317,7 @@ void speedtest1_final(void){ /* Print an SQL statement to standard output */ static void printSql(const char *zSql){ int n = (int)strlen(zSql); - while( n>0 && (zSql[n-1]==';' || isspace(zSql[n-1])) ){ n--; } + while( n>0 && (zSql[n-1]==';' || ISSPACE(zSql[n-1])) ){ n--; } if( g.bExplain ) printf("EXPLAIN "); printf("%.*s;\n", n, zSql); if( g.bExplain @@ -414,7 +416,7 @@ void speedtest1_run(void){ /* The sqlite3_trace() callback function */ static void traceCallback(void *NotUsed, const char *zSql){ int n = (int)strlen(zSql); - while( n>0 && (zSql[n-1]==';' || isspace(zSql[n-1])) ) n--; + while( n>0 && (zSql[n-1]==';' || ISSPACE(zSql[n-1])) ) n--; fprintf(stderr,"%.*s;\n", n, zSql); } diff --git a/test/wordcount.c b/test/wordcount.c index cf63e983c2..72aa6b2f0b 100644 --- a/test/wordcount.c +++ b/test/wordcount.c @@ -80,6 +80,7 @@ #include #include #include "sqlite3.h" +#define ISALPHA(X) isalpha((unsigned char)(X)) /* Return the current wall-clock time */ static sqlite3_int64 realTime(void){ @@ -392,8 +393,8 @@ int main(int argc, char **argv){ /* Process the input file */ while( fgets(zInput, sizeof(zInput), in) ){ for(i=0; zInput[i]; i++){ - if( !isalpha(zInput[i]) ) continue; - for(j=i+1; isalpha(zInput[j]); j++){} + if( !ISALPHA(zInput[i]) ) continue; + for(j=i+1; ISALPHA(zInput[j]); j++){} /* Found a new word at zInput[i] that is j-i bytes long. ** Process it into the wordcount table. */ diff --git a/tool/addopcodes.tcl b/tool/addopcodes.tcl index 757eb69938..7341a810ec 100644 --- a/tool/addopcodes.tcl +++ b/tool/addopcodes.tcl @@ -45,3 +45,15 @@ foreach x $extras { incr max puts [format "#define TK_%-29s %4d" $x $max] } + +# Some additional #defines related to token codes. +# +puts "\n/* The token codes above must all fit in 8 bits */" +puts [format "#define %-20s %-6s" TKFLG_MASK 0xff] +puts "\n/* Flags that can be added to a token code when it is not" +puts "** being stored in a u8: */" +foreach {fg val comment} { + TKFLG_DONTFOLD 0x100 {/* Omit constant folding optimizations */} +} { + puts [format "#define %-20s %-6s %s" $fg $val $comment] +} diff --git a/tool/build-all-msvc.bat b/tool/build-all-msvc.bat index a8ba1bc293..e081c73b38 100755 --- a/tool/build-all-msvc.bat +++ b/tool/build-all-msvc.bat @@ -95,6 +95,10 @@ REM example: REM REM SET NMAKE_ARGS=FOR_WINRT=1 REM +REM Using the above command before running this tool will cause the compiled +REM binaries to target the WinRT environment, which provides a subset of the +REM Win32 API. +REM SETLOCAL REM SET __ECHO=ECHO diff --git a/tool/fuzzershell.c b/tool/fuzzershell.c index 6754a071e3..2c778bce6e 100644 --- a/tool/fuzzershell.c +++ b/tool/fuzzershell.c @@ -67,6 +67,7 @@ #include #include #include "sqlite3.h" +#define ISDIGIT(X) isdigit((unsigned char)(X)) /* ** All global variables are gathered into the "g" singleton. @@ -383,7 +384,7 @@ static int integerValue(const char *zArg){ zArg++; } }else{ - while( isdigit(zArg[0]) ){ + while( ISDIGIT(zArg[0]) ){ v = v*10 + zArg[0] - '0'; zArg++; } diff --git a/tool/lemon.c b/tool/lemon.c index 2e8054b5cc..d704deb624 100644 --- a/tool/lemon.c +++ b/tool/lemon.c @@ -13,6 +13,14 @@ #include #include +#define ISSPACE(X) isspace((unsigned char)(X)) +#define ISDIGIT(X) isdigit((unsigned char)(X)) +#define ISALNUM(X) isalnum((unsigned char)(X)) +#define ISALPHA(X) isalpha((unsigned char)(X)) +#define ISUPPER(X) isupper((unsigned char)(X)) +#define ISLOWER(X) islower((unsigned char)(X)) + + #ifndef __WIN32__ # if defined(_WIN32) || defined(WIN32) # define __WIN32__ @@ -93,9 +101,9 @@ static int lemon_vsprintf(char *str, const char *zFormat, va_list ap){ int iWidth = 0; lemon_addtext(str, &nUsed, &zFormat[j], i-j, 0); c = zFormat[++i]; - if( isdigit(c) || (c=='-' && isdigit(zFormat[i+1])) ){ + if( ISDIGIT(c) || (c=='-' && ISDIGIT(zFormat[i+1])) ){ if( c=='-' ) i++; - while( isdigit(zFormat[i]) ) iWidth = iWidth*10 + zFormat[i++] - '0'; + while( ISDIGIT(zFormat[i]) ) iWidth = iWidth*10 + zFormat[i++] - '0'; if( c=='-' ) iWidth = -iWidth; c = zFormat[i]; } @@ -1578,7 +1586,7 @@ int main(int argc, char **argv) while( lem.symbols[i-1]->type==MULTITERMINAL ){ i--; } assert( strcmp(lem.symbols[i-1]->name,"{default}")==0 ); lem.nsymbol = i - 1; - for(i=1; isupper(lem.symbols[i]->name[0]); i++); + for(i=1; ISUPPER(lem.symbols[i]->name[0]); i++); lem.nterminal = i; /* Generate a reprint of the grammar, if requested on the command line */ @@ -2121,7 +2129,7 @@ static void parseonetoken(struct pstate *psp) case WAITING_FOR_DECL_OR_RULE: if( x[0]=='%' ){ psp->state = WAITING_FOR_DECL_KEYWORD; - }else if( islower(x[0]) ){ + }else if( ISLOWER(x[0]) ){ psp->lhs = Symbol_new(x); psp->nrhs = 0; psp->lhsalias = 0; @@ -2151,7 +2159,7 @@ to follow the previous rule."); } break; case PRECEDENCE_MARK_1: - if( !isupper(x[0]) ){ + if( !ISUPPER(x[0]) ){ ErrorMsg(psp->filename,psp->tokenlineno, "The precedence symbol must be a terminal."); psp->errorcnt++; @@ -2191,7 +2199,7 @@ to follow the previous rule."); } break; case LHS_ALIAS_1: - if( isalpha(x[0]) ){ + if( ISALPHA(x[0]) ){ psp->lhsalias = x; psp->state = LHS_ALIAS_2; }else{ @@ -2260,7 +2268,7 @@ to follow the previous rule."); psp->prevrule = rp; } psp->state = WAITING_FOR_DECL_OR_RULE; - }else if( isalpha(x[0]) ){ + }else if( ISALPHA(x[0]) ){ if( psp->nrhs>=MAXRHS ){ ErrorMsg(psp->filename,psp->tokenlineno, "Too many symbols on RHS of rule beginning at \"%s\".", @@ -2289,7 +2297,7 @@ to follow the previous rule."); msp->subsym = (struct symbol **) realloc(msp->subsym, sizeof(struct symbol*)*msp->nsubsym); msp->subsym[msp->nsubsym-1] = Symbol_new(&x[1]); - if( islower(x[1]) || islower(msp->subsym[0]->name[0]) ){ + if( ISLOWER(x[1]) || ISLOWER(msp->subsym[0]->name[0]) ){ ErrorMsg(psp->filename,psp->tokenlineno, "Cannot form a compound containing a non-terminal"); psp->errorcnt++; @@ -2304,7 +2312,7 @@ to follow the previous rule."); } break; case RHS_ALIAS_1: - if( isalpha(x[0]) ){ + if( ISALPHA(x[0]) ){ psp->alias[psp->nrhs-1] = x; psp->state = RHS_ALIAS_2; }else{ @@ -2326,7 +2334,7 @@ to follow the previous rule."); } break; case WAITING_FOR_DECL_KEYWORD: - if( isalpha(x[0]) ){ + if( ISALPHA(x[0]) ){ psp->declkeyword = x; psp->declargslot = 0; psp->decllinenoslot = 0; @@ -2406,7 +2414,7 @@ to follow the previous rule."); } break; case WAITING_FOR_DESTRUCTOR_SYMBOL: - if( !isalpha(x[0]) ){ + if( !ISALPHA(x[0]) ){ ErrorMsg(psp->filename,psp->tokenlineno, "Symbol name missing after %%destructor keyword"); psp->errorcnt++; @@ -2420,7 +2428,7 @@ to follow the previous rule."); } break; case WAITING_FOR_DATATYPE_SYMBOL: - if( !isalpha(x[0]) ){ + if( !ISALPHA(x[0]) ){ ErrorMsg(psp->filename,psp->tokenlineno, "Symbol name missing after %%type keyword"); psp->errorcnt++; @@ -2445,7 +2453,7 @@ to follow the previous rule."); case WAITING_FOR_PRECEDENCE_SYMBOL: if( x[0]=='.' ){ psp->state = WAITING_FOR_DECL_OR_RULE; - }else if( isupper(x[0]) ){ + }else if( ISUPPER(x[0]) ){ struct symbol *sp; sp = Symbol_new(x); if( sp->prec>=0 ){ @@ -2463,7 +2471,7 @@ to follow the previous rule."); } break; case WAITING_FOR_DECL_ARG: - if( x[0]=='{' || x[0]=='\"' || isalnum(x[0]) ){ + if( x[0]=='{' || x[0]=='\"' || ISALNUM(x[0]) ){ const char *zOld, *zNew; char *zBuf, *z; int nOld, n, nLine = 0, nNew, nBack; @@ -2524,7 +2532,7 @@ to follow the previous rule."); case WAITING_FOR_FALLBACK_ID: if( x[0]=='.' ){ psp->state = WAITING_FOR_DECL_OR_RULE; - }else if( !isupper(x[0]) ){ + }else if( !ISUPPER(x[0]) ){ ErrorMsg(psp->filename, psp->tokenlineno, "%%fallback argument \"%s\" should be a token", x); psp->errorcnt++; @@ -2545,7 +2553,7 @@ to follow the previous rule."); case WAITING_FOR_WILDCARD_ID: if( x[0]=='.' ){ psp->state = WAITING_FOR_DECL_OR_RULE; - }else if( !isupper(x[0]) ){ + }else if( !ISUPPER(x[0]) ){ ErrorMsg(psp->filename, psp->tokenlineno, "%%wildcard argument \"%s\" should be a token", x); psp->errorcnt++; @@ -2561,7 +2569,7 @@ to follow the previous rule."); } break; case WAITING_FOR_CLASS_ID: - if( !islower(x[0]) ){ + if( !ISLOWER(x[0]) ){ ErrorMsg(psp->filename, psp->tokenlineno, "%%token_class must be followed by an identifier: ", x); psp->errorcnt++; @@ -2580,12 +2588,12 @@ to follow the previous rule."); case WAITING_FOR_CLASS_TOKEN: if( x[0]=='.' ){ psp->state = WAITING_FOR_DECL_OR_RULE; - }else if( isupper(x[0]) || ((x[0]=='|' || x[0]=='/') && isupper(x[1])) ){ + }else if( ISUPPER(x[0]) || ((x[0]=='|' || x[0]=='/') && ISUPPER(x[1])) ){ struct symbol *msp = psp->tkclass; msp->nsubsym++; msp->subsym = (struct symbol **) realloc(msp->subsym, sizeof(struct symbol*)*msp->nsubsym); - if( !isupper(x[0]) ) x++; + if( !ISUPPER(x[0]) ) x++; msp->subsym[msp->nsubsym-1] = Symbol_new(x); }else{ ErrorMsg(psp->filename, psp->tokenlineno, @@ -2618,7 +2626,7 @@ static void preprocess_input(char *z){ for(i=0; z[i]; i++){ if( z[i]=='\n' ) lineno++; if( z[i]!='%' || (i>0 && z[i-1]!='\n') ) continue; - if( strncmp(&z[i],"%endif",6)==0 && isspace(z[i+6]) ){ + if( strncmp(&z[i],"%endif",6)==0 && ISSPACE(z[i+6]) ){ if( exclude ){ exclude--; if( exclude==0 ){ @@ -2626,13 +2634,13 @@ static void preprocess_input(char *z){ } } for(j=i; z[j] && z[j]!='\n'; j++) z[j] = ' '; - }else if( (strncmp(&z[i],"%ifdef",6)==0 && isspace(z[i+6])) - || (strncmp(&z[i],"%ifndef",7)==0 && isspace(z[i+7])) ){ + }else if( (strncmp(&z[i],"%ifdef",6)==0 && ISSPACE(z[i+6])) + || (strncmp(&z[i],"%ifndef",7)==0 && ISSPACE(z[i+7])) ){ if( exclude ){ exclude++; }else{ - for(j=i+7; isspace(z[j]); j++){} - for(n=0; z[j+n] && !isspace(z[j+n]); n++){} + for(j=i+7; ISSPACE(z[j]); j++){} + for(n=0; z[j+n] && !ISSPACE(z[j+n]); n++){} exclude = 1; for(k=0; kiStart ) fprintf(out,"%.*s",i-iStart,&line[iStart]); fprintf(out,"%s",name); @@ -3478,9 +3486,9 @@ PRIVATE void translate_code(struct lemon *lemp, struct rule *rp){ /* This const cast is wrong but harmless, if we're careful. */ for(cp=(char *)rp->code; *cp; cp++){ - if( isalpha(*cp) && (cp==rp->code || (!isalnum(cp[-1]) && cp[-1]!='_')) ){ + if( ISALPHA(*cp) && (cp==rp->code || (!ISALNUM(cp[-1]) && cp[-1]!='_')) ){ char saved; - for(xp= &cp[1]; isalnum(*xp) || *xp=='_'; xp++); + for(xp= &cp[1]; ISALNUM(*xp) || *xp=='_'; xp++); saved = *xp; *xp = 0; if( rp->lhsalias && strcmp(cp,rp->lhsalias)==0 ){ @@ -3645,9 +3653,9 @@ void print_stack_union( cp = sp->datatype; if( cp==0 ) cp = lemp->vartype; j = 0; - while( isspace(*cp) ) cp++; + while( ISSPACE(*cp) ) cp++; while( *cp ) stddt[j++] = *cp++; - while( j>0 && isspace(stddt[j-1]) ) j--; + while( j>0 && ISSPACE(stddt[j-1]) ) j--; stddt[j] = 0; if( lemp->tokentype && strcmp(stddt, lemp->tokentype)==0 ){ sp->dtnum = 0; @@ -3857,8 +3865,8 @@ void ReportTable( name = lemp->name ? lemp->name : "Parse"; if( lemp->arg && lemp->arg[0] ){ i = lemonStrlen(lemp->arg); - while( i>=1 && isspace(lemp->arg[i-1]) ) i--; - while( i>=1 && (isalnum(lemp->arg[i-1]) || lemp->arg[i-1]=='_') ) i--; + while( i>=1 && ISSPACE(lemp->arg[i-1]) ) i--; + while( i>=1 && (ISALNUM(lemp->arg[i-1]) || lemp->arg[i-1]=='_') ) i--; fprintf(out,"#define %sARG_SDECL %s;\n",name,lemp->arg); lineno++; fprintf(out,"#define %sARG_PDECL ,%s\n",name,lemp->arg); lineno++; fprintf(out,"#define %sARG_FETCH %s = yypParser->%s\n", @@ -4666,7 +4674,7 @@ struct symbol *Symbol_new(const char *x) sp = (struct symbol *)calloc(1, sizeof(struct symbol) ); MemoryCheck(sp); sp->name = Strsafe(x); - sp->type = isupper(*x) ? TERMINAL : NONTERMINAL; + sp->type = ISUPPER(*x) ? TERMINAL : NONTERMINAL; sp->rule = 0; sp->fallback = 0; sp->prec = -1; diff --git a/tool/mkautoconfamal.sh b/tool/mkautoconfamal.sh index 0c2668c8b7..d78d681816 100644 --- a/tool/mkautoconfamal.sh +++ b/tool/mkautoconfamal.sh @@ -56,7 +56,7 @@ mv $TMPSPACE/tmp $TMPSPACE/configure.ac cd $TMPSPACE aclocal autoconf -automake +automake --add-missing mkdir -p tea/generic echo "#ifdef USE_SYSTEM_SQLITE" > tea/generic/tclsqlite3.c diff --git a/tool/showdb.c b/tool/showdb.c index 892cacd552..38b92f1f7d 100644 --- a/tool/showdb.c +++ b/tool/showdb.c @@ -3,6 +3,8 @@ */ #include #include +#define ISDIGIT(X) isdigit((unsigned char)(X)) +#define ISPRINT(X) isprint((unsigned char)(X)) #include #include #include @@ -217,7 +219,7 @@ static unsigned char *print_byte_range( if( i+j>nByte ){ fprintf(stdout, " "); }else{ - fprintf(stdout,"%c", isprint(aData[i+j]) ? aData[i+j] : '.'); + fprintf(stdout,"%c", ISPRINT(aData[i+j]) ? aData[i+j] : '.'); } } fprintf(stdout,"\n"); @@ -600,7 +602,7 @@ static void decodeCell( }else{ zConst[0] = '\''; for(ii=1, jj=0; jj #include "sqlite3.h" +#define ISPRINT(X) isprint((unsigned char)(X)) + typedef sqlite3_int64 i64; /* 64-bit signed integer type */ @@ -131,7 +133,7 @@ int main(int argc, char **argv){ printf("%s\"", zSep); for(j=0; j #include +#define ISDIGIT(X) isdigit((unsigned char)(X)) +#define ISPRINT(X) isprint((unsigned char)(X)) + #if !defined(_MSC_VER) #include #else @@ -159,7 +162,7 @@ static void print_byte_range( if( i+j>nByte ){ fprintf(stdout, " "); }else{ - fprintf(stdout,"%c", isprint(aData[i+j]) ? aData[i+j] : '.'); + fprintf(stdout,"%c", ISPRINT(aData[i+j]) ? aData[i+j] : '.'); } } fprintf(stdout,"\n"); @@ -550,7 +553,7 @@ int main(int argc, char **argv){ print_wal_header(0); continue; } - if( !isdigit(argv[i][0]) ){ + if( !ISDIGIT(argv[i][0]) ){ fprintf(stderr, "%s: unknown option: [%s]\n", argv[0], argv[i]); continue; } diff --git a/tool/speedtest16.c b/tool/speedtest16.c index e81f1a6dba..993cc19268 100644 --- a/tool/speedtest16.c +++ b/tool/speedtest16.c @@ -29,6 +29,8 @@ #include #include "sqlite3.h" +#define ISSPACE(X) isspace((unsigned char)(X)) + /* ** hwtime.h contains inline assembler code for implementing ** high-performance timing routines. @@ -140,7 +142,7 @@ int main(int argc, char **argv){ zSql[j+1] = c; if( isComplete ){ zSql[j] = 0; - while( i #include "sqlite3.h" +#define ISSPACE(X) isspace((unsigned char)(X)) + #include "test_osinst.c" /* @@ -197,7 +199,7 @@ int main(int argc, char **argv){ zSql[j+1] = c; if( isComplete ){ zSql[j] = 0; - while( i