diff --git a/Makefile.in b/Makefile.in index 4a81e15664..a9fc93af7e 100644 --- a/Makefile.in +++ b/Makefile.in @@ -517,6 +517,9 @@ sqlite3$(TEXE): $(TOP)/src/shell.c libsqlite3.la sqlite3.h sqlite3.c: .target_source $(TOP)/tool/mksqlite3c.tcl $(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl +sqlite3-all.c: sqlite3.c $(TOP)/tool/split-sqlite3c.tcl + $(TCLSH_CMD) $(TOP)/tool/split-sqlite3c.tcl + # Rule to build the amalgamation # sqlite3.lo: sqlite3.c @@ -785,7 +788,7 @@ parse.c: $(TOP)/src/parse.y lemon$(BEXE) $(TOP)/addopcodes.awk $(NAWK) -f $(TOP)/addopcodes.awk parse.h.temp >parse.h sqlite3.h: $(TOP)/src/sqlite.h.in $(TOP)/manifest.uuid $(TOP)/VERSION - tclsh $(TOP)/tool/mksqlite3h.tcl $(TOP) >sqlite3.h + $(TCLSH_CMD) $(TOP)/tool/mksqlite3h.tcl $(TOP) >sqlite3.h keywordhash.h: $(TOP)/tool/mkkeywordhash.c $(BCC) -o mkkeywordhash$(BEXE) $(OPT_FEATURE_FLAGS) $(OPTS) $(TOP)/tool/mkkeywordhash.c diff --git a/Makefile.msc b/Makefile.msc index a6d1bf4c2f..799970d5f9 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -43,16 +43,32 @@ TCC = $(TCC) -I$(TOP)\ext\rtree # TCC = $(TCC) -DNDEBUG -# The library that programs using TCL must link against. +# The locations of the Tcl header and library files. Also, the library that +# non-stubs enabled programs using Tcl must link against. These variables +# (TCLINCDIR, TCLLIBDIR, and LIBTCL) may be overridden via the environment +# prior to running nmake in order to match the actual installed location and +# version on this machine. # -LIBTCL = tcl85.lib +!if "$(TCLINCDIR)" == "" TCLINCDIR = c:\tcl\include +!endif + +!if "$(TCLLIBDIR)" == "" TCLLIBDIR = c:\tcl\lib +!endif + +!if "$(LIBTCL)" == "" +LIBTCL = tcl85.lib +!endif # This is the command to use for tclsh - normally just "tclsh", but we may -# know the specific version we want to use +# know the specific version we want to use. This variable (TCLSH_CMD) may be +# overridden via the environment prior to running nmake in order to select a +# specific Tcl shell to use. # +!if "$(TCLSH_CMD)" == "" TCLSH_CMD = tclsh85 +!endif # Compiler options needed for programs that use the readline() library. # @@ -118,7 +134,7 @@ LTLIBOPTS = /MACHINE:$(PLATFORM) !ENDIF # nawk compatible awk. -NAWK = .\gawk.exe +NAWK = gawk.exe # You should not have to change anything below this line ############################################################################### @@ -439,7 +455,7 @@ EXTHDR = $(EXTHDR) \ # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. # -all: libsqlite3.lib sqlite3.exe libtclsqlite3.lib +all: dll libsqlite3.lib sqlite3.exe libtclsqlite3.lib libsqlite3.lib: $(LIBOBJ) $(LTLIB) $(LTLIBOPTS) /OUT:$@ $(LIBOBJ) $(TLIBS) @@ -463,13 +479,16 @@ sqlite3.exe: $(TOP)\src\shell.c libsqlite3.lib sqlite3.h -mkdir tsrc for %i in ($(SRC)) do copy /Y %i tsrc del /Q tsrc\sqlite.h.in tsrc\parse.y - $(TCLSH_CMD) $(TOP)\tool\vdbe-compress.tcl vdbe.new + $(TCLSH_CMD) $(TOP)\tool\vdbe-compress.tcl < tsrc\vdbe.c > vdbe.new move vdbe.new tsrc\vdbe.c echo > .target_source sqlite3.c: .target_source $(TOP)\tool\mksqlite3c.tcl $(TCLSH_CMD) $(TOP)\tool\mksqlite3c.tcl +sqlite3-all.c: sqlite3.c $(TOP)/tool/split-sqlite3c.tcl + $(TCLSH_CMD) $(TOP)/tool/split-sqlite3c.tcl + # Rule to build the amalgamation # sqlite3.lo: sqlite3.c @@ -720,10 +739,10 @@ tclsqlite3.exe: tclsqlite-shell.lo libsqlite3.lib # Rules to build opcodes.c and opcodes.h # opcodes.c: opcodes.h $(TOP)\mkopcodec.awk - $(NAWK) "/#define OP_/ { print }" opcodes.h | sort /+45 | $(NAWK) -f $(TOP)\mkopcodec.awk >opcodes.c + $(NAWK) "/#define OP_/ { print }" opcodes.h | sort /+45 | $(NAWK) -f $(TOP)\mkopcodec.awk > opcodes.c opcodes.h: parse.h $(TOP)\src\vdbe.c $(TOP)\mkopcodeh.awk - type parse.h $(TOP)\src\vdbe.c | $(NAWK) -f $(TOP)\mkopcodeh.awk >opcodes.h + type parse.h $(TOP)\src\vdbe.c | $(NAWK) -f $(TOP)\mkopcodeh.awk > opcodes.h # Rules to build parse.c and parse.h - the outputs of lemon. # @@ -734,16 +753,16 @@ parse.c: $(TOP)\src\parse.y lemon.exe $(TOP)\addopcodes.awk copy $(TOP)\src\parse.y . .\lemon.exe $(OPT_FEATURE_FLAGS) $(OPTS) parse.y move parse.h parse.h.temp - $(NAWK) -f $(TOP)\addopcodes.awk parse.h.temp >parse.h + $(NAWK) -f $(TOP)\addopcodes.awk parse.h.temp > parse.h sqlite3.h: $(TOP)\src\sqlite.h.in $(TOP)\manifest.uuid $(TOP)\VERSION - $(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP) >sqlite3.h + $(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP) > sqlite3.h mkkeywordhash.exe: $(TOP)\tool\mkkeywordhash.c $(BCC) -Femkkeywordhash.exe $(OPT_FEATURE_FLAGS) $(OPTS) $(TOP)\tool\mkkeywordhash.c keywordhash.h: $(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe - .\mkkeywordhash.exe >keywordhash.h + .\mkkeywordhash.exe > keywordhash.h @@ -839,7 +858,7 @@ test: testfixture.exe sqlite3.exe spaceanal_tcl.h: $(TOP)\tool\spaceanal.tcl $(NAWK) -f $(TOP)/tool/tostr.awk \ - $(TOP)\tool\spaceanal.tcl >spaceanal_tcl.h + $(TOP)\tool\spaceanal.tcl > spaceanal_tcl.h sqlite3_analyzer.exe: $(TESTFIXTURE_SRC) spaceanal_tcl.h $(LTLINK) -DTCLSH=2 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 \ @@ -866,10 +885,10 @@ clean: dll: sqlite3.dll sqlite3.def: libsqlite3.lib - echo EXPORTS >sqlite3.def + echo EXPORTS > sqlite3.def dumpbin /all libsqlite3.lib \ | $(NAWK) "/ 1 _sqlite3_/ { sub(/^.* _/,\"\");print }" \ - | sort >>sqlite3.def + | sort >> sqlite3.def sqlite3.dll: $(LIBOBJ) sqlite3.def link $(LTLINKOPTS) /DLL /DEF:sqlite3.def /OUT:$@ $(LIBOBJ) diff --git a/VERSION b/VERSION index d2577d9756..a0fc9e07cb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.7.7 +3.7.8 diff --git a/configure b/configure index e0985011af..14b71c67b1 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.62 for sqlite 3.7.7. +# Generated by GNU Autoconf 2.62 for sqlite 3.7.8. # # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, # 2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc. @@ -743,8 +743,8 @@ SHELL=${CONFIG_SHELL-/bin/sh} # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.7.7' -PACKAGE_STRING='sqlite 3.7.7' +PACKAGE_VERSION='3.7.8' +PACKAGE_STRING='sqlite 3.7.8' PACKAGE_BUGREPORT='' # Factoring default headers for most tests. @@ -1485,7 +1485,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.7.7 to adapt to many kinds of systems. +\`configure' configures sqlite 3.7.8 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1550,7 +1550,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.7.7:";; + short | recursive ) echo "Configuration of sqlite 3.7.8:";; esac cat <<\_ACEOF @@ -1666,7 +1666,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.7.7 +sqlite configure 3.7.8 generated by GNU Autoconf 2.62 Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, @@ -1680,7 +1680,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.7.7, which was +It was created by sqlite $as_me 3.7.8, which was generated by GNU Autoconf 2.62. Invocation command line was $ $0 $@ @@ -14030,7 +14030,7 @@ exec 6>&1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.7.7, which was +This file was extended by sqlite $as_me 3.7.8, which was generated by GNU Autoconf 2.62. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -14083,7 +14083,7 @@ Report bugs to ." _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_version="\\ -sqlite config.status 3.7.7 +sqlite config.status 3.7.8 configured by $0, generated by GNU Autoconf 2.62, with options \\"`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\" diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 67a0ea2096..448a5a403a 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -312,6 +312,11 @@ SQLITE_EXTENSION_INIT1 #endif +static int fts3EvalNext(Fts3Cursor *pCsr); +static int fts3EvalStart(Fts3Cursor *pCsr); +static int fts3TermSegReaderCursor( + Fts3Cursor *, const char *, int, int, Fts3MultiSegReader **); + /* ** Write a 64-bit variable-length integer to memory starting at p[0]. ** The length of data written will be between 1 and FTS3_VARINT_MAX bytes. @@ -820,9 +825,23 @@ static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){ return zRet; } +/* +** This function interprets the string at (*pp) as a non-negative integer +** value. It reads the integer and sets *pnOut to the value read, then +** sets *pp to point to the byte immediately following the last byte of +** the integer value. +** +** Only decimal digits ('0'..'9') may be part of an integer value. +** +** If *pp does not being with a decimal digit SQLITE_ERROR is returned and +** the output value undefined. Otherwise SQLITE_OK is returned. +** +** This function is used when parsing the "prefix=" FTS4 parameter. +*/ static int fts3GobbleInt(const char **pp, int *pnOut){ - const char *p = *pp; - int nInt = 0; + const char *p = *pp; /* Iterator pointer */ + int nInt = 0; /* Output value */ + for(p=*pp; p[0]>='0' && p[0]<='9'; p++){ nInt = nInt * 10 + (p[0] - '0'); } @@ -832,15 +851,30 @@ static int fts3GobbleInt(const char **pp, int *pnOut){ return SQLITE_OK; } - +/* +** This function is called to allocate an array of Fts3Index structures +** representing the indexes maintained by the current FTS table. FTS tables +** always maintain the main "terms" index, but may also maintain one or +** more "prefix" indexes, depending on the value of the "prefix=" parameter +** (if any) specified as part of the CREATE VIRTUAL TABLE statement. +** +** Argument zParam is passed the value of the "prefix=" option if one was +** specified, or NULL otherwise. +** +** If no error occurs, SQLITE_OK is returned and *apIndex set to point to +** the allocated array. *pnIndex is set to the number of elements in the +** array. If an error does occur, an SQLite error code is returned. +** +** Regardless of whether or not an error is returned, it is the responsibility +** of the caller to call sqlite3_free() on the output array to free it. +*/ static int fts3PrefixParameter( const char *zParam, /* ABC in prefix=ABC parameter to parse */ int *pnIndex, /* OUT: size of *apIndex[] array */ - struct Fts3Index **apIndex, /* OUT: Array of indexes for this table */ - struct Fts3Index **apFree /* OUT: Free this with sqlite3_free() */ + struct Fts3Index **apIndex /* OUT: Array of indexes for this table */ ){ - struct Fts3Index *aIndex; - int nIndex = 1; + struct Fts3Index *aIndex; /* Allocated array */ + int nIndex = 1; /* Number of entries in array */ if( zParam && zParam[0] ){ const char *p; @@ -851,7 +885,7 @@ static int fts3PrefixParameter( } aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex); - *apIndex = *apFree = aIndex; + *apIndex = aIndex; *pnIndex = nIndex; if( !aIndex ){ return SQLITE_NOMEM; @@ -908,8 +942,7 @@ static int fts3InitVtab( sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */ int nIndex; /* Size of aIndex[] array */ - struct Fts3Index *aIndex; /* Array of indexes for this table */ - struct Fts3Index *aFree = 0; /* Free this before returning */ + struct Fts3Index *aIndex = 0; /* Array of indexes for this table */ /* The results of parsing supported FTS4 key=value options: */ int bNoDocsize = 0; /* True to omit %_docsize table */ @@ -1046,7 +1079,7 @@ static int fts3InitVtab( } assert( pTokenizer ); - rc = fts3PrefixParameter(zPrefix, &nIndex, &aIndex, &aFree); + rc = fts3PrefixParameter(zPrefix, &nIndex, &aIndex); if( rc==SQLITE_ERROR ){ assert( zPrefix ); *pzErr = sqlite3_mprintf("error parsing prefix parameter: %s", zPrefix); @@ -1133,7 +1166,7 @@ static int fts3InitVtab( fts3_init_out: sqlite3_free(zPrefix); - sqlite3_free(aFree); + sqlite3_free(aIndex); sqlite3_free(zCompress); sqlite3_free(zUncompress); sqlite3_free((void *)aCol); @@ -1724,8 +1757,6 @@ static void fts3PoslistMerge( } /* -** nToken==1 searches for adjacent positions. -** ** This function is used to merge two position lists into one. When it is ** called, *pp1 and *pp2 must both point to position lists. A position-list is ** the part of a doclist that follows each document id. For example, if a row @@ -1745,6 +1776,8 @@ static void fts3PoslistMerge( ** *pp1 so that (pos(*pp2)>pos(*pp1) && pos(*pp2)-pos(*pp1)<=nToken). i.e. ** when the *pp1 token appears before the *pp2 token, but not more than nToken ** slots before it. +** +** e.g. nToken==1 searches for adjacent positions. */ static int fts3PoslistPhraseMerge( char **pp, /* IN/OUT: Preallocated output buffer */ @@ -1911,22 +1944,34 @@ static int fts3PoslistNearMerge( } /* -** A pointer to an instance of this structure is used as the context -** argument to sqlite3Fts3SegReaderIterate() +** An instance of this function is used to merge together the (potentially +** large number of) doclists for each term that matches a prefix query. +** See function fts3TermSelectMerge() for details. */ typedef struct TermSelect TermSelect; struct TermSelect { - int isReqPos; - char *aaOutput[16]; /* Malloc'd output buffer */ - int anOutput[16]; /* Size of output in bytes */ + char *aaOutput[16]; /* Malloc'd output buffers */ + int anOutput[16]; /* Size each output buffer in bytes */ }; - +/* +** This function is used to read a single varint from a buffer. Parameter +** pEnd points 1 byte past the end of the buffer. When this function is +** called, if *pp points to pEnd or greater, then the end of the buffer +** has been reached. In this case *pp is set to 0 and the function returns. +** +** If *pp does not point to or past pEnd, then a single varint is read +** from *pp. *pp is then set to point 1 byte past the end of the read varint. +** +** If bDescIdx is false, the value read is added to *pVal before returning. +** If it is true, the value read is subtracted from *pVal before this +** function returns. +*/ static void fts3GetDeltaVarint3( - char **pp, - char *pEnd, - int bDescIdx, - sqlite3_int64 *pVal + char **pp, /* IN/OUT: Point to read varint from */ + char *pEnd, /* End of buffer */ + int bDescIdx, /* True if docids are descending */ + sqlite3_int64 *pVal /* IN/OUT: Integer value */ ){ if( *pp>=pEnd ){ *pp = 0; @@ -1941,6 +1986,21 @@ static void fts3GetDeltaVarint3( } } +/* +** This function is used to write a single varint to a buffer. The varint +** is written to *pp. Before returning, *pp is set to point 1 byte past the +** end of the value written. +** +** If *pbFirst is zero when this function is called, the value written to +** the buffer is that of parameter iVal. +** +** If *pbFirst is non-zero when this function is called, then the value +** written is either (iVal-*piPrev) (if bDescIdx is zero) or (*piPrev-iVal) +** (if bDescIdx is non-zero). +** +** Before returning, this function always sets *pbFirst to 1 and *piPrev +** to the value of parameter iVal. +*/ static void fts3PutDeltaVarint3( char **pp, /* IN/OUT: Output pointer */ int bDescIdx, /* True for descending docids */ @@ -1961,10 +2021,34 @@ static void fts3PutDeltaVarint3( *pbFirst = 1; } -#define COMPARE_DOCID(i1, i2) ((bDescIdx?-1:1) * (i1-i2)) +/* +** This macro is used by various functions that merge doclists. The two +** arguments are 64-bit docid values. If the value of the stack variable +** bDescDoclist is 0 when this macro is invoked, then it returns (i1-i2). +** Otherwise, (i2-i1). +** +** Using this makes it easier to write code that can merge doclists that are +** sorted in either ascending or descending order. +*/ +#define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i1-i2)) + +/* +** This function does an "OR" merge of two doclists (output contains all +** positions contained in either argument doclist). If the docids in the +** input doclists are sorted in ascending order, parameter bDescDoclist +** should be false. If they are sorted in ascending order, it should be +** passed a non-zero value. +** +** If no error occurs, *paOut is set to point at an sqlite3_malloc'd buffer +** containing the output doclist and SQLITE_OK is returned. In this case +** *pnOut is set to the number of bytes in the output doclist. +** +** If an error occurs, an SQLite error code is returned. The output values +** are undefined in this case. +*/ static int fts3DoclistOrMerge( - int bDescIdx, /* True if arguments are desc */ + int bDescDoclist, /* True if arguments are desc */ char *a1, int n1, /* First doclist */ char *a2, int n2, /* Second doclist */ char **paOut, int *pnOut /* OUT: Malloc'd doclist */ @@ -1989,21 +2073,21 @@ static int fts3DoclistOrMerge( fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); while( p1 || p2 ){ - sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2); + sqlite3_int64 iDiff = DOCID_CMP(i1, i2); if( p2 && p1 && iDiff==0 ){ - fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); fts3PoslistMerge(&p, &p1, &p2); - fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); - fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); }else if( !p2 || (p1 && iDiff<0) ){ - fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); fts3PoslistCopy(&p, &p1); - fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); }else{ - fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i2); + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i2); fts3PoslistCopy(&p, &p2); - fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); } } @@ -2012,8 +2096,20 @@ static int fts3DoclistOrMerge( return SQLITE_OK; } +/* +** This function does a "phrase" merge of two doclists. In a phrase merge, +** the output contains a copy of each position from the right-hand input +** doclist for which there is a position in the left-hand input doclist +** exactly nDist tokens before it. +** +** If the docids in the input doclists are sorted in ascending order, +** parameter bDescDoclist should be false. If they are sorted in ascending +** order, it should be passed a non-zero value. +** +** The right-hand input doclist is overwritten by this function. +*/ static void fts3DoclistPhraseMerge( - int bDescIdx, /* True if arguments are desc */ + int bDescDoclist, /* True if arguments are desc */ int nDist, /* Distance from left to right (1=adjacent) */ char *aLeft, int nLeft, /* Left doclist */ char *aRight, int *pnRight /* IN/OUT: Right/output doclist */ @@ -2036,26 +2132,26 @@ static void fts3DoclistPhraseMerge( fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); while( p1 && p2 ){ - sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2); + sqlite3_int64 iDiff = DOCID_CMP(i1, i2); if( iDiff==0 ){ char *pSave = p; sqlite3_int64 iPrevSave = iPrev; int bFirstOutSave = bFirstOut; - fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); if( 0==fts3PoslistPhraseMerge(&p, nDist, 0, 1, &p1, &p2) ){ p = pSave; iPrev = iPrevSave; bFirstOut = bFirstOutSave; } - fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); - fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); }else if( iDiff<0 ){ fts3PoslistCopy(0, &p1); - fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); }else{ fts3PoslistCopy(0, &p2); - fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); } } @@ -2072,7 +2168,7 @@ static void fts3DoclistPhraseMerge( ** the responsibility of the caller to free any doclists left in the ** TermSelect.aaOutput[] array. */ -static int fts3TermSelectMerge(Fts3Table *p, TermSelect *pTS){ +static int fts3TermSelectFinishMerge(Fts3Table *p, TermSelect *pTS){ char *aOut = 0; int nOut = 0; int i; @@ -2113,24 +2209,25 @@ static int fts3TermSelectMerge(Fts3Table *p, TermSelect *pTS){ } /* -** This function is used as the sqlite3Fts3SegReaderIterate() callback when -** querying the full-text index for a doclist associated with a term or -** term-prefix. +** Merge the doclist aDoclist/nDoclist into the TermSelect object passed +** as the first argument. The merge is an "OR" merge (see function +** fts3DoclistOrMerge() for details). +** +** This function is called with the doclist for each term that matches +** a queried prefix. It merges all these doclists into one, the doclist +** for the specified prefix. Since there can be a very large number of +** doclists to merge, the merging is done pair-wise using the TermSelect +** object. +** +** This function returns SQLITE_OK if the merge is successful, or an +** SQLite error code (SQLITE_NOMEM) if an error occurs. */ -static int fts3TermSelectCb( - Fts3Table *p, /* Virtual table object */ - void *pContext, /* Pointer to TermSelect structure */ - char *zTerm, - int nTerm, - char *aDoclist, - int nDoclist +static int fts3TermSelectMerge( + Fts3Table *p, /* FTS table handle */ + TermSelect *pTS, /* TermSelect object to merge into */ + char *aDoclist, /* Pointer to doclist */ + int nDoclist /* Size of aDoclist in bytes */ ){ - TermSelect *pTS = (TermSelect *)pContext; - - UNUSED_PARAMETER(p); - UNUSED_PARAMETER(zTerm); - UNUSED_PARAMETER(nTerm); - if( pTS->aaOutput[0]==0 ){ /* If this is the first term selected, copy the doclist to the output ** buffer using memcpy(). */ @@ -2201,6 +2298,13 @@ static int fts3SegReaderCursorAppend( return SQLITE_OK; } +/* +** Add seg-reader objects to the Fts3MultiSegReader object passed as the +** 8th argument. +** +** This function returns SQLITE_OK if successful, or an SQLite error code +** otherwise. +*/ static int fts3SegReaderCursor( Fts3Table *p, /* FTS3 table handle */ int iIndex, /* Index to search (from 0 to p->nIndex-1) */ @@ -2209,11 +2313,11 @@ static int fts3SegReaderCursor( int nTerm, /* Size of zTerm in bytes */ int isPrefix, /* True for a prefix search */ int isScan, /* True to scan from zTerm to EOF */ - Fts3MultiSegReader *pCsr /* Cursor object to populate */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ ){ - int rc = SQLITE_OK; - int rc2; - sqlite3_stmt *pStmt = 0; + int rc = SQLITE_OK; /* Error code */ + sqlite3_stmt *pStmt = 0; /* Statement to iterate through segments */ + int rc2; /* Result of sqlite3_reset() */ /* If iLevel is less than 0 and this is not a scan, include a seg-reader ** for the pending-terms. If this is a scan, then this call must be being @@ -2302,24 +2406,42 @@ int sqlite3Fts3SegReaderCursor( ); } +/* +** In addition to its current configuration, have the Fts3MultiSegReader +** passed as the 4th argument also scan the doclist for term zTerm/nTerm. +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +*/ static int fts3SegReaderCursorAddZero( - Fts3Table *p, - const char *zTerm, - int nTerm, - Fts3MultiSegReader *pCsr + Fts3Table *p, /* FTS virtual table handle */ + const char *zTerm, /* Term to scan doclist of */ + int nTerm, /* Number of bytes in zTerm */ + Fts3MultiSegReader *pCsr /* Fts3MultiSegReader to modify */ ){ return fts3SegReaderCursor(p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0,pCsr); } - -int sqlite3Fts3TermSegReaderCursor( +/* +** Open an Fts3MultiSegReader to scan the doclist for term zTerm/nTerm. Or, +** if isPrefix is true, to scan the doclist for all terms for which +** zTerm/nTerm is a prefix. If successful, return SQLITE_OK and write +** a pointer to the new Fts3MultiSegReader to *ppSegcsr. Otherwise, return +** an SQLite error code. +** +** It is the responsibility of the caller to free this object by eventually +** passing it to fts3SegReaderCursorFree() +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +** Output parameter *ppSegcsr is set to 0 if an error occurs. +*/ +static int fts3TermSegReaderCursor( Fts3Cursor *pCsr, /* Virtual table cursor handle */ const char *zTerm, /* Term to query for */ int nTerm, /* Size of zTerm in bytes */ int isPrefix, /* True for a prefix search */ Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */ ){ - Fts3MultiSegReader *pSegcsr; /* Object to allocate and return */ + Fts3MultiSegReader *pSegcsr; /* Object to allocate and return */ int rc = SQLITE_NOMEM; /* Return code */ pSegcsr = sqlite3_malloc(sizeof(Fts3MultiSegReader)); @@ -2363,6 +2485,9 @@ int sqlite3Fts3TermSegReaderCursor( return rc; } +/* +** Free an Fts3MultiSegReader allocated by fts3TermSegReaderCursor(). +*/ static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){ sqlite3Fts3SegReaderFinish(pSegcsr); sqlite3_free(pSegcsr); @@ -2370,35 +2495,25 @@ static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){ /* ** This function retreives the doclist for the specified term (or term -** prefix) from the database. -** -** The returned doclist may be in one of two formats, depending on the -** value of parameter isReqPos. If isReqPos is zero, then the doclist is -** a sorted list of delta-compressed docids (a bare doclist). If isReqPos -** is non-zero, then the returned list is in the same format as is stored -** in the database without the found length specifier at the start of on-disk -** doclists. +** prefix) from the database. */ static int fts3TermSelect( Fts3Table *p, /* Virtual table handle */ Fts3PhraseToken *pTok, /* Token to query for */ int iColumn, /* Column to query (or -ve for all columns) */ - int isReqPos, /* True to include position lists in output */ int *pnOut, /* OUT: Size of buffer at *ppOut */ char **ppOut /* OUT: Malloced result buffer */ ){ int rc; /* Return code */ - Fts3MultiSegReader *pSegcsr; /* Seg-reader cursor for this term */ - TermSelect tsc; /* Context object for fts3TermSelectCb() */ + Fts3MultiSegReader *pSegcsr; /* Seg-reader cursor for this term */ + TermSelect tsc; /* Object for pair-wise doclist merging */ Fts3SegFilter filter; /* Segment term filter configuration */ pSegcsr = pTok->pSegcsr; memset(&tsc, 0, sizeof(TermSelect)); - tsc.isReqPos = isReqPos; - filter.flags = FTS3_SEGMENT_IGNORE_EMPTY + filter.flags = FTS3_SEGMENT_IGNORE_EMPTY | FTS3_SEGMENT_REQUIRE_POS | (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0) - | (isReqPos ? FTS3_SEGMENT_REQUIRE_POS : 0) | (iColumnnColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0); filter.iCol = iColumn; filter.zTerm = pTok->z; @@ -2408,13 +2523,11 @@ static int fts3TermSelect( while( SQLITE_OK==rc && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pSegcsr)) ){ - rc = fts3TermSelectCb(p, (void *)&tsc, - pSegcsr->zTerm, pSegcsr->nTerm, pSegcsr->aDoclist, pSegcsr->nDoclist - ); + rc = fts3TermSelectMerge(p, &tsc, pSegcsr->aDoclist, pSegcsr->nDoclist); } if( rc==SQLITE_OK ){ - rc = fts3TermSelectMerge(p, &tsc); + rc = fts3TermSelectFinishMerge(p, &tsc); } if( rc==SQLITE_OK ){ *ppOut = tsc.aaOutput[0]; @@ -2440,24 +2553,15 @@ static int fts3TermSelect( ** that the doclist is simply a list of docids stored as delta encoded ** varints. */ -static int fts3DoclistCountDocids(int isPoslist, char *aList, int nList){ +static int fts3DoclistCountDocids(char *aList, int nList){ int nDoc = 0; /* Return value */ if( aList ){ char *aEnd = &aList[nList]; /* Pointer to one byte after EOF */ char *p = aList; /* Cursor */ - if( !isPoslist ){ - /* The number of docids in the list is the same as the number of - ** varints. In FTS3 a varint consists of a single byte with the 0x80 - ** bit cleared and zero or more bytes with the 0x80 bit set. So to - ** count the varints in the buffer, just count the number of bytes - ** with the 0x80 bit clear. */ - while( pbase.pVtab)->pSegments==0 ); return rc; @@ -2564,7 +2668,7 @@ static int fts3FilterMethod( rc = sqlite3Fts3ReadLock(p); if( rc!=SQLITE_OK ) return rc; - rc = sqlite3Fts3EvalStart(pCsr, pCsr->pExpr, 1); + rc = fts3EvalStart(pCsr); sqlite3Fts3SegmentsClose(p); if( rc!=SQLITE_OK ) return rc; @@ -2971,6 +3075,11 @@ static int fts3RenameMethod( return rc; } +/* +** The xSavepoint() method. +** +** Flush the contents of the pending-terms table to disk. +*/ static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ UNUSED_PARAMETER(iSavepoint); assert( ((Fts3Table *)pVtab)->inTransaction ); @@ -2978,6 +3087,12 @@ static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint ); return fts3SyncMethod(pVtab); } + +/* +** The xRelease() method. +** +** This is a no-op. +*/ static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); UNUSED_PARAMETER(iSavepoint); @@ -2987,6 +3102,12 @@ static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ TESTONLY( p->mxSavepoint = iSavepoint-1 ); return SQLITE_OK; } + +/* +** The xRollbackTo() method. +** +** Discard the contents of the pending terms table. +*/ static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ Fts3Table *p = (Fts3Table*)pVtab; UNUSED_PARAMETER(iSavepoint); @@ -3136,18 +3257,6 @@ int sqlite3Fts3Init(sqlite3 *db){ return rc; } -#if !SQLITE_CORE -int sqlite3_extension_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi -){ - SQLITE_EXTENSION_INIT2(pApi) - return sqlite3Fts3Init(db); -} -#endif - - /* ** Allocate an Fts3MultiSegReader for each token in the expression headed ** by pExpr. @@ -3164,11 +3273,11 @@ int sqlite3_extension_init( ** doclist and then traversed. */ static void fts3EvalAllocateReaders( - Fts3Cursor *pCsr, - Fts3Expr *pExpr, + Fts3Cursor *pCsr, /* FTS cursor handle */ + Fts3Expr *pExpr, /* Allocate readers for this expression */ int *pnToken, /* OUT: Total number of tokens in phrase. */ int *pnOr, /* OUT: Total number of OR nodes in expr. */ - int *pRc + int *pRc /* IN/OUT: Error code */ ){ if( pExpr && SQLITE_OK==*pRc ){ if( pExpr->eType==FTSQUERY_PHRASE ){ @@ -3177,7 +3286,7 @@ static void fts3EvalAllocateReaders( *pnToken += nToken; for(i=0; ipPhrase->aToken[i]; - int rc = sqlite3Fts3TermSegReaderCursor(pCsr, + int rc = fts3TermSegReaderCursor(pCsr, pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr ); if( rc!=SQLITE_OK ){ @@ -3195,12 +3304,20 @@ static void fts3EvalAllocateReaders( } } +/* +** Arguments pList/nList contain the doclist for token iToken of phrase p. +** It is merged into the main doclist stored in p->doclist.aAll/nAll. +** +** This function assumes that pList points to a buffer allocated using +** sqlite3_malloc(). This function takes responsibility for eventually +** freeing the buffer. +*/ static void fts3EvalPhraseMergeToken( - Fts3Table *pTab, - Fts3Phrase *p, - int iToken, - char *pList, - int nList + Fts3Table *pTab, /* FTS Table pointer */ + Fts3Phrase *p, /* Phrase to merge pList/nList into */ + int iToken, /* Token pList/nList corresponds to */ + char *pList, /* Pointer to doclist */ + int nList /* Number of bytes in pList */ ){ assert( iToken!=p->iDoclistToken ); @@ -3249,9 +3366,15 @@ static void fts3EvalPhraseMergeToken( if( iToken>p->iDoclistToken ) p->iDoclistToken = iToken; } +/* +** Load the doclist for phrase p into p->doclist.aAll/nAll. The loaded doclist +** does not take deferred tokens into account. +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +*/ static int fts3EvalPhraseLoad( - Fts3Cursor *pCsr, - Fts3Phrase *p + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Phrase *p /* Phrase object */ ){ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; int iToken; @@ -3264,7 +3387,7 @@ static int fts3EvalPhraseLoad( if( pToken->pSegcsr ){ int nThis = 0; char *pThis = 0; - rc = fts3TermSelect(pTab, pToken, p->iColumn, 1, &nThis, &pThis); + rc = fts3TermSelect(pTab, pToken, p->iColumn, &nThis, &pThis); if( rc==SQLITE_OK ){ fts3EvalPhraseMergeToken(pTab, p, iToken, pThis, nThis); } @@ -3275,14 +3398,22 @@ static int fts3EvalPhraseLoad( return rc; } +/* +** This function is called on each phrase after the position lists for +** any deferred tokens have been loaded into memory. It updates the phrases +** current position list to include only those positions that are really +** instances of the phrase (after considering deferred tokens). If this +** means that the phrase does not appear in the current row, doclist.pList +** and doclist.nList are both zeroed. +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +*/ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ - int iToken; - int rc = SQLITE_OK; - - int nMaxUndeferred = pPhrase->iDoclistToken; - char *aPoslist = 0; - int nPoslist = 0; - int iPrev = -1; + int iToken; /* Used to iterate through phrase tokens */ + int rc = SQLITE_OK; /* Return code */ + char *aPoslist = 0; /* Position list for deferred tokens */ + int nPoslist = 0; /* Number of bytes in aPoslist */ + int iPrev = -1; /* Token number of previous deferred token */ assert( pPhrase->doclist.bFreeList==0 ); @@ -3328,6 +3459,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ } if( iPrev>=0 ){ + int nMaxUndeferred = pPhrase->iDoclistToken; if( nMaxUndeferred<0 ){ pPhrase->doclist.pList = aPoslist; pPhrase->doclist.nList = nPoslist; @@ -3376,9 +3508,15 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ ** expression to initialize the mechanism for returning rows. Once this ** function has been called successfully on an Fts3Phrase, it may be ** used with fts3EvalPhraseNext() to iterate through the matching docids. +** +** If parameter bOptOk is true, then the phrase may (or may not) use the +** incremental loading strategy. Otherwise, the entire doclist is loaded into +** memory within this call. +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. */ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ - int rc; + int rc; /* Error code */ Fts3PhraseToken *pFirst = &p->aToken[0]; Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; @@ -3406,7 +3544,13 @@ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ /* ** This function is used to iterate backwards (from the end to start) -** through doclists. +** through doclists. It is used by this module to iterate through phrase +** doclists in reverse and by the fts3_write.c module to iterate through +** pending-terms lists when writing to databases with "order=desc". +** +** The doclist may be sorted in ascending (parameter bDescIdx==0) or +** descending (parameter bDescIdx==1) order of docid. Regardless, this +** function iterates from the end of the doclist to the beginning. */ void sqlite3Fts3DoclistPrev( int bDescIdx, /* True if the doclist is desc */ @@ -3471,9 +3615,9 @@ void sqlite3Fts3DoclistPrev( ** successfully advanced, *pbEof is set to 0. */ static int fts3EvalPhraseNext( - Fts3Cursor *pCsr, - Fts3Phrase *p, - u8 *pbEof + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Phrase *p, /* Phrase object to advance to next docid */ + u8 *pbEof /* OUT: Set to 1 if EOF */ ){ int rc = SQLITE_OK; Fts3Doclist *pDL = &p->doclist; @@ -3519,10 +3663,10 @@ static int fts3EvalPhraseNext( /* pIter now points just past the 0x00 that terminates the position- ** list for document pDL->iDocid. However, if this position-list was - ** edited in place by fts3EvalNearTrim2(), then pIter may not actually + ** edited in place by fts3EvalNearTrim(), then pIter may not actually ** point to the start of the next docid value. The following line deals ** with this case by advancing pIter past the zero-padding added by - ** fts3EvalNearTrim2(). */ + ** fts3EvalNearTrim(). */ while( pIterpNextDocid = pIter; @@ -3534,11 +3678,27 @@ static int fts3EvalPhraseNext( return rc; } +/* +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, fts3EvalPhraseStart() is called on all phrases within the +** expression. Also the Fts3Expr.bDeferred variable is set to true for any +** expressions for which all descendent tokens are deferred. +** +** If parameter bOptOk is zero, then it is guaranteed that the +** Fts3Phrase.doclist.aAll/nAll variables contain the entire doclist for +** each phrase in the expression (subject to deferred token processing). +** Or, if bOptOk is non-zero, then one or more tokens within the expression +** may be loaded incrementally, meaning doclist.aAll/nAll is not available. +** +** If an error occurs within this function, *pRc is set to an SQLite error +** code before returning. +*/ static void fts3EvalStartReaders( - Fts3Cursor *pCsr, - Fts3Expr *pExpr, - int bOptOk, - int *pRc + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pExpr, /* Expression to initialize phrases in */ + int bOptOk, /* True to enable incremental loading */ + int *pRc /* IN/OUT: Error code */ ){ if( pExpr && SQLITE_OK==*pRc ){ if( pExpr->eType==FTSQUERY_PHRASE ){ @@ -3557,23 +3717,42 @@ static void fts3EvalStartReaders( } } +/* +** An array of the following structures is assembled as part of the process +** of selecting tokens to defer before the query starts executing (as part +** of the xFilter() method). There is one element in the array for each +** token in the FTS expression. +** +** Tokens are divided into AND/NEAR clusters. All tokens in a cluster belong +** to phrases that are connected only by AND and NEAR operators (not OR or +** NOT). When determining tokens to defer, each AND/NEAR cluster is considered +** separately. The root of a tokens AND/NEAR cluster is stored in +** Fts3TokenAndCost.pRoot. +*/ typedef struct Fts3TokenAndCost Fts3TokenAndCost; struct Fts3TokenAndCost { Fts3Phrase *pPhrase; /* The phrase the token belongs to */ int iToken; /* Position of token in phrase */ Fts3PhraseToken *pToken; /* The token itself */ - Fts3Expr *pRoot; - int nOvfl; + Fts3Expr *pRoot; /* Root of NEAR/AND cluster */ + int nOvfl; /* Number of overflow pages to load doclist */ int iCol; /* The column the token must match */ }; +/* +** This function is used to populate an allocated Fts3TokenAndCost array. +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, if an error occurs during execution, *pRc is set to an +** SQLite error code. +*/ static void fts3EvalTokenCosts( - Fts3Cursor *pCsr, - Fts3Expr *pRoot, - Fts3Expr *pExpr, - Fts3TokenAndCost **ppTC, - Fts3Expr ***ppOr, - int *pRc + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pRoot, /* Root of current AND/NEAR cluster */ + Fts3Expr *pExpr, /* Expression to consider */ + Fts3TokenAndCost **ppTC, /* Write new entries to *(*ppTC)++ */ + Fts3Expr ***ppOr, /* Write new OR root to *(*ppOr)++ */ + int *pRc /* IN/OUT: Error code */ ){ if( *pRc==SQLITE_OK && pExpr ){ if( pExpr->eType==FTSQUERY_PHRASE ){ @@ -3605,19 +3784,30 @@ static void fts3EvalTokenCosts( } } +/* +** Determine the average document (row) size in pages. If successful, +** write this value to *pnPage and return SQLITE_OK. Otherwise, return +** an SQLite error code. +** +** The average document size in pages is calculated by first calculating +** determining the average size in bytes, B. If B is less than the amount +** of data that will fit on a single leaf page of an intkey table in +** this database, then the average docsize is 1. Otherwise, it is 1 plus +** the number of overflow pages consumed by a record B bytes in size. +*/ static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){ if( pCsr->nRowAvg==0 ){ /* The average document size, which is required to calculate the cost - ** of each doclist, has not yet been determined. Read the required - ** data from the %_stat table to calculate it. - ** - ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 - ** varints, where nCol is the number of columns in the FTS3 table. - ** The first varint is the number of documents currently stored in - ** the table. The following nCol varints contain the total amount of - ** data stored in all rows of each column of the table, from left - ** to right. - */ + ** of each doclist, has not yet been determined. Read the required + ** data from the %_stat table to calculate it. + ** + ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 + ** varints, where nCol is the number of columns in the FTS3 table. + ** The first varint is the number of documents currently stored in + ** the table. The following nCol varints contain the total amount of + ** data stored in all rows of each column of the table, from left + ** to right. + */ int rc; Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; sqlite3_stmt *pStmt; @@ -3652,68 +3842,117 @@ static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){ return SQLITE_OK; } +/* +** This function is called to select the tokens (if any) that will be +** deferred. The array aTC[] has already been populated when this is +** called. +** +** This function is called once for each AND/NEAR cluster in the +** expression. Each invocation determines which tokens to defer within +** the cluster with root node pRoot. See comments above the definition +** of struct Fts3TokenAndCost for more details. +** +** If no error occurs, SQLITE_OK is returned and sqlite3Fts3DeferToken() +** called on each token to defer. Otherwise, an SQLite error code is +** returned. +*/ static int fts3EvalSelectDeferred( - Fts3Cursor *pCsr, - Fts3Expr *pRoot, - Fts3TokenAndCost *aTC, - int nTC + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pRoot, /* Consider tokens with this root node */ + Fts3TokenAndCost *aTC, /* Array of expression tokens and costs */ + int nTC /* Number of entries in aTC[] */ ){ - int nDocSize = 0; - int nDocEst = 0; - int rc = SQLITE_OK; Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - int ii; + int nDocSize = 0; /* Number of pages per doc loaded */ + int rc = SQLITE_OK; /* Return code */ + int ii; /* Iterator variable for various purposes */ + int nOvfl = 0; /* Total overflow pages used by doclists */ + int nToken = 0; /* Total number of tokens in cluster */ - int nOvfl = 0; - int nTerm = 0; + int nMinEst = 0; /* The minimum count for any phrase so far. */ + int nLoad4 = 1; /* (Phrases that will be loaded)^4. */ + /* Count the tokens in this AND/NEAR cluster. If none of the doclists + ** associated with the tokens spill onto overflow pages, or if there is + ** only 1 token, exit early. No tokens to defer in this case. */ for(ii=0; ii0 ); - for(ii=0; iinOvfl) + /* Iterate through all tokens in this AND/NEAR cluster, in ascending order + ** of the number of overflow pages that will be loaded by the pager layer + ** to retrieve the entire doclist for the token from the full-text index. + ** Load the doclists for tokens that are either: + ** + ** a. The cheapest token in the entire query (i.e. the one visited by the + ** first iteration of this loop), or + ** + ** b. Part of a multi-token phrase. + ** + ** After each token doclist is loaded, merge it with the others from the + ** same phrase and count the number of documents that the merged doclist + ** contains. Set variable "nMinEst" to the smallest number of documents in + ** any phrase doclist for which 1 or more token doclists have been loaded. + ** Let nOther be the number of other phrases for which it is certain that + ** one or more tokens will not be deferred. + ** + ** Then, for each token, defer it if loading the doclist would result in + ** loading N or more overflow pages into memory, where N is computed as: + ** + ** (nMinEst + 4^nOther - 1) / (4^nOther) + */ + for(ii=0; iinOvfl) ){ - pTC = &aTC[jj]; + pTC = &aTC[iTC]; } } assert( pTC ); - /* At this point pTC points to the cheapest remaining token. */ - if( ii==0 ){ - if( pTC->nOvfl ){ - nDocEst = (pTC->nOvfl * pTab->nPgsz + pTab->nPgsz) / 10; - }else{ + if( ii && pTC->nOvfl>=((nMinEst+(nLoad4/4)-1)/(nLoad4/4))*nDocSize ){ + /* The number of overflow pages to load for this (and therefore all + ** subsequent) tokens is greater than the estimated number of pages + ** that will be loaded if all subsequent tokens are deferred. + */ + Fts3PhraseToken *pToken = pTC->pToken; + rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol); + fts3SegReaderCursorFree(pToken->pSegcsr); + pToken->pSegcsr = 0; + }else{ + nLoad4 = nLoad4*4; + if( ii==0 || pTC->pPhrase->nToken>1 ){ + /* Either this is the cheapest token in the entire query, or it is + ** part of a multi-token phrase. Either way, the entire doclist will + ** (eventually) be loaded into memory. It may as well be now. */ Fts3PhraseToken *pToken = pTC->pToken; int nList = 0; char *pList = 0; - rc = fts3TermSelect(pTab, pToken, pTC->iCol, 1, &nList, &pList); + rc = fts3TermSelect(pTab, pToken, pTC->iCol, &nList, &pList); assert( rc==SQLITE_OK || pList==0 ); - if( rc==SQLITE_OK ){ - nDocEst = fts3DoclistCountDocids(1, pList, nList); + int nCount; fts3EvalPhraseMergeToken(pTab, pTC->pPhrase, pTC->iToken,pList,nList); + nCount = fts3DoclistCountDocids( + pTC->pPhrase->doclist.aAll, pTC->pPhrase->doclist.nAll + ); + if( ii==0 || nCountnOvfl>=(nDocEst*nDocSize) ){ - Fts3PhraseToken *pToken = pTC->pToken; - rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol); - fts3SegReaderCursorFree(pToken->pSegcsr); - pToken->pSegcsr = 0; - } - nDocEst = 1 + (nDocEst/4); } pTC->pToken = 0; } @@ -3721,36 +3960,29 @@ static int fts3EvalSelectDeferred( return rc; } -int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){ +/* +** This function is called from within the xFilter method. It initializes +** the full-text query currently stored in pCsr->pExpr. To iterate through +** the results of a query, the caller does: +** +** fts3EvalStart(pCsr); +** while( 1 ){ +** fts3EvalNext(pCsr); +** if( pCsr->bEof ) break; +** ... return row pCsr->iPrevId to the caller ... +** } +*/ +static int fts3EvalStart(Fts3Cursor *pCsr){ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; int rc = SQLITE_OK; int nToken = 0; int nOr = 0; /* Allocate a MultiSegReader for each token in the expression. */ - fts3EvalAllocateReaders(pCsr, pExpr, &nToken, &nOr, &rc); + fts3EvalAllocateReaders(pCsr, pCsr->pExpr, &nToken, &nOr, &rc); - /* Call fts3EvalPhraseStart() on all phrases in the expression. TODO: - ** This call will eventually also be responsible for determining which - ** tokens are 'deferred' until the document text is loaded into memory. - ** - ** Each token in each phrase is dealt with using one of the following - ** three strategies: - ** - ** 1. Entire doclist loaded into memory as part of the - ** fts3EvalStartReaders() call. - ** - ** 2. Doclist loaded into memory incrementally, as part of each - ** sqlite3Fts3EvalNext() call. - ** - ** 3. Token doclist is never loaded. Instead, documents are loaded into - ** memory and scanned for the token as part of the sqlite3Fts3EvalNext() - ** call. This is known as a "deferred" token. - */ - - /* If bOptOk is true, check if there are any tokens that should be deferred. - */ - if( rc==SQLITE_OK && bOptOk && nToken>1 && pTab->bHasStat ){ + /* Determine which, if any, tokens in the expression should be deferred. */ + if( rc==SQLITE_OK && nToken>1 && pTab->bHasStat ){ Fts3TokenAndCost *aTC; Fts3Expr **apOr; aTC = (Fts3TokenAndCost *)sqlite3_malloc( @@ -3766,7 +3998,7 @@ int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){ Fts3TokenAndCost *pTC = aTC; Fts3Expr **ppOr = apOr; - fts3EvalTokenCosts(pCsr, 0, pExpr, &pTC, &ppOr, &rc); + fts3EvalTokenCosts(pCsr, 0, pCsr->pExpr, &pTC, &ppOr, &rc); nToken = pTC-aTC; nOr = ppOr-apOr; @@ -3781,11 +4013,14 @@ int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){ } } - fts3EvalStartReaders(pCsr, pExpr, bOptOk, &rc); + fts3EvalStartReaders(pCsr, pCsr->pExpr, 1, &rc); return rc; } -static void fts3EvalZeroPoslist(Fts3Phrase *pPhrase){ +/* +** Invalidate the current position list for phrase pPhrase. +*/ +static void fts3EvalInvalidatePoslist(Fts3Phrase *pPhrase){ if( pPhrase->doclist.bFreeList ){ sqlite3_free(pPhrase->doclist.pList); } @@ -3794,8 +4029,30 @@ static void fts3EvalZeroPoslist(Fts3Phrase *pPhrase){ pPhrase->doclist.bFreeList = 0; } -static int fts3EvalNearTrim2( - int nNear, +/* +** This function is called to edit the position list associated with +** the phrase object passed as the fifth argument according to a NEAR +** condition. For example: +** +** abc NEAR/5 "def ghi" +** +** Parameter nNear is passed the NEAR distance of the expression (5 in +** the example above). When this function is called, *paPoslist points to +** the position list, and *pnToken is the number of phrase tokens in, the +** phrase on the other side of the NEAR operator to pPhrase. For example, +** if pPhrase refers to the "def ghi" phrase, then *paPoslist points to +** the position list associated with phrase "abc". +** +** All positions in the pPhrase position list that are not sufficiently +** close to a position in the *paPoslist position list are removed. If this +** leaves 0 positions, zero is returned. Otherwise, non-zero. +** +** Before returning, *paPoslist is set to point to the position lsit +** associated with pPhrase. And *pnToken is set to the number of tokens in +** pPhrase. +*/ +static int fts3EvalNearTrim( + int nNear, /* NEAR distance. As in "NEAR/nNear". */ char *aTmp, /* Temporary space to use */ char **paPoslist, /* IN/OUT: Position list */ int *pnToken, /* IN/OUT: Tokens in phrase of *paPoslist */ @@ -3827,6 +4084,172 @@ static int fts3EvalNearTrim2( return res; } +/* +** This function is a no-op if *pRc is other than SQLITE_OK when it is called. +** Otherwise, it advances the expression passed as the second argument to +** point to the next matching row in the database. Expressions iterate through +** matching rows in docid order. Ascending order if Fts3Cursor.bDesc is zero, +** or descending if it is non-zero. +** +** If an error occurs, *pRc is set to an SQLite error code. Otherwise, if +** successful, the following variables in pExpr are set: +** +** Fts3Expr.bEof (non-zero if EOF - there is no next row) +** Fts3Expr.iDocid (valid if bEof==0. The docid of the next row) +** +** If the expression is of type FTSQUERY_PHRASE, and the expression is not +** at EOF, then the following variables are populated with the position list +** for the phrase for the visited row: +** +** FTs3Expr.pPhrase->doclist.nList (length of pList in bytes) +** FTs3Expr.pPhrase->doclist.pList (pointer to position list) +** +** It says above that this function advances the expression to the next +** matching row. This is usually true, but there are the following exceptions: +** +** 1. Deferred tokens are not taken into account. If a phrase consists +** entirely of deferred tokens, it is assumed to match every row in +** the db. In this case the position-list is not populated at all. +** +** Or, if a phrase contains one or more deferred tokens and one or +** more non-deferred tokens, then the expression is advanced to the +** next possible match, considering only non-deferred tokens. In other +** words, if the phrase is "A B C", and "B" is deferred, the expression +** is advanced to the next row that contains an instance of "A * C", +** where "*" may match any single token. The position list in this case +** is populated as for "A * C" before returning. +** +** 2. NEAR is treated as AND. If the expression is "x NEAR y", it is +** advanced to point to the next row that matches "x AND y". +** +** See fts3EvalTestDeferredAndNear() for details on testing if a row is +** really a match, taking into account deferred tokens and NEAR operators. +*/ +static void fts3EvalNextRow( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pExpr, /* Expr. to advance to next matching row */ + int *pRc /* IN/OUT: Error code */ +){ + if( *pRc==SQLITE_OK ){ + int bDescDoclist = pCsr->bDesc; /* Used by DOCID_CMP() macro */ + assert( pExpr->bEof==0 ); + pExpr->bStart = 1; + + switch( pExpr->eType ){ + case FTSQUERY_NEAR: + case FTSQUERY_AND: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + assert( !pLeft->bDeferred || !pRight->bDeferred ); + + if( pLeft->bDeferred ){ + /* LHS is entirely deferred. So we assume it matches every row. + ** Advance the RHS iterator to find the next row visited. */ + fts3EvalNextRow(pCsr, pRight, pRc); + pExpr->iDocid = pRight->iDocid; + pExpr->bEof = pRight->bEof; + }else if( pRight->bDeferred ){ + /* RHS is entirely deferred. So we assume it matches every row. + ** Advance the LHS iterator to find the next row visited. */ + fts3EvalNextRow(pCsr, pLeft, pRc); + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = pLeft->bEof; + }else{ + /* Neither the RHS or LHS are deferred. */ + fts3EvalNextRow(pCsr, pLeft, pRc); + fts3EvalNextRow(pCsr, pRight, pRc); + while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){ + sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + if( iDiff==0 ) break; + if( iDiff<0 ){ + fts3EvalNextRow(pCsr, pLeft, pRc); + }else{ + fts3EvalNextRow(pCsr, pRight, pRc); + } + } + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = (pLeft->bEof || pRight->bEof); + } + break; + } + + case FTSQUERY_OR: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + + assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid ); + assert( pRight->bStart || pLeft->iDocid==pRight->iDocid ); + + if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ + fts3EvalNextRow(pCsr, pLeft, pRc); + }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){ + fts3EvalNextRow(pCsr, pRight, pRc); + }else{ + fts3EvalNextRow(pCsr, pLeft, pRc); + fts3EvalNextRow(pCsr, pRight, pRc); + } + + pExpr->bEof = (pLeft->bEof && pRight->bEof); + iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ + pExpr->iDocid = pLeft->iDocid; + }else{ + pExpr->iDocid = pRight->iDocid; + } + + break; + } + + case FTSQUERY_NOT: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + + if( pRight->bStart==0 ){ + fts3EvalNextRow(pCsr, pRight, pRc); + assert( *pRc!=SQLITE_OK || pRight->bStart ); + } + + fts3EvalNextRow(pCsr, pLeft, pRc); + if( pLeft->bEof==0 ){ + while( !*pRc + && !pRight->bEof + && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 + ){ + fts3EvalNextRow(pCsr, pRight, pRc); + } + } + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = pLeft->bEof; + break; + } + + default: { + Fts3Phrase *pPhrase = pExpr->pPhrase; + fts3EvalInvalidatePoslist(pPhrase); + *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof); + pExpr->iDocid = pPhrase->doclist.iDocid; + break; + } + } + } +} + +/* +** If *pRc is not SQLITE_OK, or if pExpr is not the root node of a NEAR +** cluster, then this function returns 1 immediately. +** +** Otherwise, it checks if the current row really does match the NEAR +** expression, using the data currently stored in the position lists +** (Fts3Expr->pPhrase.doclist.pList/nList) for each phrase in the expression. +** +** If the current row is a match, the position list associated with each +** phrase in the NEAR expression is edited in place to contain only those +** phrase instances sufficiently close to their peers to satisfy all NEAR +** constraints. In this case it returns 1. If the NEAR expression does not +** match the current row, 0 is returned. The position lists may or may not +** be edited if 0 is returned. +*/ static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ int res = 1; @@ -3848,7 +4271,7 @@ static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ ** ** The right-hand child of a NEAR node is always a phrase. The ** left-hand child may be either a phrase or a NEAR node. There are - ** no exceptions to this. + ** no exceptions to this - it's the way the parser in fts3_expr.c works. */ if( *pRc==SQLITE_OK && pExpr->eType==FTSQUERY_NEAR @@ -3875,7 +4298,7 @@ static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){ Fts3Phrase *pPhrase = p->pRight->pPhrase; int nNear = p->nNear; - res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase); + res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); } aPoslist = pExpr->pRight->pPhrase->doclist.pList; @@ -3885,7 +4308,7 @@ static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ Fts3Phrase *pPhrase = ( p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase ); - res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase); + res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); } } @@ -3896,128 +4319,29 @@ static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ } /* -** This macro is used by the fts3EvalNext() function. The two arguments are -** 64-bit docid values. If the current query is "ORDER BY docid ASC", then -** the macro returns (i1 - i2). Or if it is "ORDER BY docid DESC", then -** it returns (i2 - i1). This allows the same code to be used for merging -** doclists in ascending or descending order. +** This function is a helper function for fts3EvalTestDeferredAndNear(). +** Assuming no error occurs or has occurred, It returns non-zero if the +** expression passed as the second argument matches the row that pCsr +** currently points to, or zero if it does not. +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** If an error occurs during execution of this function, *pRc is set to +** the appropriate SQLite error code. In this case the returned value is +** undefined. */ -#define DOCID_CMP(i1, i2) ((pCsr->bDesc?-1:1) * (i1-i2)) - -static void fts3EvalNext( - Fts3Cursor *pCsr, - Fts3Expr *pExpr, - int *pRc +static int fts3EvalTestExpr( + Fts3Cursor *pCsr, /* FTS cursor handle */ + Fts3Expr *pExpr, /* Expr to test. May or may not be root. */ + int *pRc /* IN/OUT: Error code */ ){ - if( *pRc==SQLITE_OK ){ - assert( pExpr->bEof==0 ); - pExpr->bStart = 1; - - switch( pExpr->eType ){ - case FTSQUERY_NEAR: - case FTSQUERY_AND: { - Fts3Expr *pLeft = pExpr->pLeft; - Fts3Expr *pRight = pExpr->pRight; - assert( !pLeft->bDeferred || !pRight->bDeferred ); - if( pLeft->bDeferred ){ - fts3EvalNext(pCsr, pRight, pRc); - pExpr->iDocid = pRight->iDocid; - pExpr->bEof = pRight->bEof; - }else if( pRight->bDeferred ){ - fts3EvalNext(pCsr, pLeft, pRc); - pExpr->iDocid = pLeft->iDocid; - pExpr->bEof = pLeft->bEof; - }else{ - fts3EvalNext(pCsr, pLeft, pRc); - fts3EvalNext(pCsr, pRight, pRc); - - while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){ - sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid); - if( iDiff==0 ) break; - if( iDiff<0 ){ - fts3EvalNext(pCsr, pLeft, pRc); - }else{ - fts3EvalNext(pCsr, pRight, pRc); - } - } - - pExpr->iDocid = pLeft->iDocid; - pExpr->bEof = (pLeft->bEof || pRight->bEof); - } - break; - } - - case FTSQUERY_OR: { - Fts3Expr *pLeft = pExpr->pLeft; - Fts3Expr *pRight = pExpr->pRight; - sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); - - assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid ); - assert( pRight->bStart || pLeft->iDocid==pRight->iDocid ); - - if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ - fts3EvalNext(pCsr, pLeft, pRc); - }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){ - fts3EvalNext(pCsr, pRight, pRc); - }else{ - fts3EvalNext(pCsr, pLeft, pRc); - fts3EvalNext(pCsr, pRight, pRc); - } - - pExpr->bEof = (pLeft->bEof && pRight->bEof); - iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); - if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ - pExpr->iDocid = pLeft->iDocid; - }else{ - pExpr->iDocid = pRight->iDocid; - } - - break; - } - - case FTSQUERY_NOT: { - Fts3Expr *pLeft = pExpr->pLeft; - Fts3Expr *pRight = pExpr->pRight; - - if( pRight->bStart==0 ){ - fts3EvalNext(pCsr, pRight, pRc); - assert( *pRc!=SQLITE_OK || pRight->bStart ); - } - - fts3EvalNext(pCsr, pLeft, pRc); - if( pLeft->bEof==0 ){ - while( !*pRc - && !pRight->bEof - && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 - ){ - fts3EvalNext(pCsr, pRight, pRc); - } - } - pExpr->iDocid = pLeft->iDocid; - pExpr->bEof = pLeft->bEof; - break; - } - - default: { - Fts3Phrase *pPhrase = pExpr->pPhrase; - fts3EvalZeroPoslist(pPhrase); - *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof); - pExpr->iDocid = pPhrase->doclist.iDocid; - break; - } - } - } -} - -static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ - int bHit = 1; + int bHit = 1; /* Return value */ if( *pRc==SQLITE_OK ){ switch( pExpr->eType ){ case FTSQUERY_NEAR: case FTSQUERY_AND: bHit = ( - fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc) - && fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc) + fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc) + && fts3EvalTestExpr(pCsr, pExpr->pRight, pRc) && fts3EvalNearTest(pExpr, pRc) ); @@ -4043,27 +4367,27 @@ static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ Fts3Expr *p; for(p=pExpr; p->pPhrase==0; p=p->pLeft){ if( p->pRight->iDocid==pCsr->iPrevId ){ - fts3EvalZeroPoslist(p->pRight->pPhrase); + fts3EvalInvalidatePoslist(p->pRight->pPhrase); } } if( p->iDocid==pCsr->iPrevId ){ - fts3EvalZeroPoslist(p->pPhrase); + fts3EvalInvalidatePoslist(p->pPhrase); } } break; case FTSQUERY_OR: { - int bHit1 = fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc); - int bHit2 = fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc); + int bHit1 = fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc); + int bHit2 = fts3EvalTestExpr(pCsr, pExpr->pRight, pRc); bHit = bHit1 || bHit2; break; } case FTSQUERY_NOT: bHit = ( - fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc) - && !fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc) + fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc) + && !fts3EvalTestExpr(pCsr, pExpr->pRight, pRc) ); break; @@ -4074,7 +4398,7 @@ static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ Fts3Phrase *pPhrase = pExpr->pPhrase; assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 ); if( pExpr->bDeferred ){ - fts3EvalZeroPoslist(pPhrase); + fts3EvalInvalidatePoslist(pPhrase); } *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase); bHit = (pPhrase->doclist.pList!=0); @@ -4090,27 +4414,49 @@ static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ } /* -** Return 1 if both of the following are true: +** This function is called as the second part of each xNext operation when +** iterating through the results of a full-text query. At this point the +** cursor points to a row that matches the query expression, with the +** following caveats: +** +** * Up until this point, "NEAR" operators in the expression have been +** treated as "AND". +** +** * Deferred tokens have not yet been considered. +** +** If *pRc is not SQLITE_OK when this function is called, it immediately +** returns 0. Otherwise, it tests whether or not after considering NEAR +** operators and deferred tokens the current row is still a match for the +** expression. It returns 1 if both of the following are true: ** ** 1. *pRc is SQLITE_OK when this function returns, and ** ** 2. After scanning the current FTS table row for the deferred tokens, -** it is determined that the row does not match the query. +** it is determined that the row does *not* match the query. ** ** Or, if no error occurs and it seems the current row does match the FTS ** query, return 0. */ -static int fts3EvalLoadDeferred(Fts3Cursor *pCsr, int *pRc){ +static int fts3EvalTestDeferredAndNear(Fts3Cursor *pCsr, int *pRc){ int rc = *pRc; int bMiss = 0; if( rc==SQLITE_OK ){ + + /* If there are one or more deferred tokens, load the current row into + ** memory and scan it to determine the position list for each deferred + ** token. Then, see if this row is really a match, considering deferred + ** tokens and NEAR operators (neither of which were taken into account + ** earlier, by fts3EvalNextRow()). + */ if( pCsr->pDeferred ){ rc = fts3CursorSeek(0, pCsr); if( rc==SQLITE_OK ){ rc = sqlite3Fts3CacheDeferredDoclists(pCsr); } } - bMiss = (0==fts3EvalDeferredTest(pCsr, pCsr->pExpr, &rc)); + bMiss = (0==fts3EvalTestExpr(pCsr, pCsr->pExpr, &rc)); + + /* Free the position-lists accumulated for each deferred token above. */ sqlite3Fts3FreeDeferredDoclists(pCsr); *pRc = rc; } @@ -4121,7 +4467,7 @@ static int fts3EvalLoadDeferred(Fts3Cursor *pCsr, int *pRc){ ** Advance to the next document that matches the FTS expression in ** Fts3Cursor.pExpr. */ -int sqlite3Fts3EvalNext(Fts3Cursor *pCsr){ +static int fts3EvalNext(Fts3Cursor *pCsr){ int rc = SQLITE_OK; /* Return Code */ Fts3Expr *pExpr = pCsr->pExpr; assert( pCsr->isEof==0 ); @@ -4133,19 +4479,19 @@ int sqlite3Fts3EvalNext(Fts3Cursor *pCsr){ sqlite3_reset(pCsr->pStmt); } assert( sqlite3_data_count(pCsr->pStmt)==0 ); - fts3EvalNext(pCsr, pExpr, &rc); + fts3EvalNextRow(pCsr, pExpr, &rc); pCsr->isEof = pExpr->bEof; pCsr->isRequireSeek = 1; pCsr->isMatchinfoNeeded = 1; pCsr->iPrevId = pExpr->iDocid; - }while( pCsr->isEof==0 && fts3EvalLoadDeferred(pCsr, &rc) ); + }while( pCsr->isEof==0 && fts3EvalTestDeferredAndNear(pCsr, &rc) ); } return rc; } /* ** Restart interation for expression pExpr so that the next call to -** sqlite3Fts3EvalNext() visits the first row. Do not allow incremental +** fts3EvalNext() visits the first row. Do not allow incremental ** loading or merging of phrase doclists for this iteration. ** ** If *pRc is other than SQLITE_OK when this function is called, it is @@ -4161,7 +4507,7 @@ static void fts3EvalRestart( Fts3Phrase *pPhrase = pExpr->pPhrase; if( pPhrase ){ - fts3EvalZeroPoslist(pPhrase); + fts3EvalInvalidatePoslist(pPhrase); if( pPhrase->bIncr ){ assert( pPhrase->nToken==1 ); assert( pPhrase->aToken[0].pSegcsr ); @@ -4277,14 +4623,14 @@ static int fts3EvalGatherStats( assert( sqlite3_data_count(pCsr->pStmt)==0 ); /* Advance to the next document */ - fts3EvalNext(pCsr, pRoot, &rc); + fts3EvalNextRow(pCsr, pRoot, &rc); pCsr->isEof = pRoot->bEof; pCsr->isRequireSeek = 1; pCsr->isMatchinfoNeeded = 1; pCsr->iPrevId = pRoot->iDocid; }while( pCsr->isEof==0 && pRoot->eType==FTSQUERY_NEAR - && fts3EvalLoadDeferred(pCsr, &rc) + && fts3EvalTestDeferredAndNear(pCsr, &rc) ); if( rc==SQLITE_OK && pCsr->isEof==0 ){ @@ -4306,10 +4652,10 @@ static int fts3EvalGatherStats( */ fts3EvalRestart(pCsr, pRoot, &rc); do { - fts3EvalNext(pCsr, pRoot, &rc); + fts3EvalNextRow(pCsr, pRoot, &rc); assert( pRoot->bEof==0 ); }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK ); - fts3EvalLoadDeferred(pCsr, &rc); + fts3EvalTestDeferredAndNear(pCsr, &rc); } } return rc; @@ -4440,7 +4786,7 @@ void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){ if( pPhrase ){ int i; sqlite3_free(pPhrase->doclist.aAll); - fts3EvalZeroPoslist(pPhrase); + fts3EvalInvalidatePoslist(pPhrase); memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); for(i=0; inToken; i++){ fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr); @@ -4449,4 +4795,18 @@ void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){ } } +#if !SQLITE_CORE +/* +** Initialize API pointer table, if required. +*/ +int sqlite3_extension_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi) + return sqlite3Fts3Init(db); +} +#endif + #endif diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 1664379609..ed8043adf6 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -27,7 +27,14 @@ # define SQLITE_ENABLE_FTS3 #endif -#ifdef SQLITE_ENABLE_FTS3 +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* If not building as part of the core, include sqlite3ext.h. */ +#ifndef SQLITE_CORE +# include "sqlite3ext.h" +extern const sqlite3_api_routines *sqlite3_api; +#endif + #include "sqlite3.h" #include "fts3_tokenizer.h" #include "fts3_hash.h" @@ -290,7 +297,7 @@ struct Fts3Doclist { int bFreeList; /* True if pList should be sqlite3_free()d */ char *pList; /* Pointer to position list following iDocid */ int nList; /* Length of position list */ -} doclist; +}; /* ** A "phrase" is a sequence of one or more tokens that must match in @@ -490,19 +497,8 @@ int sqlite3Fts3InitTerm(sqlite3 *db); /* fts3_aux.c */ int sqlite3Fts3InitAux(sqlite3 *db); -int sqlite3Fts3TermSegReaderCursor( - Fts3Cursor *pCsr, /* Virtual table cursor handle */ - const char *zTerm, /* Term to query for */ - int nTerm, /* Size of zTerm in bytes */ - int isPrefix, /* True for a prefix search */ - Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */ -); - void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *); -int sqlite3Fts3EvalStart(Fts3Cursor *, Fts3Expr *, int); -int sqlite3Fts3EvalNext(Fts3Cursor *pCsr); - int sqlite3Fts3MsrIncrStart( Fts3Table*, Fts3MultiSegReader*, int, const char*, int); int sqlite3Fts3MsrIncrNext( @@ -513,5 +509,5 @@ int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *); -#endif /* SQLITE_ENABLE_FTS3 */ +#endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */ #endif /* _FTSINT_H */ diff --git a/ext/fts3/fts3_hash.c b/ext/fts3/fts3_hash.c index b7c3e8b5a7..57c59b587a 100644 --- a/ext/fts3/fts3_hash.c +++ b/ext/fts3/fts3_hash.c @@ -30,7 +30,6 @@ #include #include -#include "sqlite3.h" #include "fts3_hash.h" /* diff --git a/ext/fts3/fts3_test.c b/ext/fts3/fts3_test.c index c5f72d84f4..72735f3d12 100644 --- a/ext/fts3/fts3_test.c +++ b/ext/fts3/fts3_test.c @@ -19,6 +19,8 @@ #include #include +#ifdef SQLITE_TEST + /* Required so that the "ifdef SQLITE_ENABLE_FTS3" below works */ #include "fts3Int.h" @@ -319,3 +321,4 @@ int Sqlitetestfts3_Init(Tcl_Interp *interp){ ); return TCL_OK; } +#endif /* ifdef SQLITE_TEST */ diff --git a/ext/fts3/fts3_tokenizer.c b/ext/fts3/fts3_tokenizer.c index c4ad6a5c75..6494bb96d8 100644 --- a/ext/fts3/fts3_tokenizer.c +++ b/ext/fts3/fts3_tokenizer.c @@ -23,12 +23,7 @@ ** * The FTS3 module is being built into the core of ** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). */ -#include "sqlite3ext.h" -#ifndef SQLITE_CORE - SQLITE_EXTENSION_INIT1 -#endif #include "fts3Int.h" - #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) #include diff --git a/main.mk b/main.mk index b5dfc22511..9128e159a0 100644 --- a/main.mk +++ b/main.mk @@ -386,6 +386,9 @@ sqlite3.c: target_source $(TOP)/tool/mksqlite3c.tcl echo '#endif /* USE_SYSTEM_SQLITE */' >>tclsqlite3.c cat $(TOP)/src/tclsqlite.c >>tclsqlite3.c +sqlite3-all.c: sqlite3.c $(TOP)/tool/split-sqlite3c.tcl + tclsh $(TOP)/tool/split-sqlite3c.tcl + fts2amal.c: target_source $(TOP)/ext/fts2/mkfts2amal.tcl tclsh $(TOP)/ext/fts2/mkfts2amal.tcl @@ -568,6 +571,13 @@ $(TEST_EXTENSION): $(TOP)/src/test_loadext.c extensiontest: testfixture$(EXE) $(TEST_EXTENSION) ./testfixture$(EXE) $(TOP)/test/loadext.test +# This target will fail if the SQLite amalgamation contains any exported +# symbols that do not begin with "sqlite3_". It is run as part of the +# releasetest.tcl script. +# +checksymbols: sqlite3.o + nm -g --defined-only sqlite3.o | grep -v " sqlite3_" ; test $$? -ne 0 + # Standard install and cleanup targets # diff --git a/manifest b/manifest index a3d6a2cab4..4e10bb5947 100644 --- a/manifest +++ b/manifest @@ -1,12 +1,12 @@ -C Fix\sthe\ssqlite3session_isempty()\smethod\sso\sthat\sit\sreturns,\sas\sdocumented,\snon-zero\swhen\sno\schanges\shave\sbeen\srecorded\sby\sthe\ssession\sobject. -D 2011-07-18T15:22:56.414 +C Merge\sthe\slatest\strunk\schanges\sinto\sthe\ssessions\sbranch. +D 2011-07-22T12:49:27.667 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e +F Makefile.in 1e6988b3c11dee9bd5edc0c804bd4468d74a9cdc F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc 11082f65b452b908d93013292c17850378c39284 +F Makefile.msc 8e04f517922b9e31a5bf975f07bb66df79ae7076 F Makefile.vxworks c85ec1d8597fe2f7bc225af12ac1666e21379151 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 -F VERSION 3fcdd7fbe3eb282df3978fe77288544543767961 +F VERSION f724de7326e87b7f3b0a55f16ef4b4d993680d54 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F addopcodes.awk 17dc593f791f874d2c23a0f9360850ded0286531 F art/2005osaward.gif 0d1851b2a7c1c9d0ccce545f3e14bca42d7fd248 @@ -23,7 +23,7 @@ F art/src_logo.gif 9341ef09f0e53cd44c0c9b6fc3c16f7f3d6c2ad9 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977 F config.h.in 405a958bdb3af382a809dccb08a44694923ddd61 F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55 -F configure f9e97ee7cdc9848e2f3f5ef015fdf861f46fb1bf x +F configure 93e7e695581fa7bef4949161453d9845c5592ad0 x F configure.ac 298a759c086e72c013da459c2aec02a104f4224f F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/lemon.html f0f682f50210928c07e562621c3b7e8ab912a538 @@ -62,19 +62,19 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c ca776037493d0081da70a6afc0df80f5ce347cb7 +F ext/fts3/fts3.c f45ad45053a587ad1c005459b704b7ade8bd504e F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h fa493ccbad78a2c99ad1c984f651c0c202e68536 +F ext/fts3/fts3Int.h 30063fdd0bc433b5db1532e3a363cb0f2f7e8eb3 F ext/fts3/fts3_aux.c 0ebfa7b86cf8ff6a0861605fcc63b83ec1b70691 F ext/fts3/fts3_expr.c 23791de01b3a5d313d76e02befd2601d4096bc2b -F ext/fts3/fts3_hash.c aad95afa01cf2a5ffaa448e4b0ab043880cd1efb +F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec F ext/fts3/fts3_icu.c 6c8f395cdf9e1e3afa7fadb7e523dbbf381c6dfa F ext/fts3/fts3_porter.c 8d946908f4812c005d3d33fcbe78418b1f4eb70c F ext/fts3/fts3_snippet.c 58b2ba2b934c1e2a2f6ac857d7f3c7e1a14b4532 F ext/fts3/fts3_term.c a5457992723455a58804cb75c8cbd8978db5c2ef -F ext/fts3/fts3_test.c c0aae3219df989d3e827d07846097adf8cb09945 -F ext/fts3/fts3_tokenizer.c 6089986ebcfc19d4b7aabd2b92773368efa4354f +F ext/fts3/fts3_test.c 24fa13f330db011500acb95590da9eee24951894 +F ext/fts3/fts3_tokenizer.c 9ff7ec66ae3c5c0340fa081958e64f395c71a106 F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer1.c 0dde8f307b8045565cf63797ba9acfaff1c50c68 F ext/fts3/fts3_write.c 194829c8fd024a448fc899e5ff02a8ed06595529 @@ -115,7 +115,7 @@ F ext/session/sqlite3session.h 8dec372049532017d71c992609ca5450de7c5520 F ext/session/test_session.c ea4dc9b4a1895c8e6bddcbfe3838d7eb57df2d99 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F main.mk f56d9895882f5cdd9f9f9ba8f8a679e9202288c1 +F main.mk 47bb833f273c17812370a750b1a697cd14580b75 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac @@ -138,15 +138,15 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 8c46f0ab69ad9549c75a3a91fed87abdaa743e2f F src/btree.h f5d775cd6cfc7ac32a2535b70e8d2af48ef5f2ce F src/btreeInt.h 67978c014fa4f7cc874032dd3aacadd8db656bc3 -F src/build.c 5a428625d21ad409514afb40ad083bee25dd957a +F src/build.c 5e614e586d9f8a81c16c80b545b9e1747f96c1bb F src/callback.c 0425c6320730e6d3981acfb9202c1bed9016ad1a F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 7deec4534f3b5a0c3b4a4cbadf809d321f64f9c4 -F src/date.c d3c11de76392ea62637bfac0f4655889fc2f5a85 -F src/delete.c 7a8683a38e86d48a0e67ef949fddac6daf46d89d -F src/expr.c ab46ab0f0c44979a8164ca31728d7d10ae5e8106 +F src/date.c a3c6842bad7ae632281811de112a8ba63ff08ab3 +F src/delete.c 614d6e012aa5b624e78f3b556243497825de196b +F src/expr.c 4bbdfaf66bc614be9254ce0c26a17429067a3e07 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb -F src/fkey.c 9fabba17a4d4778dc660f0cb9d781fc86d7b9d41 +F src/fkey.c c8492fed772af1ed61251582707266227612b45b F src/func.c 59bb046d7e3df1ab512ac339ccb0a6f996a17cb7 F src/global.c c70a46f28680f8d7c097dbc0430ccf3b932e90b0 F src/hash.c 458488dcc159c301b8e7686280ab209f1fb915af @@ -156,13 +156,13 @@ F src/insert.c 1f1688a9da8b8e27114ba1909a6e74100791139b F src/journal.c 552839e54d1bf76fb8f7abe51868b66acacf6a0e F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e -F src/loadext.c 3ae0d52da013a6326310655be6473fd472347b85 +F src/loadext.c 99a161b27a499fc8ad40745b7b1900a26f0a5f51 F src/main.c 8a386605c22e4b2becfe96bab2ac7128b480d489 F src/malloc.c 591aedb20ae40813f1045f2ef253438a334775d9 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 00bd8265c81abb665c48fea1e0c234eb3b922206 F src/mem2.c e307323e86b5da1853d7111b68fd6b84ad6f09cf -F src/mem3.c 9b237d911ba9904142a804be727cc6664873f8a3 +F src/mem3.c 61c9d47b792908c532ca3a62b999cf21795c6534 F src/mem5.c c2c63b7067570b00bf33d751c39af24182316f7f F src/memjournal.c 0ebce851677a7ac035ba1512a7e65851b34530c6 F src/mutex.c 6949180803ff05a7d0e2b9334a95b4fb5a00e23f @@ -172,12 +172,12 @@ F src/mutex_os2.c 882d735098c07c8c6a5472b8dd66e19675fe117f F src/mutex_unix.c b4f4e923bb8de93ec3f251fadb50855f23df9579 F src/mutex_w32.c 5e54f3ba275bcb5d00248b8c23107df2e2f73e33 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 -F src/os.c 22ac61d06e72a0dac900400147333b07b13d8e1d +F src/os.c fcc717427a80b2ed225373f07b642dc1aad7490b F src/os.h 9dbed8c2b9c1f2f2ebabc09e49829d4777c26bf9 F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 -F src/os_unix.c 07acbb3e074e52b48a4248c06f66c9a91db1a0ce -F src/os_win.c eafcd6b91cf204a7ef29ac1ef2a1b7132e132e58 +F src/os_unix.c dcd6d5782dd30e918dc3d111cdcb1883bfb95345 +F src/os_win.c c5eadb2c0fc11347296a660f77b9844090265c0c F src/pager.c 120550e7ef01dafaa2cbb4a0528c0d87c8f12b41 F src/pager.h 3f8c783de1d4706b40b1ac15b64f5f896bcc78d1 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 @@ -188,18 +188,18 @@ F src/pragma.c ebcd20f1e654f5cb3aeef864ed69c4697719fbaa F src/prepare.c e64261559a3187698a3e7e6c8b001a4f4f98dab4 F src/printf.c 585a36b6a963df832cfb69505afa3a34ed5ef8a1 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 -F src/resolve.c 1c0f32b64f8e3f555fe1f732f9d6f501a7f05706 +F src/resolve.c 36368f44569208fa074e61f4dd0b6c4fb60ca2b4 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 -F src/select.c d9d440809025a58547e39f4f268c2a296bfb56ff -F src/shell.c 0e0173b3e79d956368013e759f084caa7995ecb1 -F src/sqlite.h.in f292d325b466081f8980911316743192cce6832a -F src/sqlite3ext.h c90bd5507099f62043832d73f6425d8d5c5da754 -F src/sqliteInt.h 3c2f3f13d8ea942ddfee1df1b78013566eae8a74 +F src/select.c d219c4b68d603cc734b6f9b1e2780fee12a1fa0d +F src/shell.c bbe7818ff5bc8614105ceb81ad67b8bdc0b671dd +F src/sqlite.h.in 4cd98228ea9cbb0359aaf725fdbebcd37c082c6a +F src/sqlite3ext.h 1a1a4f784aa9c3b00edd287940197de52487cd93 +F src/sqliteInt.h 113559f0c2202c4e648b5d647d7b2409604fb516 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac64842c86cec2fc1a1d0e5c16d3beb8ad332bf F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e -F src/tclsqlite.c 6a34149c9ff4a6f998a4cb8336d4481670b576b6 -F src/test1.c 8eef932f1024f1076a8adb8a6f5d9c5267c9e969 +F src/tclsqlite.c e85bada4bf1796274a2cd7b3db9a663f8df31880 +F src/test1.c 693d9a63dfe2c68b167080c99cab82f267f5a38e F src/test2.c 80d323d11e909cf0eb1b6fbb4ac22276483bcf31 F src/test3.c 124ff9735fb6bb7d41de180d6bac90e7b1509432 F src/test4.c d1e5a5e904d4b444cf572391fdcb017638e36ff7 @@ -212,7 +212,7 @@ F src/test_async.c 0612a752896fad42d55c3999a5122af10dcf22ad F src/test_autoext.c 30e7bd98ab6d70a62bb9ba572e4c7df347fe645e F src/test_backup.c c129c91127e9b46e335715ae2e75756e25ba27de F src/test_btree.c 47cd771250f09cdc6e12dda5bc71bc0b3abc96e2 -F src/test_config.c 470765ec36636d2c598766342b58e4c841e24512 +F src/test_config.c 70c74ef7a696c1b40d6415d48ee4c8f668cd16a8 F src/test_demovfs.c 20a4975127993f4959890016ae9ce5535a880094 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_func.c cbdec5cededa0761daedde5baf06004a9bf416b5 @@ -224,7 +224,7 @@ F src/test_intarray.h 489edb9068bb926583445cb02589344961054207 F src/test_journal.c 03313c693cca72959dcaaf79f8d76f21c01e19ff F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e F src/test_malloc.c 7ca7be34e0e09ef0ed6619544552ed95732e41f6 -F src/test_multiplex.c a7457a1ac46964b7f897d75ecfd447410ec067e6 +F src/test_multiplex.c 991a60733dbde8c529043d466c5c44d180762561 F src/test_multiplex.h e99c571bc4968b7a9363b661481f3934bfead61d F src/test_mutex.c a6bd7b9cf6e19d989e31392b06ac8d189f0d573e F src/test_onefile.c 40cf9e212a377a6511469384a64b01e6e34b2eec @@ -244,16 +244,16 @@ F src/test_vfstrace.c 0b884e06094a746da729119a2cabdc7aa790063d F src/test_wholenumber.c 6129adfbe7c7444f2e60cc785927f3aa74e12290 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/tokenize.c c819d9f72168a035d545a5bdafe9b085b20df705 -F src/trigger.c c836a6caac16ba96611558922106858f6ca3d6bf -F src/update.c a81bda229f8c3b698f8dcf8e69485c97e1347102 +F src/trigger.c 1cfb80e2290ef66ea89cb4e821caae65a02c0d56 +F src/update.c 2d67e24d5a44d8b1c0839bf2ee0c391593e852bf F src/utf.c c53eb7404b3eb5c1cbb5655c6a7a0e0ce6bd50f0 -F src/util.c 0f33bbbdfcc4a2d8cf20c3b2a16ffc3b57c58a70 +F src/util.c 06302ffd2b80408d4f6c7af71f7090e0cf8d8ff7 F src/vacuum.c 05513dca036a1e7848fe18d5ed1265ac0b32365e -F src/vdbe.c df52db6162fd94767b76bb4e9a7cb9207e83086f +F src/vdbe.c 00061273468d03c6c5c0f6144ed219c0004cf849 F src/vdbe.h 322af148cceef120bb1ec9cff7f122e76abf94da F src/vdbeInt.h 3de6588b36c833969aebab202e1766d586c37ec2 -F src/vdbeapi.c 432a8a194accb9fe9fae45338f210174c6c961b4 -F src/vdbeaux.c bb86d48ce99c288d17e8d79711713399c21f0a8d +F src/vdbeapi.c ddd061183e2c3015f676e53ee85fcaf306617e8e +F src/vdbeaux.c 7d0b75a04ac466d25a90fefd1df83361a2ab0ad7 F src/vdbeblob.c a547f286b651641bdb43a567af66d0e776a39ea2 F src/vdbemem.c 0498796b6ffbe45e32960d6a1f5adfb6e419883b F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114 @@ -261,13 +261,13 @@ F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 F src/wal.c 0c70ad7b1cac6005fa5e2cbefd23ee05e391c290 F src/wal.h 66b40bd91bc29a5be1c88ddd1f5ade8f3f48728a F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f -F src/where.c 55403ce19c506be6a321c7f129aff693d6103db5 +F src/where.c 106cd9ab3eb410dfa7d0598194c277664bb2e9a3 F test/8_3_names.test b93687beebd17f6ebf812405a6833bae5d1f4199 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 -F test/all.test 51756962d522e474338e9b2ebb26e7364d4aa125 +F test/all.test 52fc8dee494092031a556911d404ca30a749a30b F test/alter.test a3f570072b53d7c0fe463bab3f5affa8e113c487 -F test/alter2.test 75f731508f1bf27ba09a6075c66cd02216ba464b +F test/alter2.test e0c09d630d650ea32333781a4ed3c45eb02c4289 F test/alter3.test 8677e48d95536f7a6ed86a1a774744dadcc22b07 F test/alter4.test 1e5dd6b951e9f65ca66422edff02e56df82dd403 F test/altermalloc.test e81ac9657ed25c6c5bb09bebfa5a047cd8e4acfc @@ -292,7 +292,7 @@ F test/auth.test b047105c32da7db70b842fd24056723125ecc2ff F test/auth2.test 270baddc8b9c273682760cffba6739d907bd2882 F test/auth3.test a4755e6a2a2fea547ffe63c874eb569e60a28eb5 F test/autoinc.test 85ef3180a737e6580086a018c09c6f1a52759b46 -F test/autoindex1.test 860fc83f4fefb0c68ad062afc3ff43faa1534fc4 +F test/autoindex1.test 058d0b331ae6840a61bbee910d8cbae27bfd5991 F test/autovacuum.test bb7c0885e6f8f1d633045de48f2b66082162766d F test/autovacuum_ioerr2.test 598b0663074d3673a9c1bc9a16e80971313bafe6 F test/avtrans.test 0252654f4295ddda3b2cce0e894812259e655a85 @@ -333,7 +333,7 @@ F test/collate1.test e3eaa48c21e150814be1a7b852d2a8af24458d04 F test/collate2.test 04cebe4a033be319d6ddbb3bbc69464e01700b49 F test/collate3.test d28d2cfab2c3a3d4628ae4b2b7afc9965daa3b4c F test/collate4.test 3d3f123f83fd8ccda6f48d617e44e661b9870c7d -F test/collate5.test fe0f43c4740d7b71b959cac668d19e42f2e06e4d +F test/collate5.test 67f1d3e848e230ff4802815a79acb0a8b5e69bd7 F test/collate6.test 8be65a182abaac8011a622131486dafb8076e907 F test/collate7.test fac8db7aac3978466c04ae892cc74dcf2bc031aa F test/collate8.test df26649cfcbddf109c04122b340301616d3a88f6 @@ -379,6 +379,7 @@ F test/descidx1.test b1353c1a15cfbee97b13a1dcedaf0fe78163ba6a F test/descidx2.test 9f1a0c83fd57f8667c82310ca21b30a350888b5d F test/descidx3.test fe720e8b37d59f4cef808b0bf4e1b391c2e56b6f F test/diskfull.test 0cede7ef9d8f415d9d3944005c76be7589bb5ebb +F test/distinct.test 8c4d951fc40aba84421060e07b16099d2f4c2fdf F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 F test/e_createtable.test 4771686a586b6ae414f927c389b2c101cc05c028 F test/e_delete.test e2ae0d3fce5efd70fef99025e932afffc5616fab @@ -390,7 +391,7 @@ F test/e_fts3.test 75bb0aee26384ef586165e21018a17f7cd843469 F test/e_insert.test 76d4bb5da9b28014d515d91ffe29a79a1e99f2bc F test/e_reindex.test a064f0878b8f848fbca38f1f61f82f15a3000c64 F test/e_resolve.test dcce9308fb13b934ce29591105d031d3e14fbba6 -F test/e_select.test 7ac53674e822d4d77bbb4a9a4aaefa5fdc9e493f +F test/e_select.test 8d7fac7a268eaeb80b9a7ba7964505b9d30f5458 F test/e_select2.test 5c3d3da19c7b3e90ae444579db2b70098599ab92 F test/e_update.test b926341a65955d69a6375c9eb4fd82e7089bc83a F test/e_uri.test 6f35b491f80dac005c8144f38b2dfb4d96483596 @@ -404,7 +405,7 @@ F test/eval.test bc269c365ba877554948441e91ad5373f9f91be3 F test/exclusive.test 53e1841b422e554cecf0160f937c473d6d0e3062 F test/exclusive2.test 343d55130c12c67b8bf10407acec043a6c26c86b F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 -F test/exists.test 5e2d64b4eb5a9d08876599bdae2e1213d2d12e2a +F test/exists.test 8f7b27b61c2fbe5822f0a1f899c715d14e416e30 F test/expr.test 67c9fd6f8f829e239dc8b0f4a08a73c08b09196d F test/fallocate.test 43dc34b8c24be6baffadc3b4401ee15710ce83c6 F test/filectrl.test 97003734290887566e01dded09dc9e99cb937e9e @@ -467,7 +468,7 @@ F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8 F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18 F test/fts3ao.test b83f99f70e9eec85f27d75801a974b3f820e01f9 F test/fts3atoken.test 402ef2f7c2fb4b3d4fa0587df6441c1447e799b3 -F test/fts3auto.test f1cb0a55130897013ca5850dbee2945c2908a45a +F test/fts3auto.test c1a30b37002b7c764a96937fbc71065b73d69494 F test/fts3aux1.test 0b02743955d56fc0d4d66236a26177bd1b726de0 F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958 @@ -496,13 +497,13 @@ F test/fts3sort.test 63d52c1812904b751f9e1ff487472e44833f5402 F test/fts4aa.test 148d9eb54901af23b5d402b1f388f43e559e1728 F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f -F test/func3.test 7ba2ca5a1e9bca900ba2c230cf04bd67184bc1bc +F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a F test/fuzz.test 77fd50afc12847af50fcf1941679d90adebadde6 F test/fuzz2.test 207d0f9d06db3eaf47a6b7bfc835b8e2fc397167 F test/fuzz3.test aec64345184d1662bd30e6a17851ff659d596dc5 F test/fuzz_common.tcl a87dfbb88c2a6b08a38e9a070dabd129e617b45b F test/fuzz_malloc.test dd7001ac86d09c154a7dff064f4739c60e2b312c -F test/fuzzer1.test 3105b5a89a6cb0d475f0877debec942fe4143462 +F test/fuzzer1.test ddfb04f3bd5cfdda3b1aa15b78d3ad055c9cc50f F test/hook.test 4fd80e9c3ffb49de0379e2a026ef2fe7b9f3e534 F test/icu.test 70df4faca133254c042d02ae342c0a141f2663f4 F test/in.test 19b642bb134308980a92249750ea4ce3f6c75c2d @@ -512,7 +513,7 @@ F test/in4.test 64f3cc1acde1b9161ccdd8e5bde3daefdb5b2617 F test/incrblob.test 76e787ca3301d9bfa6906031c626d26f8dd707de F test/incrblob2.test edc3a96e557bd61fb39acc8d2edd43371fbbaa19 F test/incrblob3.test aedbb35ea1b6450c33b98f2b6ed98e5020be8dc7 -F test/incrblob_err.test c577c91d4ed9e8336cdb188b15d6ee2a6fe9604e +F test/incrblob_err.test d2562d2771ebffd4b3af89ef64c140dd44371597 F test/incrblobfault.test 917c0292224c64a56ef7215fd633a3a82f805be0 F test/incrvacuum.test 453d1e490d8f5ad2c9b3a54282a0690d6ae56462 F test/incrvacuum2.test ae04573b73ad52179f56e194fff0fbe43b509d23 @@ -525,7 +526,7 @@ F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 F test/insert.test aef273dd1cee84cc92407469e6bd1b3cdcb76908 F test/insert2.test 4f3a04d168c728ed5ec2c88842e772606c7ce435 F test/insert3.test 1b7db95a03ad9c5013fdf7d6722b6cd66ee55e30 -F test/insert4.test b3e02648a5fc3075c29e13c369b5127bf859b5a2 +F test/insert4.test 63ea672b0fc6d3a9a0ccee774a771510b1e684c4 F test/insert5.test 1f93cbe9742110119133d7e8e3ccfe6d7c249766 F test/intarray.test 066b7d7ac38d25bf96f87f1b017bfc687551cdd4 F test/interrupt.test 42e7cf98646fd9cb4a3b131a93ed3c50b9e149f1 @@ -551,7 +552,7 @@ F test/jrnlmode3.test c6522b276ba315fd1416198de6fc1da9e72409fb F test/keyword1.test a2400977a2e4fde43bf33754c2929fda34dbca05 F test/lastinsert.test 474d519c68cb79d07ecae56a763aa7f322c72f51 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 -F test/like.test a47f52692aac96ba82508efba74819214cdebc17 +F test/like.test 9cc5261d22f2108a27cedff8a972aa3284a4ba52 F test/like2.test 3b2ee13149ba4a8a60b59756f4e5d345573852da F test/limit.test 2db7b3b34fb925b8e847d583d2eb67531d0ce67e F test/loadext.test 0393ce12d9616aa87597dd0ec88181de181f6db0 @@ -601,11 +602,11 @@ F test/misc1.test e56baf44656dd68d6475a4b44521045a60241e9b F test/misc2.test a628db7b03e18973e5d446c67696b03de718c9fd F test/misc3.test 72c5dc87a78e7865c5ec7a969fc572913dbe96b6 F test/misc4.test 9c078510fbfff05a9869a0b6d8b86a623ad2c4f6 -F test/misc5.test 45b2e3ed5f79af2b4f38ae362eaf4c49674575bd +F test/misc5.test 9f9338f8211c7f5d1cbe16331fa65d019501aa50 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test 29032efcd3d826fbd409e2a7af873e7939f4a4e3 F test/misuse.test 30b3a458e5a70c31e74c291937b6c82204c59f33 -F test/multiplex.test 555080c87abfc72ba68e2f3df01d4a9a7a4fdf58 +F test/multiplex.test b45367b1dac7dfa4c5b8ff0f3844260804a0034d F test/mutex1.test 78b2b9bb320e51d156c4efdb71b99b051e7a4b41 F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660 F test/nan.test dc212a22b36109fd1ae37154292444ef249c5ec2 @@ -626,7 +627,7 @@ F test/pageropt.test 8146bf448cf09e87bb1867c2217b921fb5857806 F test/pagesize.test 76aa9f23ecb0741a4ed9d2e16c5fa82671f28efb F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test 0d85f2ab6963aee28c671d4c71bec038c00a1d16 -F test/permutations.test 8331b2897502df0671dd1390a3c87db44fb8757c +F test/permutations.test df39e810d98a966218ef1391afb4e25491af698f F test/pragma.test fdfc09067ea104a0c247a1a79d8093b56656f850 F test/pragma2.test 5364893491b9231dd170e3459bfc2e2342658b47 F test/printf.test 05970cde31b1a9f54bd75af60597be75a5c54fea @@ -640,7 +641,7 @@ F test/randexpr1.test 1084050991e9ba22c1c10edd8d84673b501cc25a F test/rdonly.test c267d050a1d9a6a321de502b737daf28821a518d F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 F test/releasetest.mk 2eced2f9ae701fd0a29e714a241760503ccba25a -F test/releasetest.tcl c0c0865f1dff08dde08a964ef49e83217ebedbf8 +F test/releasetest.tcl fa302d03fd9acfce6d910553a33473bfcf561958 F test/rollback.test 1a83118ea6db4e7d8c10eaa63871b5e90502ffdc F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 F test/rowid.test e58e0acef38b527ed1b0b70d3ada588f804af287 @@ -666,7 +667,7 @@ F test/select7.test dad6f00f0d49728a879d6eb6451d4752db0b0abe F test/select8.test 391de11bdd52339c30580dabbbbe97e3e9a3c79d F test/select9.test 74c0fb2c6eecb0219cbed0cbe3df136f8fbf9343 F test/selectA.test 06d1032fa9009314c95394f2ca2e60d9f7ae8532 -F test/selectB.test f305cc6660804cb239aab4e2f26b0e288b59958b +F test/selectB.test 0d072c5846071b569766e6cd7f923f646a8b2bfa F test/selectC.test f9bf1bc4581b5b8158caa6e4e4f682acb379fb25 F test/server1.test f5b790d4c0498179151ca8a7715a65a7802c859c F test/session.test c1a17c11ef7d01c24fe2b9f7871190d949a8e718 @@ -691,7 +692,7 @@ F test/speed3.test 5a419039e9da95d906adb2298af2849600c81c11 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b -F test/sqllimits1.test e90a0ed94452076f6a10209d378e06b5f75ef0a0 +F test/sqllimits1.test 0ebf7bed0b99c96f24e0b7fa5e59dbc42359c421 F test/stat.test c7b20ea43003dc2dc33335e231c27be8284c4a2a F test/stmt.test 25d64e3dbf9a3ce89558667d7f39d966fe2a71b9 F test/subquery.test b524f57c9574b2c0347045b4510ef795d4686796 @@ -705,7 +706,7 @@ F test/table.test 04ba066432430657712d167ebf28080fe878d305 F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 F test/tclsqlite.test 1ce9b6340d6d412420634e129a2e3722c651056a F test/tempdb.test 19d0f66e2e3eeffd68661a11c83ba5e6ace9128c -F test/temptable.test f42121a0d29a62f00f93274464164177ab1cc24a +F test/temptable.test 1a21a597055dcf6002b6f1ee867632dccd6e925c F test/temptrigger.test b0273db072ce5f37cf19140ceb1f0d524bbe9f05 F test/tester.tcl 3b6143fdafff36c516cf0fd663a20cd420aae36f F test/thread001.test a3e6a7254d1cb057836cb3145b60c10bf5b7e60f @@ -729,6 +730,7 @@ F test/tkt-38cb5df375.test 9e9b19857dba0896a8efdaf334d405ba423492f2 F test/tkt-3998683a16.test 6d1d04d551ed1704eb3396ca87bb9ccc8c5c1eb7 F test/tkt-3fe897352e.test 10de1a67bd5c66b238a4c96abe55531b37bb4f00 F test/tkt-4a03edc4c8.test 2865e4edbc075b954daa82f8da7cc973033ec76e +F test/tkt-54844eea3f.test a12b851128f46a695e4e378cca67409b9b8f5894 F test/tkt-5d863f876e.test 884072c2de496ddbb90c387c9ebc0d4f44a91b8e F test/tkt-5e10420e8d.test 904d1687b3c06d43e5b3555bbcf6802e7c0ffd84 F test/tkt-5ee23731f.test 3581260f2a71e51db94e1506ba6b0f7311d002a9 @@ -852,7 +854,7 @@ F test/trigger9.test 5b0789f1c5c4600961f8e68511b825b87be53e31 F test/triggerA.test eaf11a29db2a11967d2d4b49d37f92bce598194e F test/triggerB.test 56780c031b454abac2340dbb3b71ac5c56c3d7fe F test/triggerC.test 02c690febf608ae20b9af86184a9867f79855b1d -F test/triggerD.test c6add3817351451e419f6ff9e9a259b02b6e2de7 +F test/triggerD.test bfdac1143deee8fb12b6a3640d76e5669a567ff6 F test/tt3_checkpoint.c 415eccce672d681b297485fc20f44cdf0eac93af F test/types.test bf816ce73c7dfcfe26b700c19f97ef4050d194ff F test/types2.test 3555aacf8ed8dc883356e59efc314707e6247a84 @@ -889,10 +891,10 @@ F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d F test/wal.test 5617ad308bfdb8a8885220d8a261a6096a8d7e57 F test/wal2.test aa0fb2314b3235be4503c06873e41ebfc0757782 -F test/wal3.test 5c396cc22497244d627306f4c1d360167353f8dd +F test/wal3.test d512a5c8b4aa345722d11e8f1671db7eb15a0e39 F test/wal4.test 3404b048fa5e10605facaf70384e6d2943412e30 F test/wal5.test f06a0427e06db00347e32eb9fa99d6a5c0f2d088 -F test/wal6.test 07aa31ca8892d0527f2c5c5a9a2a87aa421dfaa8 +F test/wal6.test 2e3bc767d9c2ce35c47106148d43fcbd072a93b3 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd F test/wal_common.tcl a98f17fba96206122eff624db0ab13ec377be4fe F test/walbak.test 4df1c7369da0301caeb9a48fa45997fd592380e4 @@ -921,6 +923,7 @@ F test/where9.test 24f19ad14bb1b831564ced5273e681e495662848 F test/whereA.test 24c234263c8fe358f079d5e57d884fb569d2da0a F test/whereB.test 0def95db3bdec220a731c7e4bec5930327c1d8c5 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 +F test/win32lock.test 0a16a7df4a51575bda27529ac992a5a94e4a38bd F test/zeroblob.test caaecfb4f908f7bc086ed238668049f96774d688 F tool/build-shell.sh 12aa4391073a777fcb6dcc490b219a018ae98bac F tool/diffdb.c 7524b1b5df217c20cd0431f6789851a4e0cb191b @@ -957,11 +960,11 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c -F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 +F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 70c84e50209722a61d66fd737e42d49275745b62 -R 7d529af0ae98dc5a31387a39a49b9017 -U dan -Z 3511ba374807e0034b8197f50cf80466 +F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 +P d04e0fd82a15aee963e35830caf8159b4b6ccd87 8ce2b74a82264550b0e19da3e0e1a145db940a1c +R a5737ded89a8c66dbf565b51f6981ff8 +U drh +Z 81f45685ad26c120405ad7cb74da4306 diff --git a/manifest.uuid b/manifest.uuid index 7c4bfbd3cf..cd26a2668c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d04e0fd82a15aee963e35830caf8159b4b6ccd87 \ No newline at end of file +110cfd6920cf3011aeaf7e586f8db867bfc69fbb \ No newline at end of file diff --git a/src/build.c b/src/build.c index 97758cbcbd..455b35b56e 100644 --- a/src/build.c +++ b/src/build.c @@ -2791,7 +2791,7 @@ Index *sqlite3CreateIndex( /* A named index with an explicit CREATE INDEX statement */ zStmt = sqlite3MPrintf(db, "CREATE%s INDEX %.*s", onError==OE_None ? "" : " UNIQUE", - pEnd->z - pName->z + 1, + (int)(pEnd->z - pName->z) + 1, pName->z); }else{ /* An automatic index created by a PRIMARY KEY or UNIQUE constraint */ diff --git a/src/date.c b/src/date.c index c0744e8c72..f9411ea5b3 100644 --- a/src/date.c +++ b/src/date.c @@ -427,7 +427,9 @@ static int osLocaltime(time_t *t, struct tm *pTm){ #if (!defined(HAVE_LOCALTIME_R) || !HAVE_LOCALTIME_R) \ && (!defined(HAVE_LOCALTIME_S) || !HAVE_LOCALTIME_S) struct tm *pX; +#if SQLITE_THREADSAFE>0 sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); +#endif sqlite3_mutex_enter(mutex); pX = localtime(t); #ifndef SQLITE_OMIT_BUILTIN_TEST diff --git a/src/delete.c b/src/delete.c index 6e45a2e7ea..b949fe27a9 100644 --- a/src/delete.c +++ b/src/delete.c @@ -378,7 +378,9 @@ void sqlite3DeleteFrom( /* Collect rowids of every row to be deleted. */ sqlite3VdbeAddOp2(v, OP_Null, 0, iRowSet); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere,0,WHERE_DUPLICATES_OK); + pWInfo = sqlite3WhereBegin( + pParse, pTabList, pWhere, 0, 0, WHERE_DUPLICATES_OK + ); if( pWInfo==0 ) goto delete_from_cleanup; regRowid = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, iRowid); sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, regRowid); diff --git a/src/expr.c b/src/expr.c index be2f4d7c66..ab4547db9c 100644 --- a/src/expr.c +++ b/src/expr.c @@ -902,6 +902,7 @@ SrcList *sqlite3SrcListDup(sqlite3 *db, SrcList *p, int flags){ pNewItem->jointype = pOldItem->jointype; pNewItem->iCursor = pOldItem->iCursor; pNewItem->isPopulated = pOldItem->isPopulated; + pNewItem->isCorrelated = pOldItem->isCorrelated; pNewItem->zIndex = sqlite3DbStrDup(db, pOldItem->zIndex); pNewItem->notIndexed = pOldItem->notIndexed; pNewItem->pIndex = pOldItem->pIndex; diff --git a/src/fkey.c b/src/fkey.c index dda8a50d44..37d4744dda 100644 --- a/src/fkey.c +++ b/src/fkey.c @@ -560,7 +560,7 @@ static void fkScanChildren( ** clause. If the constraint is not deferred, throw an exception for ** each row found. Otherwise, for deferred constraints, increment the ** deferred constraint counter by nIncr for each row selected. */ - pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0, 0); + pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0, 0, 0); if( nIncr>0 && pFKey->isDeferred==0 ){ sqlite3ParseToplevel(pParse)->mayAbort = 1; } diff --git a/src/loadext.c b/src/loadext.c index c832e684ef..079458d143 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -84,6 +84,8 @@ # define sqlite3_create_module 0 # define sqlite3_create_module_v2 0 # define sqlite3_declare_vtab 0 +# define sqlite3_vtab_config 0 +# define sqlite3_vtab_on_conflict 0 #endif #ifdef SQLITE_OMIT_SHARED_CACHE @@ -107,6 +109,7 @@ #define sqlite3_blob_open 0 #define sqlite3_blob_read 0 #define sqlite3_blob_write 0 +#define sqlite3_blob_reopen 0 #endif /* @@ -372,6 +375,9 @@ static const sqlite3_api_routines sqlite3Apis = { 0, 0, #endif + sqlite3_blob_reopen, + sqlite3_vtab_config, + sqlite3_vtab_on_conflict, }; /* diff --git a/src/mem3.c b/src/mem3.c index e2d66815b3..1a1b791f28 100644 --- a/src/mem3.c +++ b/src/mem3.c @@ -433,7 +433,7 @@ static void *memsys3MallocUnsafe(int nByte){ ** This function assumes that the necessary mutexes, if any, are ** already held by the caller. Hence "Unsafe". */ -void memsys3FreeUnsafe(void *pOld){ +static void memsys3FreeUnsafe(void *pOld){ Mem3Block *p = (Mem3Block*)pOld; int i; u32 size, x; @@ -508,7 +508,7 @@ static void *memsys3Malloc(int nBytes){ /* ** Free memory. */ -void memsys3Free(void *pPrior){ +static void memsys3Free(void *pPrior){ assert( pPrior ); memsys3Enter(); memsys3FreeUnsafe(pPrior); @@ -518,7 +518,7 @@ void memsys3Free(void *pPrior){ /* ** Change the size of an existing memory allocation */ -void *memsys3Realloc(void *pPrior, int nBytes){ +static void *memsys3Realloc(void *pPrior, int nBytes){ int nOld; void *p; if( pPrior==0 ){ diff --git a/src/os.c b/src/os.c index ba0438adef..41ec69ce45 100644 --- a/src/os.c +++ b/src/os.c @@ -136,7 +136,7 @@ int sqlite3OsOpen( ** down into the VFS layer. Some SQLITE_OPEN_ flags (for example, ** SQLITE_OPEN_FULLMUTEX or SQLITE_OPEN_SHAREDCACHE) are blocked before ** reaching the VFS. */ - rc = pVfs->xOpen(pVfs, zPath, pFile, flags & 0x87f3f, pFlagsOut); + rc = pVfs->xOpen(pVfs, zPath, pFile, flags & 0x87f7f, pFlagsOut); assert( rc==SQLITE_OK || pFile->pMethods==0 ); return rc; } diff --git a/src/os_unix.c b/src/os_unix.c index 682e74c958..b2956c1647 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -677,7 +677,9 @@ static int sqliteErrorFromPosixError(int posixError, int sqliteIOErr) { case ENODEV: case ENXIO: case ENOENT: +#ifdef ESTALE /* ESTALE is not defined on Interix systems */ case ESTALE: +#endif case ENOSYS: /* these should force the client to close the file and reconnect */ @@ -3672,7 +3674,7 @@ static void unixShmPurge(unixFile *pFd){ if( p && p->nRef==0 ){ int i; assert( p->pInode==pFd->pInode ); - if( p->mutex ) sqlite3_mutex_free(p->mutex); + sqlite3_mutex_free(p->mutex); for(i=0; inRegion; i++){ if( p->h>=0 ){ munmap(p->apRegion[i], p->szRegion); diff --git a/src/os_win.c b/src/os_win.c index bd0f2f216a..2d4f0adad2 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -402,6 +402,54 @@ static int winLogErrorAtLine( return errcode; } +/* +** The number of times that a ReadFile(), WriteFile(), and DeleteFile() +** will be retried following a locking error - probably caused by +** antivirus software. Also the initial delay before the first retry. +** The delay increases linearly with each retry. +*/ +#ifndef SQLITE_WIN32_IOERR_RETRY +# define SQLITE_WIN32_IOERR_RETRY 10 +#endif +#ifndef SQLITE_WIN32_IOERR_RETRY_DELAY +# define SQLITE_WIN32_IOERR_RETRY_DELAY 25 +#endif +static int win32IoerrRetry = SQLITE_WIN32_IOERR_RETRY; +static int win32IoerrRetryDelay = SQLITE_WIN32_IOERR_RETRY_DELAY; + +/* +** If a ReadFile() or WriteFile() error occurs, invoke this routine +** to see if it should be retried. Return TRUE to retry. Return FALSE +** to give up with an error. +*/ +static int retryIoerr(int *pnRetry){ + DWORD e; + if( *pnRetry>=win32IoerrRetry ){ + return 0; + } + e = GetLastError(); + if( e==ERROR_ACCESS_DENIED || + e==ERROR_LOCK_VIOLATION || + e==ERROR_SHARING_VIOLATION ){ + Sleep(win32IoerrRetryDelay*(1+*pnRetry)); + ++*pnRetry; + return 1; + } + return 0; +} + +/* +** Log a I/O error retry episode. +*/ +static void logIoerr(int nRetry){ + if( nRetry ){ + sqlite3_log(SQLITE_IOERR, + "delayed %dms for lock/sharing conflict", + win32IoerrRetryDelay*nRetry*(nRetry+1)/2 + ); + } +} + #if SQLITE_OS_WINCE /************************************************************************* ** This section contains code for WinCE only. @@ -820,6 +868,7 @@ static int winRead( ){ winFile *pFile = (winFile*)id; /* file handle */ DWORD nRead; /* Number of bytes actually read from file */ + int nRetry = 0; /* Number of retrys */ assert( id!=0 ); SimulateIOError(return SQLITE_IOERR_READ); @@ -828,10 +877,12 @@ static int winRead( if( seekWinFile(pFile, offset) ){ return SQLITE_FULL; } - if( !ReadFile(pFile->h, pBuf, amt, &nRead, 0) ){ + while( !ReadFile(pFile->h, pBuf, amt, &nRead, 0) ){ + if( retryIoerr(&nRetry) ) continue; pFile->lastErrno = GetLastError(); return winLogError(SQLITE_IOERR_READ, "winRead", pFile->zPath); } + logIoerr(nRetry); if( nRead<(DWORD)amt ){ /* Unread parts of the buffer must be zero-filled */ memset(&((char*)pBuf)[nRead], 0, amt-nRead); @@ -853,6 +904,7 @@ static int winWrite( ){ int rc; /* True if error has occured, else false */ winFile *pFile = (winFile*)id; /* File handle */ + int nRetry = 0; /* Number of retries */ assert( amt>0 ); assert( pFile ); @@ -867,7 +919,12 @@ static int winWrite( int nRem = amt; /* Number of bytes yet to be written */ DWORD nWrite; /* Bytes written by each WriteFile() call */ - while( nRem>0 && WriteFile(pFile->h, aRem, nRem, &nWrite, 0) && nWrite>0 ){ + while( nRem>0 ){ + if( !WriteFile(pFile->h, aRem, nRem, &nWrite, 0) ){ + if( retryIoerr(&nRetry) ) continue; + break; + } + if( nWrite<=0 ) break; aRem += nWrite; nRem -= nWrite; } @@ -883,6 +940,8 @@ static int winWrite( return SQLITE_FULL; } return winLogError(SQLITE_IOERR_WRITE, "winWrite", pFile->zPath); + }else{ + logIoerr(nRetry); } return SQLITE_OK; } @@ -1299,6 +1358,20 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ case SQLITE_FCNTL_SYNC_OMITTED: { return SQLITE_OK; } + case SQLITE_FCNTL_WIN32_AV_RETRY: { + int *a = (int*)pArg; + if( a[0]>0 ){ + win32IoerrRetry = a[0]; + }else{ + a[0] = win32IoerrRetry; + } + if( a[1]>0 ){ + win32IoerrRetryDelay = a[1]; + }else{ + a[1] = win32IoerrRetryDelay; + } + return SQLITE_OK; + } } return SQLITE_NOTFOUND; } @@ -2316,15 +2389,13 @@ static int winOpen( ** to MX_DELETION_ATTEMPTs deletion attempts are run before giving ** up and returning an error. */ -#define MX_DELETION_ATTEMPTS 5 static int winDelete( sqlite3_vfs *pVfs, /* Not used on win32 */ const char *zFilename, /* Name of file to delete */ int syncDir /* Not used on win32 */ ){ int cnt = 0; - DWORD rc; - DWORD error = 0; + int rc; void *zConverted; UNUSED_PARAMETER(pVfs); UNUSED_PARAMETER(syncDir); @@ -2335,34 +2406,30 @@ static int winDelete( return SQLITE_NOMEM; } if( isNT() ){ - do{ - DeleteFileW(zConverted); - }while( ( ((rc = GetFileAttributesW(zConverted)) != INVALID_FILE_ATTRIBUTES) - || ((error = GetLastError()) == ERROR_ACCESS_DENIED)) - && (++cnt < MX_DELETION_ATTEMPTS) - && (Sleep(100), 1) ); + rc = 1; + while( GetFileAttributesW(zConverted)!=INVALID_FILE_ATTRIBUTES && + (rc = DeleteFileW(zConverted))==0 && retryIoerr(&cnt) ){} + rc = rc ? SQLITE_OK : SQLITE_ERROR; /* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. ** Since the ASCII version of these Windows API do not exist for WINCE, ** it's important to not reference them for WINCE builds. */ #if SQLITE_OS_WINCE==0 }else{ - do{ - DeleteFileA(zConverted); - }while( ( ((rc = GetFileAttributesA(zConverted)) != INVALID_FILE_ATTRIBUTES) - || ((error = GetLastError()) == ERROR_ACCESS_DENIED)) - && (++cnt < MX_DELETION_ATTEMPTS) - && (Sleep(100), 1) ); + rc = 1; + while( GetFileAttributesA(zConverted)!=INVALID_FILE_ATTRIBUTES && + (rc = DeleteFileA(zConverted))==0 && retryIoerr(&cnt) ){} + rc = rc ? SQLITE_OK : SQLITE_ERROR; #endif } + if( rc ){ + rc = winLogError(SQLITE_IOERR_DELETE, "winDelete", zFilename); + }else{ + logIoerr(cnt); + } free(zConverted); - OSTRACE(("DELETE \"%s\" %s\n", zFilename, - ( (rc==INVALID_FILE_ATTRIBUTES) && (error==ERROR_FILE_NOT_FOUND)) ? - "ok" : "failed" )); - - return ( (rc == INVALID_FILE_ATTRIBUTES) - && (error == ERROR_FILE_NOT_FOUND)) ? SQLITE_OK : - winLogError(SQLITE_IOERR_DELETE, "winDelete", zFilename); + OSTRACE(("DELETE \"%s\" %s\n", zFilename, (rc ? "failed" : "ok" ))); + return rc; } /* diff --git a/src/resolve.c b/src/resolve.c index 74d6aaef93..d29d2a8344 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -996,11 +996,25 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ for(i=0; ipSrc->nSrc; i++){ struct SrcList_item *pItem = &p->pSrc->a[i]; if( pItem->pSelect ){ + NameContext *pNC; /* Used to iterate name contexts */ + int nRef = 0; /* Refcount for pOuterNC and outer contexts */ const char *zSavedContext = pParse->zAuthContext; + + /* Count the total number of references to pOuterNC and all of its + ** parent contexts. After resolving references to expressions in + ** pItem->pSelect, check if this value has changed. If so, then + ** SELECT statement pItem->pSelect must be correlated. Set the + ** pItem->isCorrelated flag if this is the case. */ + for(pNC=pOuterNC; pNC; pNC=pNC->pNext) nRef += pNC->nRef; + if( pItem->zName ) pParse->zAuthContext = pItem->zName; sqlite3ResolveSelectNames(pParse, pItem->pSelect, pOuterNC); pParse->zAuthContext = zSavedContext; if( pParse->nErr || db->mallocFailed ) return WRC_Abort; + + for(pNC=pOuterNC; pNC; pNC=pNC->pNext) nRef -= pNC->nRef; + assert( pItem->isCorrelated==0 && nRef<=0 ); + pItem->isCorrelated = (nRef!=0); } } diff --git a/src/select.c b/src/select.c index 8dcfb84b8a..bd5e964e7c 100644 --- a/src/select.c +++ b/src/select.c @@ -3721,6 +3721,7 @@ int sqlite3Select( int distinct; /* Table to use for the distinct set */ int rc = 1; /* Value to return from this function */ int addrSortIndex; /* Address of an OP_OpenEphemeral instruction */ + int addrDistinctIndex; /* Address of an OP_OpenEphemeral instruction */ AggInfo sAggInfo; /* Information used by aggregate queries */ int iEnd; /* Address of the end of the query */ sqlite3 *db; /* The database connection */ @@ -3847,16 +3848,6 @@ int sqlite3Select( } #endif - /* If possible, rewrite the query to use GROUP BY instead of DISTINCT. - ** GROUP BY might use an index, DISTINCT never does. - */ - assert( p->pGroupBy==0 || (p->selFlags & SF_Aggregate)!=0 ); - if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct ){ - p->pGroupBy = sqlite3ExprListDup(db, p->pEList, 0); - pGroupBy = p->pGroupBy; - p->selFlags &= ~SF_Distinct; - } - /* If there is both a GROUP BY and an ORDER BY clause and they are ** identical, then disable the ORDER BY clause since the GROUP BY ** will cause elements to come out in the correct order. This is @@ -3869,6 +3860,30 @@ int sqlite3Select( pOrderBy = 0; } + /* If the query is DISTINCT with an ORDER BY but is not an aggregate, and + ** if the select-list is the same as the ORDER BY list, then this query + ** can be rewritten as a GROUP BY. In other words, this: + ** + ** SELECT DISTINCT xyz FROM ... ORDER BY xyz + ** + ** is transformed to: + ** + ** SELECT xyz FROM ... GROUP BY xyz + ** + ** The second form is preferred as a single index (or temp-table) may be + ** used for both the ORDER BY and DISTINCT processing. As originally + ** written the query must use a temp-table for at least one of the ORDER + ** BY and DISTINCT, and an index or separate temp-table for the other. + */ + if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct + && sqlite3ExprListCompare(pOrderBy, p->pEList)==0 + ){ + p->selFlags &= ~SF_Distinct; + p->pGroupBy = sqlite3ExprListDup(db, p->pEList, 0); + pGroupBy = p->pGroupBy; + pOrderBy = 0; + } + /* If there is an ORDER BY clause, then this sorting ** index might end up being unused if the data can be ** extracted in pre-sorted order. If that is the case, then the @@ -3904,22 +3919,21 @@ int sqlite3Select( */ if( p->selFlags & SF_Distinct ){ KeyInfo *pKeyInfo; - assert( isAgg || pGroupBy ); distinct = pParse->nTab++; pKeyInfo = keyInfoFromExprList(pParse, p->pEList); - sqlite3VdbeAddOp4(v, OP_OpenEphemeral, distinct, 0, 0, - (char*)pKeyInfo, P4_KEYINFO_HANDOFF); + addrDistinctIndex = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, distinct, 0, 0, + (char*)pKeyInfo, P4_KEYINFO_HANDOFF); sqlite3VdbeChangeP5(v, BTREE_UNORDERED); }else{ - distinct = -1; + distinct = addrDistinctIndex = -1; } /* Aggregate and non-aggregate queries are handled differently */ if( !isAgg && pGroupBy==0 ){ - /* This case is for non-aggregate queries - ** Begin the database scan - */ - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pOrderBy, 0); + ExprList *pDist = (isDistinct ? p->pEList : 0); + + /* Begin the database scan. */ + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pOrderBy, pDist, 0); if( pWInfo==0 ) goto select_end; if( pWInfo->nRowOut < p->nSelectRow ) p->nSelectRow = pWInfo->nRowOut; @@ -3932,10 +3946,52 @@ int sqlite3Select( p->addrOpenEphm[2] = -1; } - /* Use the standard inner loop - */ - assert(!isDistinct); - selectInnerLoop(pParse, p, pEList, 0, 0, pOrderBy, -1, pDest, + if( pWInfo->eDistinct ){ + VdbeOp *pOp; /* No longer required OpenEphemeral instr. */ + + assert( addrDistinctIndex>0 ); + pOp = sqlite3VdbeGetOp(v, addrDistinctIndex); + + assert( isDistinct ); + assert( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED + || pWInfo->eDistinct==WHERE_DISTINCT_UNIQUE + ); + distinct = -1; + if( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED ){ + int iJump; + int iExpr; + int iFlag = ++pParse->nMem; + int iBase = pParse->nMem+1; + int iBase2 = iBase + pEList->nExpr; + pParse->nMem += (pEList->nExpr*2); + + /* Change the OP_OpenEphemeral coded earlier to an OP_Integer. The + ** OP_Integer initializes the "first row" flag. */ + pOp->opcode = OP_Integer; + pOp->p1 = 1; + pOp->p2 = iFlag; + + sqlite3ExprCodeExprList(pParse, pEList, iBase, 1); + iJump = sqlite3VdbeCurrentAddr(v) + 1 + pEList->nExpr + 1 + 1; + sqlite3VdbeAddOp2(v, OP_If, iFlag, iJump-1); + for(iExpr=0; iExprnExpr; iExpr++){ + CollSeq *pColl = sqlite3ExprCollSeq(pParse, pEList->a[iExpr].pExpr); + sqlite3VdbeAddOp3(v, OP_Ne, iBase+iExpr, iJump, iBase2+iExpr); + sqlite3VdbeChangeP4(v, -1, (const char *)pColl, P4_COLLSEQ); + sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); + } + sqlite3VdbeAddOp2(v, OP_Goto, 0, pWInfo->iContinue); + + sqlite3VdbeAddOp2(v, OP_Integer, 0, iFlag); + assert( sqlite3VdbeCurrentAddr(v)==iJump ); + sqlite3VdbeAddOp3(v, OP_Move, iBase, iBase2, pEList->nExpr); + }else{ + pOp->opcode = OP_Noop; + } + } + + /* Use the standard inner loop. */ + selectInnerLoop(pParse, p, pEList, 0, 0, pOrderBy, distinct, pDest, pWInfo->iContinue, pWInfo->iBreak); /* End the database scan loop. @@ -4045,7 +4101,7 @@ int sqlite3Select( ** in the right order to begin with. */ sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pGroupBy, 0); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pGroupBy, 0, 0); if( pWInfo==0 ) goto select_end; if( pGroupBy==0 ){ /* The optimizer is able to deliver rows in group by order so @@ -4307,7 +4363,7 @@ int sqlite3Select( ** of output. */ resetAccumulator(pParse, &sAggInfo); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pMinMax, flag); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pMinMax, 0, flag); if( pWInfo==0 ){ sqlite3ExprListDelete(db, pDel); goto select_end; diff --git a/src/shell.c b/src/shell.c index a54c922e87..05272b2dfa 100644 --- a/src/shell.c +++ b/src/shell.c @@ -2632,6 +2632,9 @@ static const char zOptions[] = #ifdef SQLITE_ENABLE_VFSTRACE " -vfstrace enable tracing of all VFS calls\n" #endif +#ifdef SQLITE_ENABLE_MULTIPLEX + " -multiplex enable the multiplexor VFS\n" +#endif ; static void usage(int showDetail){ fprintf(stderr, @@ -2732,6 +2735,11 @@ int main(int argc, char **argv){ int makeDefault ); vfstrace_register("trace",0,(int(*)(const char*,void*))fputs,stderr,1); +#endif +#ifdef SQLITE_ENABLE_MULTIPLEX + }else if( strcmp(argv[i],"-multiplex")==0 ){ + extern int sqlite3_multiple_initialize(const char*,int); + sqlite3_multiplex_initialize(0, 1); #endif }else if( strcmp(argv[i],"-vfs")==0 ){ sqlite3_vfs *pVfs = sqlite3_vfs_find(argv[++i]); @@ -2851,8 +2859,14 @@ int main(int argc, char **argv){ i++; }else if( strcmp(z,"-vfs")==0 ){ i++; +#ifdef SQLITE_ENABLE_VFSTRACE }else if( strcmp(z,"-vfstrace")==0 ){ i++; +#endif +#ifdef SQLITE_ENABLE_MULTIPLEX + }else if( strcmp(z,"-multiplex")==0 ){ + i++; +#endif }else if( strcmp(z,"-help")==0 || strcmp(z, "--help")==0 ){ usage(1); }else{ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index fdc656fab3..7354627fd5 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -736,6 +736,23 @@ struct sqlite3_io_methods { ** Applications should not call [sqlite3_file_control()] with this ** opcode as doing so may disrupt the operation of the specialized VFSes ** that do require it. +** +** ^The [SQLITE_FCNTL_WIN32_AV_RETRY] opcode is used to configure automatic +** retry counts and intervals for certain disk I/O operations for the +** windows [VFS] in order to work to provide robustness against +** anti-virus programs. By default, the windows VFS will retry file read, +** file write, and file delete opertions up to 10 times, with a delay +** of 25 milliseconds before the first retry and with the delay increasing +** by an additional 25 milliseconds with each subsequent retry. This +** opcode allows those to values (10 retries and 25 milliseconds of delay) +** to be adjusted. The values are changed for all database connections +** within the same process. The argument is a pointer to an array of two +** integers where the first integer i the new retry count and the second +** integer is the delay. If either integer is negative, then the setting +** is not changed but instead the prior value of that setting is written +** into the array entry, allowing the current retry settings to be +** interrogated. The zDbName parameter is ignored. +** */ #define SQLITE_FCNTL_LOCKSTATE 1 #define SQLITE_GET_LOCKPROXYFILE 2 @@ -745,7 +762,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_CHUNK_SIZE 6 #define SQLITE_FCNTL_FILE_POINTER 7 #define SQLITE_FCNTL_SYNC_OMITTED 8 - +#define SQLITE_FCNTL_WIN32_AV_RETRY 9 /* ** CAPI3REF: Mutex Handle diff --git a/src/sqlite3ext.h b/src/sqlite3ext.h index e45e691fd6..50dd5b6055 100644 --- a/src/sqlite3ext.h +++ b/src/sqlite3ext.h @@ -212,6 +212,9 @@ struct sqlite3_api_routines { int (*wal_autocheckpoint)(sqlite3*,int); int (*wal_checkpoint)(sqlite3*,const char*); void *(*wal_hook)(sqlite3*,int(*)(void*,sqlite3*,const char*,int),void*); + int (*blob_reopen)(sqlite3_blob*,sqlite3_int64); + int (*vtab_config)(sqlite3*,int op,...); + int (*vtab_on_conflict)(sqlite3*); }; /* @@ -412,6 +415,9 @@ struct sqlite3_api_routines { #define sqlite3_wal_autocheckpoint sqlite3_api->wal_autocheckpoint #define sqlite3_wal_checkpoint sqlite3_api->wal_checkpoint #define sqlite3_wal_hook sqlite3_api->wal_hook +#define sqlite3_blob_reopen sqlite3_api->blob_reopen +#define sqlite3_vtab_config sqlite3_api->vtab_config +#define sqlite3_vtab_on_conflict sqlite3_api->vtab_on_conflict #endif /* SQLITE_CORE */ #define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api = 0; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index aac0cdf3bb..718de76ab8 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -964,6 +964,7 @@ struct sqlite3 { #define SQLITE_GroupByOrder 0x20 /* Disable GROUPBY cover of ORDERBY */ #define SQLITE_FactorOutConst 0x40 /* Disable factoring out constants */ #define SQLITE_IdxRealAsInt 0x80 /* Store REAL as INT in indices */ +#define SQLITE_DistinctOpt 0x80 /* DISTINCT using indexes */ #define SQLITE_OptMask 0xff /* Mask of all disablable opts */ /* @@ -1855,6 +1856,7 @@ struct SrcList { u8 isPopulated; /* Temporary table associated with SELECT is populated */ u8 jointype; /* Type of join between this able and the previous */ u8 notIndexed; /* True if there is a NOT INDEXED clause */ + u8 isCorrelated; /* True if sub-query is correlated */ #ifndef SQLITE_OMIT_EXPLAIN u8 iSelectId; /* If pSelect!=0, the id of the sub-select in EQP */ #endif @@ -1974,6 +1976,7 @@ struct WhereInfo { u16 wctrlFlags; /* Flags originally passed to sqlite3WhereBegin() */ u8 okOnePass; /* Ok to use one-pass algorithm for UPDATE or DELETE */ u8 untestedTerms; /* Not all WHERE terms resolved by outer loop */ + u8 eDistinct; SrcList *pTabList; /* List of tables in the join */ int iTop; /* The very beginning of the WHERE loop */ int iContinue; /* Jump here to continue with next record */ @@ -1985,6 +1988,9 @@ struct WhereInfo { WhereLevel a[1]; /* Information about each nest loop in WHERE */ }; +#define WHERE_DISTINCT_UNIQUE 1 +#define WHERE_DISTINCT_ORDERED 2 + /* ** A NameContext defines a context in which to resolve table and column ** names. The context consists of a list of tables (the pSrcList) field and @@ -2747,7 +2753,7 @@ Expr *sqlite3LimitWhere(Parse *, SrcList *, Expr *, ExprList *, Expr *, Expr *, #endif void sqlite3DeleteFrom(Parse*, SrcList*, Expr*); void sqlite3Update(Parse*, SrcList*, ExprList*, Expr*, int); -WhereInfo *sqlite3WhereBegin(Parse*, SrcList*, Expr*, ExprList**, u16); +WhereInfo *sqlite3WhereBegin(Parse*, SrcList*, Expr*, ExprList**,ExprList*,u16); void sqlite3WhereEnd(WhereInfo*); int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int); void sqlite3ExprCodeGetColumnOfTable(Vdbe*, Table*, int, int, int); diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 8894b4f694..15ef2181a2 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -107,6 +107,11 @@ typedef struct IncrblobChannel IncrblobChannel; /* ** There is one instance of this structure for each SQLite database ** that has been opened by the SQLite TCL interface. +** +** If this module is built with SQLITE_TEST defined (to create the SQLite +** testfixture executable), then it may be configured to use either +** sqlite3_prepare_v2() or sqlite3_prepare() to prepare SQL statements. +** If SqliteDb.bLegacyPrepare is true, sqlite3_prepare() is used. */ typedef struct SqliteDb SqliteDb; struct SqliteDb { @@ -136,6 +141,9 @@ struct SqliteDb { IncrblobChannel *pIncrblob;/* Linked list of open incrblob channels */ int nStep, nSort, nIndex; /* Statistics for most recent operation */ int nTransaction; /* Number of nested [transaction] methods */ +#ifdef SQLITE_TEST + int bLegacyPrepare; /* True to use sqlite3_prepare() */ +#endif }; struct IncrblobChannel { @@ -430,20 +438,33 @@ static SqlFunc *findSqlFunc(SqliteDb *pDb, const char *zName){ return pNew; } +/* +** Free a single SqlPreparedStmt object. +*/ +static void dbFreeStmt(SqlPreparedStmt *pStmt){ +#ifdef SQLITE_TEST + if( sqlite3_sql(pStmt->pStmt)==0 ){ + Tcl_Free((char *)pStmt->zSql); + } +#endif + sqlite3_finalize(pStmt->pStmt); + Tcl_Free((char *)pStmt); +} + /* ** Finalize and free a list of prepared statements */ -static void flushStmtCache( SqliteDb *pDb ){ +static void flushStmtCache(SqliteDb *pDb){ SqlPreparedStmt *pPreStmt; + SqlPreparedStmt *pNext; - while( pDb->stmtList ){ - sqlite3_finalize( pDb->stmtList->pStmt ); - pPreStmt = pDb->stmtList; - pDb->stmtList = pDb->stmtList->pNext; - Tcl_Free( (char*)pPreStmt ); + for(pPreStmt = pDb->stmtList; pPreStmt; pPreStmt=pNext){ + pNext = pPreStmt->pNext; + dbFreeStmt(pPreStmt); } pDb->nStmt = 0; pDb->stmtLast = 0; + pDb->stmtList = 0; } /* @@ -1074,6 +1095,27 @@ static int DbTransPostCmd( return rc; } +/* +** Unless SQLITE_TEST is defined, this function is a simple wrapper around +** sqlite3_prepare_v2(). If SQLITE_TEST is defined, then it uses either +** sqlite3_prepare_v2() or legacy interface sqlite3_prepare(), depending +** on whether or not the [db_use_legacy_prepare] command has been used to +** configure the connection. +*/ +static int dbPrepare( + SqliteDb *pDb, /* Database object */ + const char *zSql, /* SQL to compile */ + sqlite3_stmt **ppStmt, /* OUT: Prepared statement */ + const char **pzOut /* OUT: Pointer to next SQL statement */ +){ +#ifdef SQLITE_TEST + if( pDb->bLegacyPrepare ){ + return sqlite3_prepare(pDb->db, zSql, -1, ppStmt, pzOut); + } +#endif + return sqlite3_prepare_v2(pDb->db, zSql, -1, ppStmt, pzOut); +} + /* ** Search the cache for a prepared-statement object that implements the ** first SQL statement in the buffer pointed to by parameter zIn. If @@ -1144,7 +1186,7 @@ static int dbPrepareAndBind( if( pPreStmt==0 ){ int nByte; - if( SQLITE_OK!=sqlite3_prepare_v2(pDb->db, zSql, -1, &pStmt, pzOut) ){ + if( SQLITE_OK!=dbPrepare(pDb, zSql, &pStmt, pzOut) ){ Tcl_SetObjResult(interp, dbTextToObj(sqlite3_errmsg(pDb->db))); return TCL_ERROR; } @@ -1171,6 +1213,14 @@ static int dbPrepareAndBind( pPreStmt->nSql = (*pzOut - zSql); pPreStmt->zSql = sqlite3_sql(pStmt); pPreStmt->apParm = (Tcl_Obj **)&pPreStmt[1]; +#ifdef SQLITE_TEST + if( pPreStmt->zSql==0 ){ + char *zCopy = Tcl_Alloc(pPreStmt->nSql + 1); + memcpy(zCopy, zSql, pPreStmt->nSql); + zCopy[pPreStmt->nSql] = '\0'; + pPreStmt->zSql = zCopy; + } +#endif } assert( pPreStmt ); assert( strlen30(pPreStmt->zSql)==pPreStmt->nSql ); @@ -1224,7 +1274,6 @@ static int dbPrepareAndBind( return TCL_OK; } - /* ** Release a statement reference obtained by calling dbPrepareAndBind(). ** There should be exactly one call to this function for each call to @@ -1249,8 +1298,7 @@ static void dbReleaseStmt( if( pDb->maxStmt<=0 || discard ){ /* If the cache is turned off, deallocated the statement */ - sqlite3_finalize(pPreStmt->pStmt); - Tcl_Free((char *)pPreStmt); + dbFreeStmt(pPreStmt); }else{ /* Add the prepared statement to the beginning of the cache list. */ pPreStmt->pNext = pDb->stmtList; @@ -1270,11 +1318,11 @@ static void dbReleaseStmt( /* If we have too many statement in cache, remove the surplus from ** the end of the cache list. */ while( pDb->nStmt>pDb->maxStmt ){ - sqlite3_finalize(pDb->stmtLast->pStmt); - pDb->stmtLast = pDb->stmtLast->pPrev; - Tcl_Free((char*)pDb->stmtLast->pNext); + SqlPreparedStmt *pLast = pDb->stmtLast; + pDb->stmtLast = pLast->pPrev; pDb->stmtLast->pNext = 0; pDb->nStmt--; + dbFreeStmt(pLast); } } } @@ -1407,9 +1455,12 @@ static void dbEvalRowInfo( ** no further rows available. This is similar to SQLITE_DONE. */ static int dbEvalStep(DbEvalContext *p){ + const char *zPrevSql = 0; /* Previous value of p->zSql */ + while( p->zSql[0] || p->pPreStmt ){ int rc; if( p->pPreStmt==0 ){ + zPrevSql = (p->zSql==zPrevSql ? 0 : p->zSql); rc = dbPrepareAndBind(p->pDb, p->zSql, &p->zSql, &p->pPreStmt); if( rc!=TCL_OK ) return rc; }else{ @@ -1436,8 +1487,19 @@ static int dbEvalStep(DbEvalContext *p){ if( rcs!=SQLITE_OK ){ /* If a run-time error occurs, report the error and stop reading ** the SQL. */ - Tcl_SetObjResult(pDb->interp, dbTextToObj(sqlite3_errmsg(pDb->db))); dbReleaseStmt(pDb, pPreStmt, 1); +#if SQLITE_TEST + if( p->pDb->bLegacyPrepare && rcs==SQLITE_SCHEMA && zPrevSql ){ + /* If the runtime error was an SQLITE_SCHEMA, and the database + ** handle is configured to use the legacy sqlite3_prepare() + ** interface, retry prepare()/step() on the same SQL statement. + ** This only happens once. If there is a second SQLITE_SCHEMA + ** error, the error will be returned to the caller. */ + p->zSql = zPrevSql; + continue; + } +#endif + Tcl_SetObjResult(pDb->interp, dbTextToObj(sqlite3_errmsg(pDb->db))); return TCL_ERROR; }else{ dbReleaseStmt(pDb, pPreStmt, 0); @@ -3072,7 +3134,7 @@ static int DbMain(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){ }else{ flags &= ~SQLITE_OPEN_NOMUTEX; } - }else if( strcmp(zArg, "-fullmutex")==0 ){ + }else if( strcmp(zArg, "-fullmutex")==0 ){ int b; if( Tcl_GetBooleanFromObj(interp, objv[i+1], &b) ) return TCL_ERROR; if( b ){ @@ -3673,6 +3735,44 @@ static int init_all_cmd( init_all(slave); return TCL_OK; } + +/* +** Tclcmd: db_use_legacy_prepare DB BOOLEAN +** +** The first argument to this command must be a database command created by +** [sqlite3]. If the second argument is true, then the handle is configured +** to use the sqlite3_prepare_v2() function to prepare statements. If it +** is false, sqlite3_prepare(). +*/ +static int db_use_legacy_prepare_cmd( + ClientData cd, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + Tcl_CmdInfo cmdInfo; + SqliteDb *pDb; + int bPrepare; + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB BOOLEAN"); + return TCL_ERROR; + } + + if( !Tcl_GetCommandInfo(interp, Tcl_GetString(objv[1]), &cmdInfo) ){ + Tcl_AppendResult(interp, "no such db: ", Tcl_GetString(objv[1]), (char*)0); + return TCL_ERROR; + } + pDb = (SqliteDb*)cmdInfo.objClientData; + if( Tcl_GetBooleanFromObj(interp, objv[2], &bPrepare) ){ + return TCL_ERROR; + } + + pDb->bLegacyPrepare = bPrepare; + + Tcl_ResetResult(interp); + return TCL_OK; +} #endif /* @@ -3783,7 +3883,12 @@ static void init_all(Tcl_Interp *interp){ Sqlitetestfts3_Init(interp); #endif - Tcl_CreateObjCommand(interp,"load_testfixture_extensions",init_all_cmd,0,0); + Tcl_CreateObjCommand( + interp, "load_testfixture_extensions", init_all_cmd, 0, 0 + ); + Tcl_CreateObjCommand( + interp, "db_use_legacy_prepare", db_use_legacy_prepare_cmd, 0, 0 + ); #ifdef SQLITE_SSE Sqlitetestsse_Init(interp); diff --git a/src/test1.c b/src/test1.c index 7fdf54b128..3301ab7403 100644 --- a/src/test1.c +++ b/src/test1.c @@ -5096,6 +5096,39 @@ static int file_control_lockproxy_test( return TCL_OK; } +/* +** tclcmd: file_control_win32_av_retry DB NRETRY DELAY +** +** This TCL command runs the sqlite3_file_control interface with +** the SQLITE_FCNTL_WIN32_AV_RETRY opcode. +*/ +static int file_control_win32_av_retry( + ClientData clientData, /* Pointer to sqlite3_enable_XXX function */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + sqlite3 *db; + int rc; + int a[2]; + char z[100]; + + if( objc!=4 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", + Tcl_GetStringFromObj(objv[0], 0), " DB NRETRY DELAY", 0); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){ + return TCL_ERROR; + } + if( Tcl_GetIntFromObj(interp, objv[2], &a[0]) ) return TCL_ERROR; + if( Tcl_GetIntFromObj(interp, objv[3], &a[1]) ) return TCL_ERROR; + rc = sqlite3_file_control(db, NULL, SQLITE_FCNTL_WIN32_AV_RETRY, (void*)a); + sqlite3_snprintf(sizeof(z), z, "%d %d %d", rc, a[0], a[1]); + Tcl_AppendResult(interp, z, (char*)0); + return TCL_OK; +} + /* ** tclcmd: sqlite3_vfs_list @@ -5574,6 +5607,94 @@ static int test_test_control( return TCL_OK; } +#if SQLITE_OS_WIN +/* +** Information passed from the main thread into the windows file locker +** background thread. +*/ +struct win32FileLocker { + HANDLE h; /* Handle of the file to be locked */ + int delay1; /* Delay before locking */ + int delay2; /* Delay before unlocking */ + int ok; /* Finished ok */ + int err; /* True if an error occurs */ +}; +#endif + + +#if SQLITE_OS_WIN +/* +** The background thread that does file locking. +*/ +static void win32_file_locker(void *pAppData){ + struct win32FileLocker *p = (struct win32FileLocker*)pAppData; + if( p->delay1 ) Sleep(p->delay1); + if( LockFile(p->h, 0, 0, 100000000, 0) ){ + Sleep(p->delay2); + UnlockFile(p->h, 0, 0, 100000000, 0); + p->ok = 1; + }else{ + p->err = 1; + } + CloseHandle(p->h); + p->h = 0; + p->delay1 = 0; + p->delay2 = 0; +} +#endif + +#if SQLITE_OS_WIN +/* +** lock_win32_file FILENAME DELAY1 DELAY2 +** +** Get an exclusive manditory lock on file for DELAY2 milliseconds. +** Wait DELAY1 milliseconds before acquiring the lock. +*/ +static int win32_file_lock( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + static struct win32FileLocker x = { 0, 0, 0 }; + const char *zFilename; + int retry = 0; + + if( objc!=4 && objc!=1 ){ + Tcl_WrongNumArgs(interp, 1, objv, "FILENAME DELAY1 DELAY2"); + return TCL_ERROR; + } + if( objc==1 ){ + char zBuf[200]; + sqlite3_snprintf(sizeof(zBuf), zBuf, "%d %d %d %d %d", + x.ok, x.err, x.delay1, x.delay2, x.h); + Tcl_AppendResult(interp, zBuf, (char*)0); + return TCL_OK; + } + while( x.h && retry<30 ){ + retry++; + Sleep(100); + } + if( x.h ){ + Tcl_AppendResult(interp, "busy", (char*)0); + return TCL_ERROR; + } + if( Tcl_GetIntFromObj(interp, objv[2], &x.delay1) ) return TCL_ERROR; + if( Tcl_GetIntFromObj(interp, objv[3], &x.delay2) ) return TCL_ERROR; + zFilename = Tcl_GetString(objv[1]); + x.h = CreateFile(zFilename, GENERIC_READ|GENERIC_WRITE, + FILE_SHARE_READ|FILE_SHARE_WRITE, 0, OPEN_ALWAYS, + FILE_ATTRIBUTE_NORMAL, 0); + if( !x.h ){ + Tcl_AppendResult(interp, "cannot open file: ", zFilename, (char*)0); + return TCL_ERROR; + } + _beginthread(win32_file_locker, 0, (void*)&x); + Sleep(0); + return TCL_OK; +} +#endif + /* ** optimization_control DB OPT BOOLEAN @@ -5754,6 +5875,9 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "restore_prng_state", restore_prng_state, 0 }, { "reset_prng_state", reset_prng_state, 0 }, { "optimization_control", optimization_control,0}, +#if SQLITE_OS_WIN + { "lock_win32_file", win32_file_lock, 0 }, +#endif { "tcl_objproc", runAsObjProc, 0 }, /* sqlite3_column_*() API */ @@ -5802,7 +5926,8 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "file_control_lasterrno_test", file_control_lasterrno_test, 0 }, { "file_control_lockproxy_test", file_control_lockproxy_test, 0 }, { "file_control_chunksize_test", file_control_chunksize_test, 0 }, - { "file_control_sizehint_test", file_control_sizehint_test, 0 }, + { "file_control_sizehint_test", file_control_sizehint_test, 0 }, + { "file_control_win32_av_retry", file_control_win32_av_retry, 0 }, { "sqlite3_vfs_list", vfs_list, 0 }, { "sqlite3_create_function_v2", test_create_function_v2, 0 }, diff --git a/src/test_config.c b/src/test_config.c index f8f82e596d..a848814ccd 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -583,6 +583,7 @@ Tcl_SetVar2(interp, "sqlite_options", "long_double", LINKVAR( DEFAULT_PAGE_SIZE ); LINKVAR( DEFAULT_FILE_FORMAT ); LINKVAR( MAX_ATTACHED ); + LINKVAR( MAX_DEFAULT_PAGE_SIZE ); { static const int cv_TEMP_STORE = SQLITE_TEMP_STORE; diff --git a/src/test_multiplex.c b/src/test_multiplex.c index f709c9a99f..732df82cd7 100644 --- a/src/test_multiplex.c +++ b/src/test_multiplex.c @@ -45,6 +45,7 @@ #include "sqlite3.h" #include #include +#include #include "test_multiplex.h" #ifndef SQLITE_CORE @@ -78,20 +79,26 @@ /************************ Shim Definitions ******************************/ -#define SQLITE_MULTIPLEX_VFS_NAME "multiplex" +#ifndef SQLITE_MULTIPLEX_VFS_NAME +# define SQLITE_MULTIPLEX_VFS_NAME "multiplex" +#endif /* This is the limit on the chunk size. It may be changed by calling ** the xFileControl() interface. It will be rounded up to a -** multiple of MAX_PAGE_SIZE. We default it here to 1GB. +** multiple of MAX_PAGE_SIZE. We default it here to 2GiB less 64KiB. */ -#define SQLITE_MULTIPLEX_CHUNK_SIZE (MAX_PAGE_SIZE*16384) +#ifndef SQLITE_MULTIPLEX_CHUNK_SIZE +# define SQLITE_MULTIPLEX_CHUNK_SIZE 2147418112 +#endif /* Default limit on number of chunks. Care should be taken ** so that values for chunks numbers fit in the SQLITE_MULTIPLEX_EXT_FMT ** format specifier. It may be changed by calling ** the xFileControl() interface. */ -#define SQLITE_MULTIPLEX_MAX_CHUNKS 32 +#ifndef SQLITE_MULTIPLEX_MAX_CHUNKS +# define SQLITE_MULTIPLEX_MAX_CHUNKS 32 +#endif /* If SQLITE_MULTIPLEX_EXT_OVWR is defined, the ** last SQLITE_MULTIPLEX_EXT_SZ characters of the @@ -119,13 +126,15 @@ typedef struct multiplexConn multiplexConn; ** group. */ struct multiplexGroup { - sqlite3_file **pReal; /* Handles to each chunk */ - char *bOpen; /* array of bools - 0 if chunk not opened */ + struct multiplexReal { /* For each chunk */ + sqlite3_file *p; /* Handle for the chunk */ + char *z; /* Name of this chunk */ + } *aReal; /* list of all chunks */ + int nReal; /* Number of chunks */ char *zName; /* Base filename of this group */ int nName; /* Length of base filename */ int flags; /* Flags used for original opening */ - int nChunkSize; /* Chunk size used for this group */ - int nMaxChunks; /* Max number of chunks for this group */ + unsigned int szChunk; /* Chunk size used for this group */ int bEnabled; /* TRUE to use Multiplex VFS for this file */ multiplexGroup *pNext, *pPrev; /* Doubly linked list of all group objects */ }; @@ -184,12 +193,6 @@ static struct { /* List of multiplexGroup objects. */ multiplexGroup *pGroups; - - /* Storage for temp file names. Allocated during - ** initialization to the max pathname of the underlying VFS. - */ - char *zName; - } gMultiplex; /************************* Utility Routines *********************************/ @@ -265,7 +268,8 @@ static int multiplexGetTempname(sqlite3_vfs *pOrigVfs, int nBuf, char *zBuf){ attempts++; sqlite3_randomness(8, &zBuf[j]); for(i=0; i<8; i++){ - zBuf[j+i] = (char)zChars[ ((unsigned char)zBuf[j+i])%(sizeof(zChars)-1) ]; + unsigned char uc = (unsigned char)zBuf[j+i]; + zBuf[j+i] = (char)zChars[uc%(sizeof(zChars)-1)]; } memcpy(&zBuf[j+i], ".tmp", 5); rc = pOrigVfs->xAccess(pOrigVfs, zBuf, SQLITE_ACCESS_EXISTS, &exists); @@ -279,35 +283,69 @@ static int multiplexGetTempname(sqlite3_vfs *pOrigVfs, int nBuf, char *zBuf){ return rc; } +/* Compute the filename for the iChunk-th chunk +*/ +static int multiplexSubFilename(multiplexGroup *pGroup, int iChunk){ + if( iChunk>=pGroup->nReal ){ + struct multiplexReal *p; + p = sqlite3_realloc(pGroup->aReal, (iChunk+1)*sizeof(*p)); + if( p==0 ){ + return SQLITE_NOMEM; + } + memset(&p[pGroup->nReal], 0, sizeof(p[0])*(iChunk+1-pGroup->nReal)); + pGroup->aReal = p; + pGroup->nReal = iChunk+1; + } + if( pGroup->aReal[iChunk].z==0 ){ + char *z; + int n = pGroup->nName; + pGroup->aReal[iChunk].z = z = sqlite3_malloc( n+3 ); + if( z==0 ){ + return SQLITE_NOMEM; + } + memcpy(z, pGroup->zName, n+1); + if( iChunk>0 ){ +#ifdef SQLITE_ENABLE_8_3_NAMES + if( n>3 && z[n-3]=='.' ){ + n--; + }else if( n>4 && z[n-4]=='.' ){ + n -= 2; + } +#endif + sqlite3_snprintf(3,&z[n],"%02d",iChunk); + } + } + return SQLITE_OK; +} + /* Translate an sqlite3_file* that is really a multiplexGroup* into ** the sqlite3_file* for the underlying original VFS. */ -static sqlite3_file *multiplexSubOpen(multiplexConn *pConn, int iChunk, int *rc, int *pOutFlags){ - multiplexGroup *pGroup = pConn->pGroup; +static sqlite3_file *multiplexSubOpen( + multiplexGroup *pGroup, + int iChunk, + int *rc, + int *pOutFlags +){ + sqlite3_file *pSubOpen = 0; sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ - if( iChunknMaxChunks ){ - sqlite3_file *pSubOpen = pGroup->pReal[iChunk]; /* Real file descriptor */ - if( !pGroup->bOpen[iChunk] ){ - memcpy(gMultiplex.zName, pGroup->zName, pGroup->nName+1); - if( iChunk ){ -#ifdef SQLITE_MULTIPLEX_EXT_OVWR - sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, gMultiplex.zName+pGroup->nName-SQLITE_MULTIPLEX_EXT_SZ, SQLITE_MULTIPLEX_EXT_FMT, iChunk); -#else - sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, gMultiplex.zName+pGroup->nName, SQLITE_MULTIPLEX_EXT_FMT, iChunk); -#endif - } - *rc = pOrigVfs->xOpen(pOrigVfs, gMultiplex.zName, pSubOpen, pGroup->flags, pOutFlags); - if( *rc==SQLITE_OK ){ - pGroup->bOpen[iChunk] = -1; - return pSubOpen; - } - return NULL; + *rc = multiplexSubFilename(pGroup, iChunk); + if( (*rc)==SQLITE_OK && (pSubOpen = pGroup->aReal[iChunk].p)==0 ){ + pSubOpen = sqlite3_malloc( pOrigVfs->szOsFile ); + if( pSubOpen==0 ){ + *rc = SQLITE_NOMEM; + return 0; + } + pGroup->aReal[iChunk].p = pSubOpen; + *rc = pOrigVfs->xOpen(pOrigVfs, pGroup->aReal[iChunk].z, pSubOpen, + pGroup->flags, pOutFlags); + if( *rc!=SQLITE_OK ){ + sqlite3_free(pSubOpen); + pGroup->aReal[iChunk].p = 0; + return 0; } - *rc = SQLITE_OK; - return pSubOpen; } - *rc = SQLITE_FULL; - return NULL; + return pSubOpen; } /* @@ -366,6 +404,36 @@ static int multiplexFuncInit( return rc; } +/* +** Close a single sub-file in the connection group. +*/ +static void multiplexSubClose( + multiplexGroup *pGroup, + int iChunk, + sqlite3_vfs *pOrigVfs +){ + sqlite3_file *pSubOpen = pGroup->aReal[iChunk].p; + if( pSubOpen ){ + if( pOrigVfs ) pOrigVfs->xDelete(pOrigVfs, pGroup->aReal[iChunk].z, 0); + pSubOpen->pMethods->xClose(pSubOpen); + sqlite3_free(pGroup->aReal[iChunk].p); + } + sqlite3_free(pGroup->aReal[iChunk].z); + memset(&pGroup->aReal[iChunk], 0, sizeof(pGroup->aReal[iChunk])); +} + +/* +** Deallocate memory held by a multiplexGroup +*/ +static void multiplexFreeComponents(multiplexGroup *pGroup){ + int i; + for(i=0; inReal; i++){ multiplexSubClose(pGroup, i, 0); } + sqlite3_free(pGroup->aReal); + pGroup->aReal = 0; + pGroup->nReal = 0; +} + + /************************* VFS Method Wrappers *****************************/ /* @@ -382,16 +450,17 @@ static int multiplexOpen( int flags, /* Flags to control the opening */ int *pOutFlags /* Flags showing results of opening */ ){ - int rc = SQLITE_OK; /* Result code */ - multiplexConn *pMultiplexOpen; /* The new multiplex file descriptor */ - multiplexGroup *pGroup; /* Corresponding multiplexGroup object */ - sqlite3_file *pSubOpen; /* Real file descriptor */ + int rc = SQLITE_OK; /* Result code */ + multiplexConn *pMultiplexOpen; /* The new multiplex file descriptor */ + multiplexGroup *pGroup; /* Corresponding multiplexGroup object */ + sqlite3_file *pSubOpen = 0; /* Real file descriptor */ sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ int nName; - int i; int sz; + char *zToFree = 0; UNUSED_PARAMETER(pVfs); + memset(pConn, 0, pVfs->szOsFile); /* We need to create a group structure and manage ** access to this group of files. @@ -405,28 +474,22 @@ static int multiplexOpen( ** it. */ if( !zName ){ - rc = multiplexGetTempname(pOrigVfs, pOrigVfs->mxPathname, gMultiplex.zName); - zName = gMultiplex.zName; + zName = zToFree = sqlite3_malloc( pOrigVfs->mxPathname + 10 ); + if( zName==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = multiplexGetTempname(pOrigVfs, pOrigVfs->mxPathname, zToFree); + } } if( rc==SQLITE_OK ){ /* allocate space for group */ nName = multiplexStrlen30(zName); - sz = sizeof(multiplexGroup) /* multiplexGroup */ - + (sizeof(sqlite3_file *)*SQLITE_MULTIPLEX_MAX_CHUNKS) /* pReal[] */ - + (pOrigVfs->szOsFile*SQLITE_MULTIPLEX_MAX_CHUNKS) /* *pReal */ - + SQLITE_MULTIPLEX_MAX_CHUNKS /* bOpen[] */ - + nName + 1; /* zName */ -#ifndef SQLITE_MULTIPLEX_EXT_OVWR - sz += SQLITE_MULTIPLEX_EXT_SZ; - assert(nName+SQLITE_MULTIPLEX_EXT_SZ < pOrigVfs->mxPathname); -#else - assert(nName >= SQLITE_MULTIPLEX_EXT_SZ); - assert(nName < pOrigVfs->mxPathname); -#endif + sz = sizeof(multiplexGroup) /* multiplexGroup */ + + nName + 1; /* zName */ pGroup = sqlite3_malloc( sz ); if( pGroup==0 ){ - rc=SQLITE_NOMEM; + rc = SQLITE_NOMEM; } } @@ -436,32 +499,58 @@ static int multiplexOpen( pMultiplexOpen->pGroup = pGroup; memset(pGroup, 0, sz); pGroup->bEnabled = -1; - pGroup->nChunkSize = SQLITE_MULTIPLEX_CHUNK_SIZE; - pGroup->nMaxChunks = SQLITE_MULTIPLEX_MAX_CHUNKS; - pGroup->pReal = (sqlite3_file **)p; - p += (sizeof(sqlite3_file *)*pGroup->nMaxChunks); - for(i=0; inMaxChunks; i++){ - pGroup->pReal[i] = (sqlite3_file *)p; - p += pOrigVfs->szOsFile; + pGroup->szChunk = SQLITE_MULTIPLEX_CHUNK_SIZE; + if( flags & SQLITE_OPEN_URI ){ + const char *zChunkSize; + zChunkSize = sqlite3_uri_parameter(zName, "chunksize"); + if( zChunkSize ){ + unsigned int n = 0; + int i; + for(i=0; zChunkSize[i]>='0' && zChunkSize[i]<='9'; i++){ + n = n*10 + zChunkSize[i] - '0'; + } + if( n>0 ){ + pGroup->szChunk = (n+0xffff)&~0xffff; + }else{ + /* A zero or negative chunksize disabled the multiplexor */ + pGroup->bEnabled = 0; + } + } } - /* bOpen[] vals should all be zero from memset above */ - pGroup->bOpen = p; - p += pGroup->nMaxChunks; pGroup->zName = p; /* save off base filename, name length, and original open flags */ memcpy(pGroup->zName, zName, nName+1); pGroup->nName = nName; pGroup->flags = flags; - pSubOpen = multiplexSubOpen(pMultiplexOpen, 0, &rc, pOutFlags); + rc = multiplexSubFilename(pGroup, 1); + if( rc==SQLITE_OK ){ + pSubOpen = multiplexSubOpen(pGroup, 0, &rc, pOutFlags); + } if( pSubOpen ){ - /* if this file is already larger than chunk size, disable - ** the multiplex feature. - */ + int exists, rc2, rc3; sqlite3_int64 sz; - int rc2 = pSubOpen->pMethods->xFileSize(pSubOpen, &sz); - if( (rc2==SQLITE_OK) && (sz>pGroup->nChunkSize) ){ - pGroup->bEnabled = 0; + + rc2 = pSubOpen->pMethods->xFileSize(pSubOpen, &sz); + if( rc2==SQLITE_OK ){ + /* If the first overflow file exists and if the size of the main file + ** is different from the chunk size, that means the chunk size is set + ** set incorrectly. So fix it. + ** + ** Or, if the first overflow file does not exist and the main file is + ** larger than the chunk size, that means the chunk size is too small. + ** But we have no way of determining the intended chunk size, so + ** just disable the multiplexor all togethre. + */ + rc3 = pOrigVfs->xAccess(pOrigVfs, pGroup->aReal[1].z, + SQLITE_ACCESS_EXISTS, &exists); + if( rc3==SQLITE_OK && exists && sz==(sz&0xffff0000) && sz>0 + && sz!=pGroup->szChunk ){ + pGroup->szChunk = sz; + }else if( rc3==SQLITE_OK && !exists && sz>pGroup->szChunk ){ + pGroup->bEnabled = 0; + } } + if( pSubOpen->pMethods->iVersion==1 ){ pMultiplexOpen->base.pMethods = &gMultiplex.sIoMethodsV1; }else{ @@ -472,17 +561,18 @@ static int multiplexOpen( if( gMultiplex.pGroups ) gMultiplex.pGroups->pPrev = pGroup; gMultiplex.pGroups = pGroup; }else{ + multiplexFreeComponents(pGroup); sqlite3_free(pGroup); } } multiplexLeave(); + sqlite3_free(zToFree); return rc; } /* ** This is the xDelete method used for the "multiplex" VFS. -** It attempts to delete the filename specified, as well -** as additional files with the SQLITE_MULTIPLEX_EXT_FMT extension. +** It attempts to delete the filename specified. */ static int multiplexDelete( sqlite3_vfs *pVfs, /* The multiplex VFS */ @@ -490,41 +580,7 @@ static int multiplexDelete( int syncDir ){ sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ - int rc = SQLITE_OK; - int nName = multiplexStrlen30(zName); - int i; - - UNUSED_PARAMETER(pVfs); - - multiplexEnter(); - memcpy(gMultiplex.zName, zName, nName+1); - for(i=0; ixAccess(pOrigVfs, gMultiplex.zName, - SQLITE_ACCESS_EXISTS, &exists); - if( rc2==SQLITE_OK && exists ){ - /* if it exists, delete it */ - rc2 = pOrigVfs->xDelete(pOrigVfs, gMultiplex.zName, syncDir); - if( rc2!=SQLITE_OK ) rc = rc2; - }else{ - /* stop at first "gap" */ - break; - } - } - multiplexLeave(); - return rc; + return pOrigVfs->xDelete(pOrigVfs, zName, syncDir); } static int multiplexAccess(sqlite3_vfs *a, const char *b, int c, int *d){ @@ -572,17 +628,8 @@ static int multiplexClose(sqlite3_file *pConn){ multiplexConn *p = (multiplexConn*)pConn; multiplexGroup *pGroup = p->pGroup; int rc = SQLITE_OK; - int i; multiplexEnter(); - /* close any open handles */ - for(i=0; inMaxChunks; i++){ - if( pGroup->bOpen[i] ){ - sqlite3_file *pSubOpen = pGroup->pReal[i]; - int rc2 = pSubOpen->pMethods->xClose(pSubOpen); - if( rc2!=SQLITE_OK ) rc = rc2; - pGroup->bOpen[i] = 0; - } - } + multiplexFreeComponents(pGroup); /* remove from linked list */ if( pGroup->pNext ) pGroup->pNext->pPrev = pGroup->pPrev; if( pGroup->pPrev ){ @@ -610,17 +657,22 @@ static int multiplexRead( int rc = SQLITE_OK; multiplexEnter(); if( !pGroup->bEnabled ){ - sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); - rc = ( !pSubOpen ) ? SQLITE_IOERR_READ : pSubOpen->pMethods->xRead(pSubOpen, pBuf, iAmt, iOfst); + sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL); + if( pSubOpen==0 ){ + rc = SQLITE_IOERR_READ; + }else{ + rc = pSubOpen->pMethods->xRead(pSubOpen, pBuf, iAmt, iOfst); + } }else{ while( iAmt > 0 ){ - int i = (int)(iOfst / pGroup->nChunkSize); - sqlite3_file *pSubOpen = multiplexSubOpen(p, i, &rc, NULL); + int i = (int)(iOfst / pGroup->szChunk); + sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, i, &rc, NULL); if( pSubOpen ){ - int extra = ((int)(iOfst % pGroup->nChunkSize) + iAmt) - pGroup->nChunkSize; + int extra = ((int)(iOfst % pGroup->szChunk) + iAmt) - pGroup->szChunk; if( extra<0 ) extra = 0; iAmt -= extra; - rc = pSubOpen->pMethods->xRead(pSubOpen, pBuf, iAmt, iOfst % pGroup->nChunkSize); + rc = pSubOpen->pMethods->xRead(pSubOpen, pBuf, iAmt, + iOfst % pGroup->szChunk); if( rc!=SQLITE_OK ) break; pBuf = (char *)pBuf + iAmt; iOfst += iAmt; @@ -650,17 +702,23 @@ static int multiplexWrite( int rc = SQLITE_OK; multiplexEnter(); if( !pGroup->bEnabled ){ - sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); - rc = ( !pSubOpen ) ? SQLITE_IOERR_WRITE : pSubOpen->pMethods->xWrite(pSubOpen, pBuf, iAmt, iOfst); + sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL); + if( pSubOpen==0 ){ + rc = SQLITE_IOERR_WRITE; + }else{ + rc = pSubOpen->pMethods->xWrite(pSubOpen, pBuf, iAmt, iOfst); + } }else{ while( iAmt > 0 ){ - int i = (int)(iOfst / pGroup->nChunkSize); - sqlite3_file *pSubOpen = multiplexSubOpen(p, i, &rc, NULL); + int i = (int)(iOfst / pGroup->szChunk); + sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, i, &rc, NULL); if( pSubOpen ){ - int extra = ((int)(iOfst % pGroup->nChunkSize) + iAmt) - pGroup->nChunkSize; + int extra = ((int)(iOfst % pGroup->szChunk) + iAmt) - + pGroup->szChunk; if( extra<0 ) extra = 0; iAmt -= extra; - rc = pSubOpen->pMethods->xWrite(pSubOpen, pBuf, iAmt, iOfst % pGroup->nChunkSize); + rc = pSubOpen->pMethods->xWrite(pSubOpen, pBuf, iAmt, + iOfst % pGroup->szChunk); if( rc!=SQLITE_OK ) break; pBuf = (char *)pBuf + iAmt; iOfst += iAmt; @@ -685,38 +743,24 @@ static int multiplexTruncate(sqlite3_file *pConn, sqlite3_int64 size){ int rc = SQLITE_OK; multiplexEnter(); if( !pGroup->bEnabled ){ - sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); - rc = ( !pSubOpen ) ? SQLITE_IOERR_TRUNCATE : pSubOpen->pMethods->xTruncate(pSubOpen, size); + sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL); + if( pSubOpen==0 ){ + rc = SQLITE_IOERR_TRUNCATE; + }else{ + rc = pSubOpen->pMethods->xTruncate(pSubOpen, size); + } }else{ int rc2; int i; sqlite3_file *pSubOpen; sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ - memcpy(gMultiplex.zName, pGroup->zName, pGroup->nName+1); /* delete the chunks above the truncate limit */ - for(i=(int)(size / pGroup->nChunkSize)+1; inMaxChunks; i++){ - /* close any open chunks before deleting them */ - if( pGroup->bOpen[i] ){ - pSubOpen = pGroup->pReal[i]; - rc2 = pSubOpen->pMethods->xClose(pSubOpen); - if( rc2!=SQLITE_OK ) rc = SQLITE_IOERR_TRUNCATE; - pGroup->bOpen[i] = 0; - } -#ifdef SQLITE_MULTIPLEX_EXT_OVWR - sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, - gMultiplex.zName+pGroup->nName-SQLITE_MULTIPLEX_EXT_SZ, - SQLITE_MULTIPLEX_EXT_FMT, i); -#else - sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, - gMultiplex.zName+pGroup->nName, - SQLITE_MULTIPLEX_EXT_FMT, i); -#endif - rc2 = pOrigVfs->xDelete(pOrigVfs, gMultiplex.zName, 0); - if( rc2!=SQLITE_OK ) rc = SQLITE_IOERR_TRUNCATE; + for(i=(int)(size / pGroup->szChunk)+1; inReal; i++){ + multiplexSubClose(pGroup, i, pOrigVfs); } - pSubOpen = multiplexSubOpen(p, (int)(size / pGroup->nChunkSize), &rc2, NULL); + pSubOpen = multiplexSubOpen(pGroup, (int)(size/pGroup->szChunk), &rc2,0); if( pSubOpen ){ - rc2 = pSubOpen->pMethods->xTruncate(pSubOpen, size % pGroup->nChunkSize); + rc2 = pSubOpen->pMethods->xTruncate(pSubOpen, size % pGroup->szChunk); if( rc2!=SQLITE_OK ) rc = rc2; }else{ rc = SQLITE_IOERR_TRUNCATE; @@ -734,10 +778,9 @@ static int multiplexSync(sqlite3_file *pConn, int flags){ int rc = SQLITE_OK; int i; multiplexEnter(); - for(i=0; inMaxChunks; i++){ - /* if we don't have it open, we don't need to sync it */ - if( pGroup->bOpen[i] ){ - sqlite3_file *pSubOpen = pGroup->pReal[i]; + for(i=0; inReal; i++){ + sqlite3_file *pSubOpen = pGroup->aReal[i].p; + if( pSubOpen ){ int rc2 = pSubOpen->pMethods->xSync(pSubOpen, flags); if( rc2!=SQLITE_OK ) rc = rc2; } @@ -757,39 +800,28 @@ static int multiplexFileSize(sqlite3_file *pConn, sqlite3_int64 *pSize){ int i; multiplexEnter(); if( !pGroup->bEnabled ){ - sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); - rc = ( !pSubOpen ) ? SQLITE_IOERR_FSTAT : pSubOpen->pMethods->xFileSize(pSubOpen, pSize); + sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL); + if( pSubOpen==0 ){ + rc = SQLITE_IOERR_FSTAT; + }else{ + rc = pSubOpen->pMethods->xFileSize(pSubOpen, pSize); + } }else{ + sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; *pSize = 0; - for(i=0; inMaxChunks; i++){ - sqlite3_file *pSubOpen = NULL; - /* if not opened already, check to see if the chunk exists */ - if( pGroup->bOpen[i] ){ - pSubOpen = pGroup->pReal[i]; + for(i=0; 1; i++){ + sqlite3_file *pSubOpen = 0; + int exists = 0; + rc = multiplexSubFilename(pGroup, i); + if( rc ) break; + rc2 = pOrigVfs->xAccess(pOrigVfs, pGroup->aReal[i].z, + SQLITE_ACCESS_EXISTS, &exists); + if( rc2==SQLITE_OK && exists){ + /* if it exists, open it */ + pSubOpen = multiplexSubOpen(pGroup, i, &rc, NULL); }else{ - sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ - int exists = 0; - memcpy(gMultiplex.zName, pGroup->zName, pGroup->nName+1); - if( i ){ -#ifdef SQLITE_MULTIPLEX_EXT_OVWR - sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, - gMultiplex.zName+pGroup->nName-SQLITE_MULTIPLEX_EXT_SZ, - SQLITE_MULTIPLEX_EXT_FMT, i); -#else - sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, - gMultiplex.zName+pGroup->nName, - SQLITE_MULTIPLEX_EXT_FMT, i); -#endif - } - rc2 = pOrigVfs->xAccess(pOrigVfs, gMultiplex.zName, - SQLITE_ACCESS_EXISTS, &exists); - if( rc2==SQLITE_OK && exists){ - /* if it exists, open it */ - pSubOpen = multiplexSubOpen(p, i, &rc, NULL); - }else{ - /* stop at first "gap" */ - break; - } + /* stop at first "gap" */ + break; } if( pSubOpen ){ sqlite3_int64 sz; @@ -797,7 +829,7 @@ static int multiplexFileSize(sqlite3_file *pConn, sqlite3_int64 *pSize){ if( rc2!=SQLITE_OK ){ rc = rc2; }else{ - if( sz>pGroup->nChunkSize ){ + if( sz>pGroup->szChunk ){ rc = SQLITE_IOERR_FSTAT; } *pSize += sz; @@ -816,7 +848,7 @@ static int multiplexFileSize(sqlite3_file *pConn, sqlite3_int64 *pSize){ static int multiplexLock(sqlite3_file *pConn, int lock){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); if( pSubOpen ){ return pSubOpen->pMethods->xLock(pSubOpen, lock); } @@ -828,7 +860,7 @@ static int multiplexLock(sqlite3_file *pConn, int lock){ static int multiplexUnlock(sqlite3_file *pConn, int lock){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); if( pSubOpen ){ return pSubOpen->pMethods->xUnlock(pSubOpen, lock); } @@ -840,7 +872,7 @@ static int multiplexUnlock(sqlite3_file *pConn, int lock){ static int multiplexCheckReservedLock(sqlite3_file *pConn, int *pResOut){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); if( pSubOpen ){ return pSubOpen->pMethods->xCheckReservedLock(pSubOpen, pResOut); } @@ -867,28 +899,20 @@ static int multiplexFileControl(sqlite3_file *pConn, int op, void *pArg){ break; case MULTIPLEX_CTRL_SET_CHUNK_SIZE: if( pArg ) { - int nChunkSize = *(int *)pArg; - if( nChunkSize<1 ){ + unsigned int szChunk = *(unsigned*)pArg; + if( szChunk<1 ){ rc = SQLITE_MISUSE; }else{ /* Round up to nearest multiple of MAX_PAGE_SIZE. */ - nChunkSize = (nChunkSize + (MAX_PAGE_SIZE-1)); - nChunkSize &= ~(MAX_PAGE_SIZE-1); - pGroup->nChunkSize = nChunkSize; + szChunk = (szChunk + (MAX_PAGE_SIZE-1)); + szChunk &= ~(MAX_PAGE_SIZE-1); + pGroup->szChunk = szChunk; rc = SQLITE_OK; } } break; case MULTIPLEX_CTRL_SET_MAX_CHUNKS: - if( pArg ) { - int nMaxChunks = *(int *)pArg; - if(( nMaxChunks<1 ) || ( nMaxChunks>SQLITE_MULTIPLEX_MAX_CHUNKS )){ - rc = SQLITE_MISUSE; - }else{ - pGroup->nMaxChunks = nMaxChunks; - rc = SQLITE_OK; - } - } + rc = SQLITE_OK; break; case SQLITE_FCNTL_SIZE_HINT: case SQLITE_FCNTL_CHUNK_SIZE: @@ -896,7 +920,7 @@ static int multiplexFileControl(sqlite3_file *pConn, int op, void *pArg){ rc = SQLITE_OK; break; default: - pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL); if( pSubOpen ){ rc = pSubOpen->pMethods->xFileControl(pSubOpen, op, pArg); } @@ -910,7 +934,7 @@ static int multiplexFileControl(sqlite3_file *pConn, int op, void *pArg){ static int multiplexSectorSize(sqlite3_file *pConn){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); if( pSubOpen ){ return pSubOpen->pMethods->xSectorSize(pSubOpen); } @@ -922,7 +946,7 @@ static int multiplexSectorSize(sqlite3_file *pConn){ static int multiplexDeviceCharacteristics(sqlite3_file *pConn){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); if( pSubOpen ){ return pSubOpen->pMethods->xDeviceCharacteristics(pSubOpen); } @@ -940,9 +964,9 @@ static int multiplexShmMap( ){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); if( pSubOpen ){ - return pSubOpen->pMethods->xShmMap(pSubOpen, iRegion, szRegion, bExtend, pp); + return pSubOpen->pMethods->xShmMap(pSubOpen, iRegion, szRegion, bExtend,pp); } return SQLITE_IOERR; } @@ -957,7 +981,7 @@ static int multiplexShmLock( ){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); if( pSubOpen ){ return pSubOpen->pMethods->xShmLock(pSubOpen, ofst, n, flags); } @@ -969,7 +993,7 @@ static int multiplexShmLock( static void multiplexShmBarrier(sqlite3_file *pConn){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); if( pSubOpen ){ pSubOpen->pMethods->xShmBarrier(pSubOpen); } @@ -980,7 +1004,7 @@ static void multiplexShmBarrier(sqlite3_file *pConn){ static int multiplexShmUnmap(sqlite3_file *pConn, int deleteFlag){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); if( pSubOpen ){ return pSubOpen->pMethods->xShmUnmap(pSubOpen, deleteFlag); } @@ -1010,11 +1034,6 @@ int sqlite3_multiplex_initialize(const char *zOrigVfsName, int makeDefault){ if( !gMultiplex.pMutex ){ return SQLITE_NOMEM; } - gMultiplex.zName = sqlite3_malloc(pOrigVfs->mxPathname); - if( !gMultiplex.zName ){ - sqlite3_mutex_free(gMultiplex.pMutex); - return SQLITE_NOMEM; - } gMultiplex.pGroups = NULL; gMultiplex.isInitialized = 1; gMultiplex.pOrigVfs = pOrigVfs; @@ -1047,7 +1066,8 @@ int sqlite3_multiplex_initialize(const char *zOrigVfsName, int makeDefault){ gMultiplex.sIoMethodsV1.xCheckReservedLock = multiplexCheckReservedLock; gMultiplex.sIoMethodsV1.xFileControl = multiplexFileControl; gMultiplex.sIoMethodsV1.xSectorSize = multiplexSectorSize; - gMultiplex.sIoMethodsV1.xDeviceCharacteristics = multiplexDeviceCharacteristics; + gMultiplex.sIoMethodsV1.xDeviceCharacteristics = + multiplexDeviceCharacteristics; gMultiplex.sIoMethodsV2 = gMultiplex.sIoMethodsV1; gMultiplex.sIoMethodsV2.iVersion = 2; gMultiplex.sIoMethodsV2.xShmMap = multiplexShmMap; @@ -1074,7 +1094,6 @@ int sqlite3_multiplex_shutdown(void){ if( gMultiplex.isInitialized==0 ) return SQLITE_MISUSE; if( gMultiplex.pGroups ) return SQLITE_MISUSE; gMultiplex.isInitialized = 0; - sqlite3_free(gMultiplex.zName); sqlite3_mutex_free(gMultiplex.pMutex); sqlite3_vfs_unregister(&gMultiplex.sThisVfs); memset(&gMultiplex, 0, sizeof(gMultiplex)); @@ -1176,16 +1195,16 @@ static int test_multiplex_dump( Tcl_NewIntObj(pGroup->flags)); /* count number of chunks with open handles */ - for(i=0; inMaxChunks; i++){ - if( pGroup->bOpen[i] ) nChunks++; + for(i=0; inReal; i++){ + if( pGroup->aReal[i].p!=0 ) nChunks++; } Tcl_ListObjAppendElement(interp, pGroupTerm, Tcl_NewIntObj(nChunks)); Tcl_ListObjAppendElement(interp, pGroupTerm, - Tcl_NewIntObj(pGroup->nChunkSize)); + Tcl_NewIntObj(pGroup->szChunk)); Tcl_ListObjAppendElement(interp, pGroupTerm, - Tcl_NewIntObj(pGroup->nMaxChunks)); + Tcl_NewIntObj(pGroup->nReal)); Tcl_ListObjAppendElement(interp, pResult, pGroupTerm); } diff --git a/src/trigger.c b/src/trigger.c index 6242de5e6c..22c4877b6a 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -117,15 +117,28 @@ void sqlite3BeginTrigger( goto trigger_cleanup; } } + if( !pTableName || db->mallocFailed ){ + goto trigger_cleanup; + } + + /* A long-standing parser bug is that this syntax was allowed: + ** + ** CREATE TRIGGER attached.demo AFTER INSERT ON attached.tab .... + ** ^^^^^^^^ + ** + ** To maintain backwards compatibility, ignore the database + ** name on pTableName if we are reparsing our of SQLITE_MASTER. + */ + if( db->init.busy && iDb!=1 ){ + sqlite3DbFree(db, pTableName->a[0].zDatabase); + pTableName->a[0].zDatabase = 0; + } /* If the trigger name was unqualified, and the table is a temp table, ** then set iDb to 1 to create the trigger in the temporary database. ** If sqlite3SrcListLookup() returns 0, indicating the table does not ** exist, the error is caught by the block below. */ - if( !pTableName || db->mallocFailed ){ - goto trigger_cleanup; - } pTab = sqlite3SrcListLookup(pParse, pTableName); if( db->init.busy==0 && pName2->n==0 && pTab && pTab->pSchema==db->aDb[1].pSchema ){ diff --git a/src/update.c b/src/update.c index a2373dea59..97c14e8f33 100644 --- a/src/update.c +++ b/src/update.c @@ -311,7 +311,9 @@ void sqlite3Update( /* Begin the database scan */ sqlite3VdbeAddOp2(v, OP_Null, 0, regOldRowid); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere,0, WHERE_ONEPASS_DESIRED); + pWInfo = sqlite3WhereBegin( + pParse, pTabList, pWhere, 0, 0, WHERE_ONEPASS_DESIRED + ); if( pWInfo==0 ) goto update_cleanup; okOnePass = pWInfo->okOnePass; diff --git a/src/util.c b/src/util.c index de73577203..67e43b4ba8 100644 --- a/src/util.c +++ b/src/util.c @@ -1149,12 +1149,15 @@ int sqlite3AbsInt32(int x){ #ifdef SQLITE_ENABLE_8_3_NAMES /* -** If SQLITE_ENABLE_8_3_NAME is set at compile-time and if the database +** If SQLITE_ENABLE_8_3_NAMES is set at compile-time and if the database ** filename in zBaseFilename is a URI with the "8_3_names=1" parameter and ** if filename in z[] has a suffix (a.k.a. "extension") that is longer than ** three characters, then shorten the suffix on z[] to be the last three ** characters of the original suffix. ** +** If SQLITE_ENABLE_8_3_NAMES is set to 2 at compile-time, then always +** do the suffix shortening regardless of URI parameter. +** ** Examples: ** ** test.db-journal => test.nal @@ -1162,9 +1165,12 @@ int sqlite3AbsInt32(int x){ ** test.db-shm => test.shm */ void sqlite3FileSuffix3(const char *zBaseFilename, char *z){ +#if SQLITE_ENABLE_8_3_NAMES<2 const char *zOk; zOk = sqlite3_uri_parameter(zBaseFilename, "8_3_names"); - if( zOk && sqlite3GetBoolean(zOk) ){ + if( zOk && sqlite3GetBoolean(zOk) ) +#endif + { int i, sz; sz = sqlite3Strlen30(z); for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){} diff --git a/src/vdbe.c b/src/vdbe.c index f925a9d68d..9c0fb5c032 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3182,7 +3182,7 @@ case OP_OpenEphemeral: { if( pOp->p4.pKeyInfo ){ int pgno; assert( pOp->p4type==P4_KEYINFO ); - rc = sqlite3BtreeCreateTable(pCx->pBt, &pgno, BTREE_BLOBKEY); + rc = sqlite3BtreeCreateTable(pCx->pBt, &pgno, BTREE_BLOBKEY | pOp->p5); if( rc==SQLITE_OK ){ assert( pgno==MASTER_ROOT+1 ); rc = sqlite3BtreeCursor(pCx->pBt, pgno, 1, diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 136a8cd168..dfe8f87a15 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -488,7 +488,7 @@ int sqlite3_step(sqlite3_stmt *pStmt){ && cnt++ < SQLITE_MAX_SCHEMA_RETRY && (rc2 = rc = sqlite3Reprepare(v))==SQLITE_OK ){ sqlite3_reset(pStmt); - v->expired = 0; + assert( v->expired==0 ); } if( rc2!=SQLITE_OK && ALWAYS(v->isPrepareV2) && ALWAYS(db->pErr) ){ /* This case occurs after failing to recompile an sql statement. diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 8c442fce51..3fd9c2b52d 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1506,6 +1506,7 @@ void sqlite3VdbeMakeReady( memset(zCsr, 0, zEnd-zCsr); zCsr += (zCsr - (u8*)0)&7; assert( EIGHT_BYTE_ALIGNMENT(zCsr) ); + p->expired = 0; /* Memory for registers, parameters, cursor, etc, is allocated in two ** passes. On the first pass, we try to reuse unused space at the diff --git a/src/where.c b/src/where.c index cf30d94d67..d312232868 100644 --- a/src/where.c +++ b/src/where.c @@ -253,6 +253,7 @@ struct WhereCost { #define WHERE_VIRTUALTABLE 0x08000000 /* Use virtual-table processing */ #define WHERE_MULTI_OR 0x10000000 /* OR using multiple indices */ #define WHERE_TEMP_INDEX 0x20000000 /* Uses an ephemeral index */ +#define WHERE_DISTINCT 0x40000000 /* Correct order for DISTINCT */ /* ** Initialize a preallocated WhereClause structure. @@ -1397,6 +1398,162 @@ static int referencesOtherTables( return 0; } +/* +** This function searches the expression list passed as the second argument +** for an expression of type TK_COLUMN that refers to the same column and +** uses the same collation sequence as the iCol'th column of index pIdx. +** Argument iBase is the cursor number used for the table that pIdx refers +** to. +** +** If such an expression is found, its index in pList->a[] is returned. If +** no expression is found, -1 is returned. +*/ +static int findIndexCol( + Parse *pParse, /* Parse context */ + ExprList *pList, /* Expression list to search */ + int iBase, /* Cursor for table associated with pIdx */ + Index *pIdx, /* Index to match column of */ + int iCol /* Column of index to match */ +){ + int i; + const char *zColl = pIdx->azColl[iCol]; + + for(i=0; inExpr; i++){ + Expr *p = pList->a[i].pExpr; + if( p->op==TK_COLUMN + && p->iColumn==pIdx->aiColumn[iCol] + && p->iTable==iBase + ){ + CollSeq *pColl = sqlite3ExprCollSeq(pParse, p); + if( ALWAYS(pColl) && 0==sqlite3StrICmp(pColl->zName, zColl) ){ + return i; + } + } + } + + return -1; +} + +/* +** This routine determines if pIdx can be used to assist in processing a +** DISTINCT qualifier. In other words, it tests whether or not using this +** index for the outer loop guarantees that rows with equal values for +** all expressions in the pDistinct list are delivered grouped together. +** +** For example, the query +** +** SELECT DISTINCT a, b, c FROM tbl WHERE a = ? +** +** can benefit from any index on columns "b" and "c". +*/ +static int isDistinctIndex( + Parse *pParse, /* Parsing context */ + WhereClause *pWC, /* The WHERE clause */ + Index *pIdx, /* The index being considered */ + int base, /* Cursor number for the table pIdx is on */ + ExprList *pDistinct, /* The DISTINCT expressions */ + int nEqCol /* Number of index columns with == */ +){ + Bitmask mask = 0; /* Mask of unaccounted for pDistinct exprs */ + int i; /* Iterator variable */ + + if( pIdx->zName==0 || pDistinct==0 || pDistinct->nExpr>=BMS ) return 0; + testcase( pDistinct->nExpr==BMS-1 ); + + /* Loop through all the expressions in the distinct list. If any of them + ** are not simple column references, return early. Otherwise, test if the + ** WHERE clause contains a "col=X" clause. If it does, the expression + ** can be ignored. If it does not, and the column does not belong to the + ** same table as index pIdx, return early. Finally, if there is no + ** matching "col=X" expression and the column is on the same table as pIdx, + ** set the corresponding bit in variable mask. + */ + for(i=0; inExpr; i++){ + WhereTerm *pTerm; + Expr *p = pDistinct->a[i].pExpr; + if( p->op!=TK_COLUMN ) return 0; + pTerm = findTerm(pWC, p->iTable, p->iColumn, ~(Bitmask)0, WO_EQ, 0); + if( pTerm ){ + Expr *pX = pTerm->pExpr; + CollSeq *p1 = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight); + CollSeq *p2 = sqlite3ExprCollSeq(pParse, p); + if( p1==p2 ) continue; + } + if( p->iTable!=base ) return 0; + mask |= (((Bitmask)1) << i); + } + + for(i=nEqCol; mask && inColumn; i++){ + int iExpr = findIndexCol(pParse, pDistinct, base, pIdx, i); + if( iExpr<0 ) break; + mask &= ~(((Bitmask)1) << iExpr); + } + + return (mask==0); +} + + +/* +** Return true if the DISTINCT expression-list passed as the third argument +** is redundant. A DISTINCT list is redundant if the database contains a +** UNIQUE index that guarantees that the result of the query will be distinct +** anyway. +*/ +static int isDistinctRedundant( + Parse *pParse, + SrcList *pTabList, + WhereClause *pWC, + ExprList *pDistinct +){ + Table *pTab; + Index *pIdx; + int i; + int iBase; + + /* If there is more than one table or sub-select in the FROM clause of + ** this query, then it will not be possible to show that the DISTINCT + ** clause is redundant. */ + if( pTabList->nSrc!=1 ) return 0; + iBase = pTabList->a[0].iCursor; + pTab = pTabList->a[0].pTab; + + /* If any of the expressions is an IPK column on table iBase, then return + ** true. Note: The (p->iTable==iBase) part of this test may be false if the + ** current SELECT is a correlated sub-query. + */ + for(i=0; inExpr; i++){ + Expr *p = pDistinct->a[i].pExpr; + if( p->op==TK_COLUMN && p->iTable==iBase && p->iColumn<0 ) return 1; + } + + /* Loop through all indices on the table, checking each to see if it makes + ** the DISTINCT qualifier redundant. It does so if: + ** + ** 1. The index is itself UNIQUE, and + ** + ** 2. All of the columns in the index are either part of the pDistinct + ** list, or else the WHERE clause contains a term of the form "col=X", + ** where X is a constant value. The collation sequences of the + ** comparison and select-list expressions must match those of the index. + */ + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->onError==OE_None ) continue; + for(i=0; inColumn; i++){ + int iCol = pIdx->aiColumn[i]; + if( 0==findTerm(pWC, iBase, iCol, ~(Bitmask)0, WO_EQ, pIdx) + && 0>findIndexCol(pParse, pDistinct, iBase, pIdx, i) + ){ + break; + } + } + if( i==pIdx->nColumn ){ + /* This index implies that the DISTINCT qualifier is redundant. */ + return 1; + } + } + + return 0; +} /* ** This routine decides if pIdx can be used to satisfy the ORDER BY @@ -1433,7 +1590,10 @@ static int isSortingIndex( struct ExprList_item *pTerm; /* A term of the ORDER BY clause */ sqlite3 *db = pParse->db; - assert( pOrderBy!=0 ); + if( !pOrderBy ) return 0; + if( wsFlags & WHERE_COLUMN_IN ) return 0; + if( pIdx->bUnordered ) return 0; + nTerm = pOrderBy->nExpr; assert( nTerm>0 ); @@ -1746,6 +1906,10 @@ static void bestAutomaticIndex( WhereTerm *pWCEnd; /* End of pWC->a[] */ Table *pTable; /* Table tht might be indexed */ + if( pParse->nQueryLoop<=(double)1 ){ + /* There is no point in building an automatic index for a single scan */ + return; + } if( (pParse->db->flags & SQLITE_AutoIndex)==0 ){ /* Automatic indices are disabled at run-time */ return; @@ -1758,6 +1922,10 @@ static void bestAutomaticIndex( /* The NOT INDEXED clause appears in the SQL. */ return; } + if( pSrc->isCorrelated ){ + /* The source is a correlated sub-query. No point in indexing it. */ + return; + } assert( pParse->nQueryLoop >= (double)1 ); pTable = pSrc->pTab; @@ -2689,6 +2857,7 @@ static void bestBtreeIndex( Bitmask notReady, /* Mask of cursors not available for indexing */ Bitmask notValid, /* Cursors not available for any purpose */ ExprList *pOrderBy, /* The ORDER BY clause */ + ExprList *pDistinct, /* The select-list if query is DISTINCT */ WhereCost *pCost /* Lowest cost query plan */ ){ int iCur = pSrc->iCursor; /* The cursor of the table to be accessed */ @@ -2829,7 +2998,8 @@ static void bestBtreeIndex( int nInMul = 1; /* Number of distinct equalities to lookup */ int estBound = 100; /* Estimated reduction in search space */ int nBound = 0; /* Number of range constraints seen */ - int bSort = 0; /* True if external sort required */ + int bSort = !!pOrderBy; /* True if external sort required */ + int bDist = !!pDistinct; /* True if index cannot help with DISTINCT */ int bLookup = 0; /* True if not a covering index */ WhereTerm *pTerm; /* A single term of the WHERE clause */ #ifdef SQLITE_ENABLE_STAT2 @@ -2893,17 +3063,20 @@ static void bestBtreeIndex( ** naturally scan rows in the required order, set the appropriate flags ** in wsFlags. Otherwise, if there is an ORDER BY clause but the index ** will scan rows in a different order, set the bSort variable. */ - if( pOrderBy ){ - if( (wsFlags & WHERE_COLUMN_IN)==0 - && pProbe->bUnordered==0 - && isSortingIndex(pParse, pWC->pMaskSet, pProbe, iCur, pOrderBy, - nEq, wsFlags, &rev) - ){ - wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_ORDERBY; - wsFlags |= (rev ? WHERE_REVERSE : 0); - }else{ - bSort = 1; - } + if( isSortingIndex( + pParse, pWC->pMaskSet, pProbe, iCur, pOrderBy, nEq, wsFlags, &rev) + ){ + bSort = 0; + wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_ORDERBY; + wsFlags |= (rev ? WHERE_REVERSE : 0); + } + + /* If there is a DISTINCT qualifier and this index will scan rows in + ** order of the DISTINCT expressions, clear bDist and set the appropriate + ** flags in wsFlags. */ + if( isDistinctIndex(pParse, pWC, pProbe, iCur, pDistinct, nEq) ){ + bDist = 0; + wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_DISTINCT; } /* If currently calculating the cost of using an index (not the IPK @@ -2938,12 +3111,13 @@ static void bestBtreeIndex( } #ifdef SQLITE_ENABLE_STAT2 - /* If the constraint is of the form x=VALUE and histogram + /* If the constraint is of the form x=VALUE or x IN (E1,E2,...) + ** and we do not think that values of x are unique and if histogram ** data is available for column x, then it might be possible ** to get a better estimate on the number of rows based on ** VALUE and how common that value is according to the histogram. */ - if( nRow>(double)1 && nEq==1 && pFirstTerm!=0 ){ + if( nRow>(double)1 && nEq==1 && pFirstTerm!=0 && aiRowEst[1]>1 ){ if( pFirstTerm->eOperator & (WO_EQ|WO_ISNULL) ){ testcase( pFirstTerm->eOperator==WO_EQ ); testcase( pFirstTerm->eOperator==WO_ISNULL ); @@ -3020,6 +3194,9 @@ static void bestBtreeIndex( if( bSort ){ cost += nRow*estLog(nRow)*3; } + if( bDist ){ + cost += nRow*estLog(nRow)*3; + } /**** Cost of using this index has now been computed ****/ @@ -3165,7 +3342,7 @@ static void bestIndex( }else #endif { - bestBtreeIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost); + bestBtreeIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, 0, pCost); } } @@ -4127,7 +4304,7 @@ static Bitmask codeOneLoopStart( if( pOrTerm->leftCursor==iCur || pOrTerm->eOperator==WO_AND ){ WhereInfo *pSubWInfo; /* Info for single OR-term scan */ /* Loop through table entries that match term pOrTerm. */ - pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrTerm->pExpr, 0, + pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrTerm->pExpr, 0, 0, WHERE_OMIT_OPEN | WHERE_OMIT_CLOSE | WHERE_FORCE_TABLE | WHERE_ONETABLE_ONLY); if( pSubWInfo ){ @@ -4368,6 +4545,7 @@ WhereInfo *sqlite3WhereBegin( SrcList *pTabList, /* A list of all tables to be scanned */ Expr *pWhere, /* The WHERE clause */ ExprList **ppOrderBy, /* An ORDER BY clause, or NULL */ + ExprList *pDistinct, /* The select-list for DISTINCT queries - or NULL */ u16 wctrlFlags /* One of the WHERE_* flags defined in sqliteInt.h */ ){ int i; /* Loop counter */ @@ -4428,6 +4606,10 @@ WhereInfo *sqlite3WhereBegin( pWInfo->savedNQueryLoop = pParse->nQueryLoop; pMaskSet = (WhereMaskSet*)&pWC[1]; + /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via + ** sqlite3_test_ctrl(SQLITE_TESTCTRL_OPTIMIZATIONS,...) */ + if( db->flags & SQLITE_DistinctOpt ) pDistinct = 0; + /* Split the WHERE clause into separate subexpressions where each ** subexpression is separated by an AND operator. */ @@ -4495,6 +4677,15 @@ WhereInfo *sqlite3WhereBegin( goto whereBeginError; } + /* Check if the DISTINCT qualifier, if there is one, is redundant. + ** If it is, then set pDistinct to NULL and WhereInfo.eDistinct to + ** WHERE_DISTINCT_UNIQUE to tell the caller to ignore the DISTINCT. + */ + if( pDistinct && isDistinctRedundant(pParse, pTabList, pWC, pDistinct) ){ + pDistinct = 0; + pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; + } + /* Chose the best index to use for each table in the FROM clause. ** ** This loop fills in the following fields: @@ -4578,6 +4769,7 @@ WhereInfo *sqlite3WhereBegin( int doNotReorder; /* True if this table should not be reordered */ WhereCost sCost; /* Cost information from best[Virtual]Index() */ ExprList *pOrderBy; /* ORDER BY clause for index to optimize */ + ExprList *pDist; /* DISTINCT clause for index to optimize */ doNotReorder = (pTabItem->jointype & (JT_LEFT|JT_CROSS))!=0; if( j!=iFrom && doNotReorder ) break; @@ -4588,6 +4780,7 @@ WhereInfo *sqlite3WhereBegin( } mask = (isOptimal ? m : notReady); pOrderBy = ((i==0 && ppOrderBy )?*ppOrderBy:0); + pDist = (i==0 ? pDistinct : 0); if( pTabItem->pIndex==0 ) nUnconstrained++; WHERETRACE(("=== trying table %d with isOptimal=%d ===\n", @@ -4602,7 +4795,7 @@ WhereInfo *sqlite3WhereBegin( #endif { bestBtreeIndex(pParse, pWC, pTabItem, mask, notReady, pOrderBy, - &sCost); + pDist, &sCost); } assert( isOptimal || (sCost.used¬Ready)==0 ); @@ -4663,6 +4856,10 @@ WhereInfo *sqlite3WhereBegin( if( (bestPlan.plan.wsFlags & WHERE_ORDERBY)!=0 ){ *ppOrderBy = 0; } + if( (bestPlan.plan.wsFlags & WHERE_DISTINCT)!=0 ){ + assert( pWInfo->eDistinct==0 ); + pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; + } andFlags &= bestPlan.plan.wsFlags; pLevel->plan = bestPlan.plan; testcase( bestPlan.plan.wsFlags & WHERE_INDEXED ); diff --git a/test/all.test b/test/all.test index 9671246681..54302d02e9 100644 --- a/test/all.test +++ b/test/all.test @@ -38,6 +38,7 @@ run_test_suite pcache10 run_test_suite pcache50 run_test_suite pcache90 run_test_suite pcache100 +run_test_suite prepare if {$::tcl_platform(platform)=="unix"} { ifcapable !default_autovacuum { diff --git a/test/alter2.test b/test/alter2.test index 712960c0ea..66b1950945 100644 --- a/test/alter2.test +++ b/test/alter2.test @@ -137,6 +137,7 @@ do_test alter2-1.8 { do_test alter2-1.9 { # ALTER TABLE abc ADD COLUMN d; alter_table abc {CREATE TABLE abc(a, b, c, d);} + if {[permutation] == "prepare"} { db cache flush } execsql { SELECT * FROM abc; } execsql { UPDATE abc SET d = 11 WHERE c IS NULL AND a<4; diff --git a/test/autoindex1.test b/test/autoindex1.test index bc628dc256..6c8d54d125 100644 --- a/test/autoindex1.test +++ b/test/autoindex1.test @@ -248,4 +248,14 @@ do_execsql_test autoindex1-600 { 0 1 1 {SEARCH SUBQUERY 1 AS y USING AUTOMATIC COVERING INDEX (sheep_no=?) (~8 rows)} } + +do_execsql_test autoindex1-700 { + CREATE TABLE t5(a, b, c); + EXPLAIN QUERY PLAN SELECT a FROM t5 WHERE b=10 ORDER BY c; +} { + 0 0 0 {SCAN TABLE t5 (~100000 rows)} + 0 0 0 {USE TEMP B-TREE FOR ORDER BY} +} + + finish_test diff --git a/test/collate5.test b/test/collate5.test index 2e4b89d5ca..e5bea7a5a5 100644 --- a/test/collate5.test +++ b/test/collate5.test @@ -57,17 +57,17 @@ do_test collate5-1.1 { execsql { SELECT DISTINCT a FROM collate5t1; } -} {A B N} +} {a b n} do_test collate5-1.2 { execsql { SELECT DISTINCT b FROM collate5t1; } -} {{} Apple apple banana} +} {apple Apple banana {}} do_test collate5-1.3 { execsql { SELECT DISTINCT a, b FROM collate5t1; } -} {A Apple a apple B banana N {}} +} {a apple A Apple b banana n {}} # Ticket #3376 # diff --git a/test/distinct.test b/test/distinct.test new file mode 100644 index 0000000000..87456de876 --- /dev/null +++ b/test/distinct.test @@ -0,0 +1,166 @@ +# 2011 July 1 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this script is the DISTINCT modifier. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +set testprefix distinct + + +proc is_distinct_noop {sql} { + set sql1 $sql + set sql2 [string map {DISTINCT ""} $sql] + + set program1 [list] + set program2 [list] + db eval "EXPLAIN $sql1" { + if {$opcode != "Noop"} { lappend program1 $opcode } + } + db eval "EXPLAIN $sql2" { + if {$opcode != "Noop"} { lappend program2 $opcode } + } + + return [expr {$program1==$program2}] +} + +proc do_distinct_noop_test {tn sql} { + uplevel [list do_test $tn [list is_distinct_noop $sql] 1] +} +proc do_distinct_not_noop_test {tn sql} { + uplevel [list do_test $tn [list is_distinct_noop $sql] 0] +} + +proc do_temptables_test {tn sql temptables} { + uplevel [list do_test $tn [subst -novar { + set ret "" + db eval "EXPLAIN [set sql]" { + if {$opcode == "OpenEphemeral"} { + if {$p5 != "10" && $p5!="00"} { error "p5 = $p5" } + if {$p5 == "10"} { + lappend ret hash + } else { + lappend ret btree + } + } + } + set ret + }] $temptables] +} + + +#------------------------------------------------------------------------- +# The following tests - distinct-1.* - check that the planner correctly +# detects cases where a UNIQUE index means that a DISTINCT clause is +# redundant. Currently the planner only detects such cases when there +# is a single table in the FROM clause. +# +do_execsql_test 1.0 { + CREATE TABLE t1(a, b, c, d); + CREATE UNIQUE INDEX i1 ON t1(b, c); + CREATE UNIQUE INDEX i2 ON t1(d COLLATE nocase); + + CREATE TABLE t2(x INTEGER PRIMARY KEY, y); + + CREATE TABLE t3(c1 PRIMARY KEY, c2); + CREATE INDEX i3 ON t3(c2); +} +foreach {tn noop sql} { + + 1 1 "SELECT DISTINCT b, c FROM t1" + 2 1 "SELECT DISTINCT c FROM t1 WHERE b = ?" + 3 1 "SELECT DISTINCT rowid FROM t1" + 4 1 "SELECT DISTINCT rowid, a FROM t1" + 5 1 "SELECT DISTINCT x FROM t2" + 6 1 "SELECT DISTINCT * FROM t2" + 7 1 "SELECT DISTINCT * FROM (SELECT * FROM t2)" + + 8 1 "SELECT DISTINCT * FROM t1" + + 8 0 "SELECT DISTINCT a, b FROM t1" + + 9 0 "SELECT DISTINCT c FROM t1 WHERE b IN (1,2)" + 10 0 "SELECT DISTINCT c FROM t1" + 11 0 "SELECT DISTINCT b FROM t1" + + 12 0 "SELECT DISTINCT a, d FROM t1" + 13 0 "SELECT DISTINCT a, b, c COLLATE nocase FROM t1" + 14 1 "SELECT DISTINCT a, d COLLATE nocase FROM t1" + 15 0 "SELECT DISTINCT a, d COLLATE binary FROM t1" + 16 1 "SELECT DISTINCT a, b, c COLLATE binary FROM t1" + + 16 0 "SELECT DISTINCT t1.rowid FROM t1, t2" + 17 0 { /* Technically, it would be possible to detect that DISTINCT + ** is a no-op in cases like the following. But SQLite does not + ** do so. */ + SELECT DISTINCT t1.rowid FROM t1, t2 WHERE t1.rowid=t2.rowid } + + 18 1 "SELECT DISTINCT c1, c2 FROM t3" + 19 1 "SELECT DISTINCT c1 FROM t3" + 20 1 "SELECT DISTINCT * FROM t3" + 21 0 "SELECT DISTINCT c2 FROM t3" + + 22 0 "SELECT DISTINCT * FROM (SELECT 1, 2, 3 UNION SELECT 4, 5, 6)" + 23 1 "SELECT DISTINCT rowid FROM (SELECT 1, 2, 3 UNION SELECT 4, 5, 6)" + + 24 0 "SELECT DISTINCT rowid/2 FROM t1" + 25 1 "SELECT DISTINCT rowid/2, rowid FROM t1" + 26 1 "SELECT DISTINCT rowid/2, b FROM t1 WHERE c = ?" +} { + if {$noop} { + do_distinct_noop_test 1.$tn $sql + } else { + do_distinct_not_noop_test 1.$tn $sql + } +} + +#------------------------------------------------------------------------- +# The following tests - distinct-2.* - test cases where an index is +# used to deliver results in order of the DISTINCT expressions. +# +drop_all_tables +do_execsql_test 2.0 { + CREATE TABLE t1(a, b, c); + + CREATE INDEX i1 ON t1(a, b); + CREATE INDEX i2 ON t1(b COLLATE nocase, c COLLATE nocase); + + INSERT INTO t1 VALUES('a', 'b', 'c'); + INSERT INTO t1 VALUES('A', 'B', 'C'); + INSERT INTO t1 VALUES('a', 'b', 'c'); + INSERT INTO t1 VALUES('A', 'B', 'C'); +} + +foreach {tn sql temptables res} { + 1 "a, b FROM t1" {} {A B a b} + 2 "b, a FROM t1" {} {B A b a} + 3 "a, b, c FROM t1" {hash} {a b c A B C} + 4 "a, b, c FROM t1 ORDER BY a, b, c" {btree} {A B C a b c} + 5 "b FROM t1 WHERE a = 'a'" {} {b} + 6 "b FROM t1" {hash} {b B} + 7 "a FROM t1" {} {A a} + 8 "b COLLATE nocase FROM t1" {} {b} + 9 "b COLLATE nocase FROM t1 ORDER BY b COLLATE nocase" {} {B} +} { + do_execsql_test 2.$tn.1 "SELECT DISTINCT $sql" $res + do_temptables_test 2.$tn.2 "SELECT DISTINCT $sql" $temptables +} + +do_execsql_test 2.A { + SELECT (SELECT DISTINCT o.a FROM t1 AS i) FROM t1 AS o; +} {a A a A} + + + + +finish_test diff --git a/test/e_select.test b/test/e_select.test index 6194872fa5..e0f5f0fb91 100644 --- a/test/e_select.test +++ b/test/e_select.test @@ -1238,8 +1238,8 @@ do_select_tests e_select-5 { 3.1 "SELECT x FROM h2" {One Two Three Four one two three four} 3.2 "SELECT x FROM h1, h2 ON (x=b)" {One one Four four} - 4.1 "SELECT DISTINCT x FROM h2" {four one three two} - 4.2 "SELECT DISTINCT x FROM h1, h2 ON (x=b)" {four one} + 4.1 "SELECT DISTINCT x FROM h2" {One Two Three Four} + 4.2 "SELECT DISTINCT x FROM h1, h2 ON (x=b)" {One Four} } # EVIDENCE-OF: R-02054-15343 For the purposes of detecting duplicate @@ -1253,11 +1253,11 @@ do_select_tests e_select-5.5 { # sequence to compare text values with apply. # do_select_tests e_select-5.6 { - 1 "SELECT DISTINCT b FROM h1" {I IV four i iv one} - 2 "SELECT DISTINCT b COLLATE nocase FROM h1" {four i iv one} - 3 "SELECT DISTINCT x FROM h2" {four one three two} + 1 "SELECT DISTINCT b FROM h1" {one I i four IV iv} + 2 "SELECT DISTINCT b COLLATE nocase FROM h1" {one I four IV} + 3 "SELECT DISTINCT x FROM h2" {One Two Three Four} 4 "SELECT DISTINCT x COLLATE binary FROM h2" { - Four One Three Two four one three two + One Two Three Four one two three four } } diff --git a/test/exists.test b/test/exists.test index c7c14a2402..fb73797d29 100644 --- a/test/exists.test +++ b/test/exists.test @@ -159,7 +159,7 @@ foreach jm {rollback wal} { sql1 { DROP INDEX IF EXISTS i2 } sql2 { CREATE INDEX aux.i2 ON t2(x) } sql1 { DROP INDEX IF EXISTS i2 } - sql2 { SELECT name FROM aux.sqlite_master WHERE type = 'index' } + sql2 { SELECT * FROM aux.sqlite_master WHERE type = 'index' } } {} # VIEW objects. diff --git a/test/fts3auto.test b/test/fts3auto.test index 484fd6ace2..1c58a17204 100644 --- a/test/fts3auto.test +++ b/test/fts3auto.test @@ -107,15 +107,17 @@ proc do_fts3query_test {tn args} { " $matchinfo_asc } -# fts3_make_deferrable TABLE TOKEN +# fts3_make_deferrable TABLE TOKEN ?NROW? # -proc fts3_make_deferrable {tbl token} { +proc fts3_make_deferrable {tbl token {nRow 0}} { set stmt [sqlite3_prepare db "SELECT * FROM $tbl" -1 dummy] set name [sqlite3_column_name $stmt 0] sqlite3_finalize $stmt - set nRow [db one "SELECT count(*) FROM $tbl"] + if {$nRow==0} { + set nRow [db one "SELECT count(*) FROM $tbl"] + } set pgsz [db one "PRAGMA page_size"] execsql BEGIN for {set i 0} {$i < ($nRow * $pgsz * 1.2)/100} {incr i} { @@ -653,6 +655,50 @@ foreach {tn pending create} { catchsql { COMMIT } } +foreach {tn create} { + 1 "fts4(x)" + 2 "fts4(x, order=DESC)" +} { + execsql [subst { + DROP TABLE IF EXISTS t1; + CREATE VIRTUAL TABLE t1 USING $create; + }] + + foreach {x} { + "F E N O T K X V A X I E X A P G Q V H U" + "R V A E T C V Q N I E L O N U G J K L U" + "U Y I G W M V F J L X I D C H F P J Q B" + "S G D Z X R P G S S Y B K A S G A I L L" + "L S I C H T Z S R Q P R N K J X L F M J" + "C C C D P X B Z C M A D A C X S B T X V" + "W Y J M D R G V R K B X S A W R I T N C" + "P K L W T M S P O Y Y V V O E H Q A I R" + "C D Y I C Z F H J C O Y A Q F L S B D K" + "P G S C Y C Y V I M B D S Z D D Y W I E" + "Z K Z U E E S F Y X T U A L W O U J C Q" + "P A T Z S W L P L Q V Y Y I P W U X S S" + "I U I H U O F Z F R H R F T N D X A G M" + "N A B M S H K X S O Y D T X S B R Y H Z" + "L U D A S K I L S V Z J P U B E B Y H M" + } { + execsql { INSERT INTO t1 VALUES($x) } + } + + # Add extra documents to the database such that token "B" will be considered + # deferrable if considering the other tokens means that 2 or fewer documents + # will be loaded into memory. + # + fts3_make_deferrable t1 B 2 + + # B is not deferred in either of the first two tests below, since filtering + # on "M" or "D" returns 10 documents or so. But filtering on "M * D" only + # returns 2, so B is deferred in this case. + # + do_fts3query_test 7.$tn.1 t1 {"M B"} + do_fts3query_test 7.$tn.2 t1 {"B D"} + do_fts3query_test 7.$tn.3 -deferred B t1 {"M B D"} +} + set sqlite_fts3_enable_parentheses $sfep finish_test diff --git a/test/func3.test b/test/func3.test index e58d730f3a..d5a462fe33 100644 --- a/test/func3.test +++ b/test/func3.test @@ -17,24 +17,26 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -do_test func3-1.1 { - set destroyed 0 - proc destroy {} { set ::destroyed 1 } - sqlite3_create_function_v2 db f2 -1 any -func f2 -destroy destroy - set destroyed -} 0 -do_test func3-1.2 { - sqlite3_create_function_v2 db f2 -1 utf8 -func f2 - set destroyed -} 0 -do_test func3-1.3 { - sqlite3_create_function_v2 db f2 -1 utf16le -func f2 - set destroyed -} 0 -do_test func3-1.4 { - sqlite3_create_function_v2 db f2 -1 utf16be -func f2 - set destroyed -} 1 +ifcapable utf16 { + do_test func3-1.1 { + set destroyed 0 + proc destroy {} { set ::destroyed 1 } + sqlite3_create_function_v2 db f2 -1 any -func f2 -destroy destroy + set destroyed + } 0 + do_test func3-1.2 { + sqlite3_create_function_v2 db f2 -1 utf8 -func f2 + set destroyed + } 0 + do_test func3-1.3 { + sqlite3_create_function_v2 db f2 -1 utf16le -func f2 + set destroyed + } 0 + do_test func3-1.4 { + sqlite3_create_function_v2 db f2 -1 utf16be -func f2 + set destroyed + } 1 +} do_test func3-2.1 { set destroyed 0 diff --git a/test/fuzzer1.test b/test/fuzzer1.test index d0575d2d00..6c23211859 100644 --- a/test/fuzzer1.test +++ b/test/fuzzer1.test @@ -1376,7 +1376,7 @@ do_test fuzzer1-2.3 { AND f2.distance<=200 AND streetname.n>=f2.word AND streetname.n<=(f2.word || x'F7BFBFBF') } -} {steelewood tallia tallu talwyn taymouth thelema trailer {tyler finley}} +} {{tyler finley} trailer taymouth steelewood tallia tallu talwyn thelema} finish_test diff --git a/test/incrblob_err.test b/test/incrblob_err.test index b4c258772a..35d37fee92 100644 --- a/test/incrblob_err.test +++ b/test/incrblob_err.test @@ -35,6 +35,7 @@ do_malloc_test 1 -tclprep { } } -tclbody { set ::blob [db incrblob blobs v 1] + fconfigure $::blob -translation binary set rc [catch {puts -nonewline $::blob $::data}] if {$rc} { error "out of memory" } } @@ -71,8 +72,7 @@ do_malloc_test 3 -tclprep { if {$rc} { error "out of memory" } -} - +} do_ioerr_test incrblob_err-4 -cksum 1 -sqlprep { CREATE TABLE blobs(k, v BLOB); @@ -87,6 +87,7 @@ do_ioerr_test incrblob_err-5 -cksum 1 -sqlprep { INSERT INTO blobs VALUES(1, zeroblob(length(CAST($::data AS BLOB)))); } -tclbody { set ::blob [db incrblob blobs v 1] + fconfigure $::blob -translation binary puts -nonewline $::blob $::data close $::blob } @@ -96,6 +97,7 @@ do_ioerr_test incrblob_err-6 -cksum 1 -sqlprep { INSERT INTO blobs VALUES(1, $::data || $::data || $::data); } -tclbody { set ::blob [db incrblob blobs v 1] + fconfigure $::blob -translation binary seek $::blob -20 end puts -nonewline $::blob "12345678900987654321" close $::blob diff --git a/test/insert4.test b/test/insert4.test index 44b428a1c7..cb02b9dccb 100644 --- a/test/insert4.test +++ b/test/insert4.test @@ -112,7 +112,7 @@ do_test insert4-2.4.1 { INSERT INTO t3 SELECT DISTINCT * FROM t2; SELECT * FROM t3; } -} {1 9 9 1} +} {9 1 1 9} xferopt_test insert4-2.4.2 0 do_test insert4-2.4.3 { catchsql { diff --git a/test/like.test b/test/like.test index bd9a6c39c1..45d5a97c5b 100644 --- a/test/like.test +++ b/test/like.test @@ -70,12 +70,15 @@ do_test like-1.4 { } } {ABC abc} do_test like-1.5.1 { + # Use sqlite3_exec() to verify fix for ticket [25ee81271091] 2011-06-26 + sqlite3_exec db {PRAGMA case_sensitive_like=on} +} {0 {}} +do_test like-1.5.2 { execsql { - PRAGMA case_sensitive_like=on; SELECT x FROM t1 WHERE x LIKE 'abc' ORDER BY 1; } } {abc} -do_test like-1.5.2 { +do_test like-1.5.3 { execsql { PRAGMA case_sensitive_like; -- no argument; does not change setting SELECT x FROM t1 WHERE x LIKE 'abc' ORDER BY 1; diff --git a/test/misc5.test b/test/misc5.test index 34b2284bc1..b3832f18ae 100644 --- a/test/misc5.test +++ b/test/misc5.test @@ -505,7 +505,7 @@ ifcapable subquery { ) ORDER BY LOWER(artist) ASC; } - } {one} + } {two} } # Ticket #1370. Do not overwrite small files (less than 1024 bytes) diff --git a/test/multiplex.test b/test/multiplex.test index 9278e84274..28ce1c77e7 100644 --- a/test/multiplex.test +++ b/test/multiplex.test @@ -49,6 +49,7 @@ proc multiplex_set {db name chunk_size max_chunks} { # and files with the chunk extension. proc multiplex_delete {name} { global g_max_chunks + forcedelete $name for {set i 0} {$i<$g_max_chunks} {incr i} { forcedelete [multiplex_name $name $i] forcedelete [multiplex_name $name-journal $i] @@ -78,10 +79,9 @@ do_test multiplex-1.8 { sqlite3_multiplex_shutdown } {SQLITE_OK} do_test multiplex-1.9.1 { sqlite3_multiplex_initialize "" 1 } {SQLITE_OK} do_test multiplex-1.9.2 { sqlite3 db test.db } {} do_test multiplex-1.9.3 { multiplex_set db main 32768 16 } {SQLITE_OK} -do_test multiplex-1.9.4 { multiplex_set db main 32768 -1 } {SQLITE_MISUSE} -do_test multiplex-1.9.5 { multiplex_set db main -1 16 } {SQLITE_MISUSE} +do_test multiplex-1.9.4 { multiplex_set db main 32768 -1 } {SQLITE_OK} do_test multiplex-1.9.6 { multiplex_set db main 31 16 } {SQLITE_OK} -do_test multiplex-1.9.7 { multiplex_set db main 32768 100 } {SQLITE_MISUSE} +do_test multiplex-1.9.7 { multiplex_set db main 32768 100 } {SQLITE_OK} do_test multiplex-1.9.8 { multiplex_set db main 1073741824 1 } {SQLITE_OK} do_test multiplex-1.9.9 { db close } {} do_test multiplex-1.9.10 { sqlite3_multiplex_shutdown } {SQLITE_OK} @@ -89,10 +89,9 @@ do_test multiplex-1.9.10 { sqlite3_multiplex_shutdown } {SQLITE_OK} do_test multiplex-1.10.1 { sqlite3_multiplex_initialize "" 1 } {SQLITE_OK} do_test multiplex-1.10.2 { sqlite3 db test.db } {} do_test multiplex-1.10.3 { lindex [ catchsql { SELECT multiplex_control(2, 32768); } ] 0 } {0} -do_test multiplex-1.10.4 { lindex [ catchsql { SELECT multiplex_control(3, -1); } ] 0 } {1} -do_test multiplex-1.10.5 { lindex [ catchsql { SELECT multiplex_control(2, -1); } ] 0 } {1} +do_test multiplex-1.10.4 { lindex [ catchsql { SELECT multiplex_control(3, -1); } ] 0 } {0} do_test multiplex-1.10.6 { lindex [ catchsql { SELECT multiplex_control(2, 31); } ] 0 } {0} -do_test multiplex-1.10.7 { lindex [ catchsql { SELECT multiplex_control(3, 100); } ] 0 } {1} +do_test multiplex-1.10.7 { lindex [ catchsql { SELECT multiplex_control(3, 100); } ] 0 } {0} do_test multiplex-1.10.8 { lindex [ catchsql { SELECT multiplex_control(2, 1073741824); } ] 0 } {0} do_test multiplex-1.10.9 { db close } {} do_test multiplex-1.10.10 { sqlite3_multiplex_shutdown } {SQLITE_OK} @@ -146,8 +145,9 @@ do_test multiplex-1.13.7 { sqlite3_multiplex_shutdown } sqlite3_multiplex_initialize "" 1 multiplex_set db main 32768 16 +file delete -force test.x do_test multiplex-2.1.2 { - sqlite3 db test.db + sqlite3 db test.x execsql { PRAGMA page_size=1024; PRAGMA auto_vacuum=OFF; @@ -159,7 +159,7 @@ do_test multiplex-2.1.2 { INSERT INTO t1 VALUES(2, randomblob(1100)); } } {} -do_test multiplex-2.1.3 { file size [multiplex_name test.db 0] } {4096} +do_test multiplex-2.1.3 { file size [multiplex_name test.x 0] } {4096} do_test multiplex-2.1.4 { execsql { INSERT INTO t1 VALUES(3, randomblob(1100)) } } {} @@ -167,10 +167,10 @@ do_test multiplex-2.1.4 { do_test multiplex-2.2.1 { execsql { INSERT INTO t1 VALUES(3, randomblob(1100)) } } {} -do_test multiplex-2.2.3 { file size [multiplex_name test.db 0] } {6144} +do_test multiplex-2.2.3 { file size [multiplex_name test.x 0] } {6144} do_test multiplex-2.3.1 { - sqlite3 db2 test2.db + sqlite3 db2 test2.x db2 close } {} @@ -181,7 +181,7 @@ do_test multiplex-2.4.1 { do_test multiplex-2.4.2 { execsql { INSERT INTO t1 VALUES(3, randomblob(1100)) } } {} -do_test multiplex-2.4.4 { file size [multiplex_name test.db 0] } {7168} +do_test multiplex-2.4.4 { file size [multiplex_name test.x 0] } {7168} do_test multiplex-2.4.99 { db close sqlite3_multiplex_shutdown @@ -189,9 +189,9 @@ do_test multiplex-2.4.99 { do_test multiplex-2.5.1 { - multiplex_delete test.db + multiplex_delete test.x sqlite3_multiplex_initialize "" 1 - sqlite3 db test.db + sqlite3 db test.x multiplex_set db main 4096 16 } {SQLITE_OK} @@ -236,8 +236,8 @@ do_test multiplex-2.5.8 { db eval {SELECT a,length(b) FROM t1 WHERE a=4} } {4 4000} -do_test multiplex-2.5.9 { file size [multiplex_name test.db 0] } [list $g_chunk_size] -do_test multiplex-2.5.10 { file size [multiplex_name test.db 1] } [list $g_chunk_size] +do_test multiplex-2.5.9 { file size [multiplex_name test.x 0] } [list $g_chunk_size] +do_test multiplex-2.5.10 { file size [multiplex_name test.x 1] } [list $g_chunk_size] do_test multiplex-2.5.99 { db close @@ -523,50 +523,6 @@ do_faultsim_test multiplex-5.5 -prep { multiplex_set db main 32768 16 } -# test that mismatch filesize is detected -# -# Do not run this test if $::G(perm:presql) is set. If it is set, then the -# expected IO error will occur within the Tcl [sqlite3] wrapper, not within -# the first SQL statement executed below. This breaks the test case. -# -if {0==[info exists ::G(perm:presql)] || $::G(perm:presql) == ""} { - set all_journal_modes {delete persist truncate memory off} - foreach jmode $all_journal_modes { - do_test multiplex-5.6.1.$jmode { - sqlite3_multiplex_shutdown - multiplex_delete test.db - sqlite3 db test.db - db eval { - PRAGMA page_size = 1024; - PRAGMA auto_vacuum = off; - } - db eval "PRAGMA journal_mode = $jmode;" - } $jmode - do_test multiplex-5.6.2.$jmode { - execsql { - CREATE TABLE t1(a, b); - INSERT INTO t1 VALUES(1, randomblob(15000)); - INSERT INTO t1 VALUES(2, randomblob(15000)); - INSERT INTO t1 VALUES(3, randomblob(15000)); - INSERT INTO t1 VALUES(4, randomblob(15000)); - INSERT INTO t1 VALUES(5, randomblob(15000)); - } - db close - sqlite3_multiplex_initialize "" 1 - sqlite3 db test.db - multiplex_set db main 4096 16 - } {SQLITE_OK} - do_test multiplex-5.6.3.$jmode { - catchsql { - INSERT INTO t1 VALUES(6, randomblob(15000)); - } - } {1 {disk I/O error}} - do_test multiplex-5.6.4.$jmode { - db close - } {} - } -} - #------------------------------------------------------------------------- # Test that you can vacuum a multiplex'ed DB. @@ -575,8 +531,9 @@ ifcapable vacuum { sqlite3_multiplex_shutdown do_test multiplex-6.0.0 { multiplex_delete test.db + multiplex_delete test.x sqlite3_multiplex_initialize "" 1 - sqlite3 db test.db + sqlite3 db test.x multiplex_set db main 4096 16 } {SQLITE_OK} @@ -592,8 +549,8 @@ do_test multiplex-6.1.0 { INSERT INTO t1 VALUES(2, randomblob($g_chunk_size)); } } {} -do_test multiplex-6.2.1 { file size [multiplex_name test.db 0] } [list $g_chunk_size] -do_test multiplex-6.2.2 { file size [multiplex_name test.db 1] } [list $g_chunk_size] +do_test multiplex-6.2.1 { file size [multiplex_name test.x 0] } [list $g_chunk_size] +do_test multiplex-6.2.2 { file size [multiplex_name test.x 1] } [list $g_chunk_size] do_test multiplex-6.3.0 { execsql { VACUUM } @@ -601,7 +558,7 @@ do_test multiplex-6.3.0 { do_test multiplex-6.99 { db close - multiplex_delete test.db + multiplex_delete test.x sqlite3_multiplex_shutdown } {SQLITE_OK} diff --git a/test/permutations.test b/test/permutations.test index 354b4084a8..ad6199ac17 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -783,6 +783,15 @@ test_suite "no_optimization" -description { optimization_control $::dbhandle all 0 } +test_suite "prepare" -description { + Run tests with the db connection using sqlite3_prepare() instead of _v2(). +} -dbconfig { + db_use_legacy_prepare $::dbhandle 1 + #$::dbhandle cache size 0 +} -files [ + test_set $allquicktests -exclude *malloc* *ioerr* *fault* +] + # End of tests ############################################################################# diff --git a/test/releasetest.tcl b/test/releasetest.tcl index 84542bb8a5..7725630187 100644 --- a/test/releasetest.tcl +++ b/test/releasetest.tcl @@ -155,10 +155,10 @@ array set ::Configs { array set ::Platforms { Linux-x86_64 { + "Debug-One" "checksymbols test" "Secure-Delete" test "Unlock-Notify" "QUICKTEST_INCLUDE=notify2.test test" "Update-Delete-Limit" test - "Debug-One" test "Extra-Robustness" test "Device-Two" test "Ftrapv" test @@ -177,6 +177,7 @@ array set ::Platforms { } } + # End of configuration section. ######################################################################### ######################################################################### diff --git a/test/selectB.test b/test/selectB.test index 3fdf85c0f9..b9d979acb7 100644 --- a/test/selectB.test +++ b/test/selectB.test @@ -355,7 +355,7 @@ for {set ii 3} {$ii <= 4} {incr ii} { SELECT DISTINCT (a/10) FROM t1 UNION ALL SELECT DISTINCT(d%2) FROM t2 ) } - } {0 1 0 1} + } {0 1 1 0} do_test selectB-$ii.20 { execsql { diff --git a/test/sqllimits1.test b/test/sqllimits1.test index bf4a262678..8f2521353f 100644 --- a/test/sqllimits1.test +++ b/test/sqllimits1.test @@ -319,18 +319,22 @@ do_test sqllimits1-5.14.4 { catch {sqlite3_bind_text $::STMT 1 $::str1 -1} res set res } {SQLITE_TOOBIG} -do_test sqllimits1-5.14.5 { - catch {sqlite3_bind_text16 $::STMT 1 $::str1 -1} res - set res -} {SQLITE_TOOBIG} +ifcapable utf16 { + do_test sqllimits1-5.14.5 { + catch {sqlite3_bind_text16 $::STMT 1 $::str1 -1} res + set res + } {SQLITE_TOOBIG} +} do_test sqllimits1-5.14.6 { catch {sqlite3_bind_text $::STMT 1 $::str1 $np1} res set res } {SQLITE_TOOBIG} -do_test sqllimits1-5.14.7 { - catch {sqlite3_bind_text16 $::STMT 1 $::str1 $np1} res - set res -} {SQLITE_TOOBIG} +ifcapable utf16 { + do_test sqllimits1-5.14.7 { + catch {sqlite3_bind_text16 $::STMT 1 $::str1 $np1} res + set res + } {SQLITE_TOOBIG} +} do_test sqllimits1-5.14.8 { set n [expr {$np1-1}] catch {sqlite3_bind_text $::STMT 1 $::str1 $n} res diff --git a/test/temptable.test b/test/temptable.test index ad85f7e9f6..d82ea36444 100644 --- a/test/temptable.test +++ b/test/temptable.test @@ -292,6 +292,7 @@ do_test temptable-5.1 { execsql { CREATE TEMP TABLE mask(a,b,c) } db2 + if {[permutation]=="prepare"} { db2 cache flush } execsql { CREATE INDEX mask ON t2(x); SELECT * FROM t2; diff --git a/test/tkt-54844eea3f.test b/test/tkt-54844eea3f.test new file mode 100644 index 0000000000..7b04f6935d --- /dev/null +++ b/test/tkt-54844eea3f.test @@ -0,0 +1,67 @@ +# 2011 July 8 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing that bug [54844eea3f] has been fixed. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +set ::testprefix tkt-54844eea3f + +do_test 1.0 { + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY); + INSERT INTO t1 VALUES(1); + INSERT INTO t1 VALUES(4); + + CREATE TABLE t2(b INTEGER PRIMARY KEY); + INSERT INTO t2 VALUES(1); + INSERT INTO t2 VALUES(2); + INSERT INTO t2 SELECT b+2 FROM t2; + INSERT INTO t2 SELECT b+4 FROM t2; + INSERT INTO t2 SELECT b+8 FROM t2; + INSERT INTO t2 SELECT b+16 FROM t2; + + CREATE TABLE t3(c INTEGER PRIMARY KEY); + INSERT INTO t3 VALUES(1); + INSERT INTO t3 VALUES(2); + INSERT INTO t3 VALUES(3); + } +} {} + +do_test 1.1 { + execsql { + SELECT 'test-2', t3.c, ( + SELECT count(*) + FROM t1 JOIN (SELECT DISTINCT t3.c AS p FROM t2) AS x ON t1.a=x.p + ) + FROM t3; + } +} {test-2 1 1 test-2 2 0 test-2 3 0} + +do_test 1.2 { + execsql { + CREATE TABLE t4(a, b, c); + INSERT INTO t4 VALUES('a', 1, 'one'); + INSERT INTO t4 VALUES('a', 2, 'two'); + INSERT INTO t4 VALUES('b', 1, 'three'); + INSERT INTO t4 VALUES('b', 2, 'four'); + SELECT ( + SELECT c FROM ( + SELECT * FROM t4 WHERE a=out.a ORDER BY b LIMIT 10 OFFSET 1 + ) WHERE b=out.b + ) FROM t4 AS out; + } +} {{} two {} four} + + +finish_test diff --git a/test/triggerD.test b/test/triggerD.test index 08945081da..0cce4f3cde 100644 --- a/test/triggerD.test +++ b/test/triggerD.test @@ -14,6 +14,12 @@ # the use of these columns in triggers will refer to the column and not # to the actual ROWID. Ticket [34d2ae1c6d08b5271ba5e5592936d4a1d913ffe3] # +# Also, verify that triggers created like this: +# +# CREATE TRIGGER attached.trig AFTER INSERT ON attached.tab ... +# +# can be reparsed as a main database. Ticket [d6ddba6706353915ceedc56b4e3] +# set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -171,4 +177,43 @@ do_test triggerD-3.2 { } {10003 20004} +############################################################################# +# +# Ticket [d6ddba6706353915ceedc56b4e3e72ecb4d77ba4] +# +# The following syntax really should not be allowed: +# +# CREATE TRIGGER xyz.trig BEFORE UPDATE ON xyz.tab BEGIN ... +# +# But a long-standing bug does allow it. And the "xyz.tab" slips into +# the sqlite_master table. We cannot fix the bug simply by disallowing +# "xyz.tab" since that could break legacy applications. We have to +# fix the system so that the "xyz." on "xyz.tab" is ignored. +# Verify that this is the case. +# +do_test triggerD-4.1 { + db close + file delete -force test.db test2.db + sqlite3 db test.db + db eval { + CREATE TABLE t1(x); + ATTACH 'test2.db' AS db2; + CREATE TABLE db2.t2(y); + CREATE TABLE db2.log(z); + CREATE TRIGGER db2.trig AFTER INSERT ON db2.t2 BEGIN + INSERT INTO log(z) VALUES(new.y); + END; + INSERT INTO t2 VALUES(123); + SELECT * FROM log; + } +} {123} +do_test triggerD-4.2 { + sqlite3 db2 test2.db + db2 eval { + INSERT INTO t2 VALUES(234); + SELECT * FROM log; + } +} {123 234} +db2 close + finish_test diff --git a/test/wal3.test b/test/wal3.test index bd296154e0..f7b55eb9bc 100644 --- a/test/wal3.test +++ b/test/wal3.test @@ -707,7 +707,7 @@ T delete # and continues. # set nConn 50 -if { [string match *BSD $tcl_platform(os)] } { set nConn 35 } +if { [string match *BSD $tcl_platform(os)] } { set nConn 25 } do_test wal3-9.0 { file delete -force test.db test.db-journal test.db wal sqlite3 db test.db @@ -784,4 +784,3 @@ do_multiclient_test tn { } finish_test - diff --git a/test/wal6.test b/test/wal6.test index 6fae48e9ea..ec31bb8b0f 100644 --- a/test/wal6.test +++ b/test/wal6.test @@ -43,19 +43,11 @@ foreach jmode $all_journal_modes { } {1 2} # Under Windows, you'll get an error trying to delete -# a file this is already opened. For now, make sure -# we get that error, then close the first connection +# a file this is already opened. Close the first connection # so the other tests work. if {$tcl_platform(platform)=="windows"} { if {$jmode=="persist" || $jmode=="truncate"} { - do_test wal6-1.2.$jmode.win { - sqlite3 db2 test.db - catchsql { - PRAGMA journal_mode=WAL; - } db2 - } {1 {disk I/O error}} - db2 close - db close + db close } } @@ -87,4 +79,3 @@ if {$tcl_platform(platform)=="windows"} { } finish_test - diff --git a/test/win32lock.test b/test/win32lock.test new file mode 100644 index 0000000000..802ea27d16 --- /dev/null +++ b/test/win32lock.test @@ -0,0 +1,108 @@ +# 2011 July 11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this script is recovery from transient manditory locks +# that sometimes appear on database files due to anti-virus software. +# + +if {$tcl_platform(platform)!="windows"} return + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +set testprefix win32lock + +db close +sqlite3_shutdown +test_sqlite3_log xLog +proc xLog {error_code msg} { + lappend ::log $msg +} +sqlite3 db test.db + +do_test win32lock-1.1 { + db eval { + PRAGMA cache_size=10; + CREATE TABLE t1(x,y); + INSERT INTO t1 VALUES(1,randomblob(100000)); + INSERT INTO t1 VALUES(2,randomblob(50000)); + INSERT INTO t1 VALUES(3,randomblob(25000)); + INSERT INTO t1 VALUES(4,randomblob(12500)); + SELECT x, length(y) FROM t1 ORDER BY rowid; + } +} {1 100000 2 50000 3 25000 4 12500} + +unset -nocomplain delay1 rc msg +set delay1 50 +set rc 0 +set old_pending_byte [sqlite3_test_control_pending_byte 0x40000000] +while {1} { + sqlite3_sleep 10 + lock_win32_file test.db 0 $::delay1 + set rc [catch {db eval {SELECT x, length(y) FROM t1 ORDER BY rowid}} msg] + if {$rc} { + do_test win32lock-1.2-$delay1-fin { + set ::msg + } {disk I/O error} + break + } else { + do_test win32lock-1.2-$delay1 { + set ::msg + } {1 100000 2 50000 3 25000 4 12500} + if {$::log!=""} { + do_test win32lock-1.2-$delay1-log1 { + regsub {\d+} $::log # x + set x + } {{delayed #ms for lock/sharing conflict}} + } + incr delay1 50 + } + set ::log {} +} + +do_test win32lock-2.0 { + file_control_win32_av_retry db -1 -1 +} {0 10 25} +do_test win32lock-2.1 { + file_control_win32_av_retry db 1 1 +} {0 1 1} + +set delay1 50 +while {1} { + sqlite3_sleep 10 + lock_win32_file test.db 0 $::delay1 + set rc [catch {db eval {SELECT x, length(y) FROM t1 ORDER BY rowid}} msg] + if {$rc} { + do_test win32lock-2.2-$delay1-fin { + set ::msg + } {disk I/O error} + break + } else { + do_test win32lock-2.2-$delay1 { + set ::msg + } {1 100000 2 50000 3 25000 4 12500} + if {$::log!=""} { + do_test win32lock-2.2-$delay1-log1 { + regsub {\d+} $::log # x + set x + } {{delayed #ms for lock/sharing conflict}} + } + incr delay1 50 + } + set ::log {} +} + +file_control_win32_av_retry db 10 25 +sqlite3_test_control_pending_byte $old_pending_byte +sqlite3_shutdown +test_sqlite3_log +sqlite3_initialize +finish_test diff --git a/tool/symbols.sh b/tool/symbols.sh index 8aec00569a..74b1243e6e 100644 --- a/tool/symbols.sh +++ b/tool/symbols.sh @@ -12,10 +12,10 @@ gcc -c -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE \ -DSQLITE_ENABLE_COLUMN_METADATA -DSQLITE_ENABLE_ATOMIC_WRITE \ -DSQLITE_ENABLE_ICU \ sqlite3.c -nm sqlite3.o | grep ' T ' | sort -k 3 +nm sqlite3.o | grep ' [TD] ' | sort -k 3 echo '****** Surplus symbols from a build including RTREE, FTS4 & ICU ******' -nm sqlite3.o | grep ' T ' | grep -v ' sqlite3_' +nm sqlite3.o | grep ' [TD] ' | grep -v ' .*sqlite3_' echo '****** Dependencies of the core. No extensions. No OS interface *******' gcc -c -DSQLITE_ENABLE_MEMORY_MANAGEMENT -DSQLITE_ENABLE_STAT2 \ diff --git a/tool/warnings.sh b/tool/warnings.sh index e1fa2b2d91..2eb3992f09 100644 --- a/tool/warnings.sh +++ b/tool/warnings.sh @@ -8,7 +8,11 @@ echo '********** No optimizations. Includes FTS4 and RTREE *********' gcc -c -Wshadow -Wall -Wextra -pedantic-errors -Wno-long-long -std=c89 \ -ansi -DHAVE_STDINT_H -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_RTREE \ sqlite3.c -echo '********** Optimized -O3. Includes FTS4 and RTREE *********' +echo '********** No optimizations. ENABLE_STAT2. THREADSAFE=0 *******' +gcc -c -Wshadow -Wall -Wextra -pedantic-errors -Wno-long-long -std=c89 \ + -ansi -DSQLITE_ENABLE_STAT2 -DSQLITE_THREADSAFE=0 \ + sqlite3.c +echo '********** Optimized -O3. Includes FTS4 and RTREE ************' gcc -O3 -c -Wshadow -Wall -Wextra -pedantic-errors -Wno-long-long -std=c89 \ -ansi -DHAVE_STDINT_H -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_RTREE \ sqlite3.c