diff --git a/Makefile.in b/Makefile.in index 569e5e2d2a..b8dffe63fd 100644 --- a/Makefile.in +++ b/Makefile.in @@ -242,8 +242,10 @@ SRC = \ $(TOP)/src/os.c \ $(TOP)/src/os.h \ $(TOP)/src/os_common.h \ + $(TOP)/src/os_setup.h \ $(TOP)/src/os_unix.c \ $(TOP)/src/os_win.c \ + $(TOP)/src/os_win.h \ $(TOP)/src/pager.c \ $(TOP)/src/pager.h \ $(TOP)/src/parse.y \ @@ -466,6 +468,8 @@ HDR = \ opcodes.h \ $(TOP)/src/os.h \ $(TOP)/src/os_common.h \ + $(TOP)/src/os_setup.h \ + $(TOP)/src/os_win.h \ $(TOP)/src/pager.h \ $(TOP)/src/pcache.h \ parse.h \ diff --git a/Makefile.msc b/Makefile.msc index 5fa6ce2358..a2e129038b 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1,6 +1,9 @@ # # nmake Makefile for SQLite # +############################################################################### +############################## START OF OPTIONS ############################### +############################################################################### # The toplevel directory of the source tree. This is the directory # that contains this "Makefile.msc". @@ -13,6 +16,13 @@ TOP = . USE_AMALGAMATION = 1 !ENDIF +# Set this non-0 to use the library paths and other options necessary for +# Windows Phone 8.1. +# +!IFNDEF USE_WP81_OPTS +USE_WP81_OPTS = 0 +!ENDIF + # Set this non-0 to split the SQLite amalgamation file into chunks to # be used for debugging with Visual Studio. # @@ -116,6 +126,25 @@ DEBUG = 0 OPTIMIZATIONS = 2 !ENDIF +# These are the "standard" SQLite compilation options used when compiling for +# the Windows platform. +# +!IFNDEF OPT_FEATURE_FLAGS +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1 +!ENDIF + +############################################################################### +############################### END OF OPTIONS ################################ +############################################################################### + +# This assumes that MSVC is always installed in 32-bit Program Files directory +# and sets the variable for use in locating other 32-bit installs accordingly. +# +PROGRAMFILES_X86 = $(VCINSTALLDIR)\..\.. +PROGRAMFILES_X86 = $(PROGRAMFILES_X86:\\=\) + # Check for the predefined command macro CC. This should point to the compiler # binary for the target platform. If it is not defined, simply define it to # the legacy default value 'cl.exe'. @@ -140,6 +169,15 @@ LD = link.exe RC = rc.exe !ENDIF +# Check for the MSVC runtime library path macro. Othertise, this value will +# default to the 'lib' directory underneath the MSVC installation directory. +# +!IFNDEF CRTLIBPATH +CRTLIBPATH = $(VCINSTALLDIR)\lib +!ENDIF + +CRTLIBPATH = $(CRTLIBPATH:\\=\) + # Check for the command macro NCC. This should point to the compiler binary # for the platform the compilation process is taking place on. If it is not # defined, simply define it to have the same value as the CC macro. When @@ -168,8 +206,8 @@ NCC = $(NCC:\\=\) NCC = $(CC) !ENDIF -# Check for the MSVC runtime library path macro. Othertise, this -# value will default to the 'lib' directory underneath the MSVC +# Check for the MSVC native runtime library path macro. Othertise, +# this value will default to the 'lib' directory underneath the MSVC # installation directory. # !IFNDEF NCRTLIBPATH @@ -308,7 +346,6 @@ TCC = $(TCC) -DSQLITE_ENABLE_IOTRACE RCC = $(RCC) -DSQLITE_ENABLE_IOTRACE !ENDIF -# # Prevent warnings about "insecure" MSVC runtime library functions # being used. # @@ -316,28 +353,24 @@ TCC = $(TCC) -D_CRT_SECURE_NO_DEPRECATE -D_CRT_SECURE_NO_WARNINGS BCC = $(BCC) -D_CRT_SECURE_NO_DEPRECATE -D_CRT_SECURE_NO_WARNINGS RCC = $(RCC) -D_CRT_SECURE_NO_DEPRECATE -D_CRT_SECURE_NO_WARNINGS -# # Prevent warnings about "deprecated" POSIX functions being used. # TCC = $(TCC) -D_CRT_NONSTDC_NO_DEPRECATE -D_CRT_NONSTDC_NO_WARNINGS BCC = $(BCC) -D_CRT_NONSTDC_NO_DEPRECATE -D_CRT_NONSTDC_NO_WARNINGS RCC = $(RCC) -D_CRT_NONSTDC_NO_DEPRECATE -D_CRT_NONSTDC_NO_WARNINGS -# # Use the SQLite debugging heap subsystem? # !IF $(MEMDEBUG)!=0 TCC = $(TCC) -DSQLITE_MEMDEBUG=1 RCC = $(RCC) -DSQLITE_MEMDEBUG=1 -# # Use native Win32 heap subsystem instead of malloc/free? # !ELSEIF $(WIN32HEAP)!=0 TCC = $(TCC) -DSQLITE_WIN32_MALLOC=1 RCC = $(RCC) -DSQLITE_WIN32_MALLOC=1 -# # Validate the heap on every call into the native Win32 heap subsystem? # !IF $(DEBUG)>2 @@ -440,19 +473,25 @@ OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_SESSION=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_PREUPDATE_HOOK=1 # END standard options -# BEGIN required Windows option -OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_MAX_TRIGGER_DEPTH=100 -# END required Windows option +# These are the required SQLite compilation options used when compiling for +# the Windows platform. +# +REQ_FEATURE_FLAGS = $(REQ_FEATURE_FLAGS) -DSQLITE_MAX_TRIGGER_DEPTH=100 -TCC = $(TCC) $(OPT_FEATURE_FLAGS) -RCC = $(RCC) $(OPT_FEATURE_FLAGS) +# Add the required and optional SQLite compilation options into the command +# lines used to invoke the MSVC code and resource compilers. +# +TCC = $(TCC) $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) +RCC = $(RCC) $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) -# Add in any optional parameters specified on the make commane line -# ie. make "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1". +# Add in any optional parameters specified on the commane line, e.g. +# nmake /f Makefile.msc all "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1" +# TCC = $(TCC) $(OPTS) RCC = $(RCC) $(OPTS) # If compiling for debugging, add some defines. +# !IF $(DEBUG)>0 TCC = $(TCC) -D_DEBUG BCC = $(BCC) -D_DEBUG @@ -461,6 +500,7 @@ RCC = $(RCC) -D_DEBUG # If optimizations are enabled or disabled (either implicitly or # explicitly), add the necessary flags. +# !IF $(DEBUG)>0 || $(OPTIMIZATIONS)==0 TCC = $(TCC) -Od BCC = $(BCC) -Od @@ -476,12 +516,14 @@ BCC = $(BCC) -O1 !ENDIF # If symbols are enabled (or compiling for debugging), enable PDBs. +# !IF $(DEBUG)>0 || $(SYMBOLS)!=0 TCC = $(TCC) -Zi BCC = $(BCC) -Zi !ENDIF # If ICU support is enabled, add the compiler options for it. +# !IF $(USE_ICU)!=0 TCC = $(TCC) -DSQLITE_ENABLE_ICU=1 RCC = $(RCC) -DSQLITE_ENABLE_ICU=1 @@ -493,6 +535,7 @@ RCC = $(RCC) -I$(ICUINCDIR) # Command line prefixes for compiling code, compiling resources, # linking, etc. +# LTCOMPILE = $(TCC) -Fo$@ LTRCOMPILE = $(RCC) -r LTLIB = lib.exe @@ -514,36 +557,71 @@ LTLIBOPTS = /MACHINE:$(PLATFORM) !IF $(FOR_WINRT)!=0 LTLINKOPTS = $(LTLINKOPTS) /APPCONTAINER !IF "$(VISUALSTUDIOVERSION)"=="12.0" +!IFNDEF STORELIBPATH !IF "$(PLATFORM)"=="x86" -LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(VCINSTALLDIR)\lib\store" +STORELIBPATH = $(CRTLIBPATH)\store !ELSEIF "$(PLATFORM)"=="x64" -LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(VCINSTALLDIR)\lib\store\amd64" +STORELIBPATH = $(CRTLIBPATH)\store\amd64 !ELSEIF "$(PLATFORM)"=="ARM" -LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(VCINSTALLDIR)\lib\store\arm" +STORELIBPATH = $(CRTLIBPATH)\store\arm !ELSE -LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(VCINSTALLDIR)\lib\store" +STORELIBPATH = $(CRTLIBPATH)\store +!ENDIF +!ENDIF +STORELIBPATH = $(STORELIBPATH:\\=\) +LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(STORELIBPATH)" +!ENDIF +!ENDIF + +# When compiling for Windows Phone 8.1, an extra library path is +# required. +# +!IF $(USE_WP81_OPTS)!=0 +!IFNDEF WP81LIBPATH +!IF "$(PLATFORM)"=="x86" +WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\x86 +!ELSEIF "$(PLATFORM)"=="ARM" +WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\ARM +!ELSE +WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\x86 !ENDIF !ENDIF !ENDIF +# When compiling for Windows Phone 8.1, some extra linker options +# are also required. +# +!IF $(USE_WP81_OPTS)!=0 +!IFDEF WP81LIBPATH +LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(WP81LIBPATH)" +!ENDIF +LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE +LTLINKOPTS = $(LTLINKOPTS) WindowsPhoneCore.lib RuntimeObject.lib PhoneAppModelHost.lib +LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:kernel32.lib /NODEFAULTLIB:ole32.lib +!ENDIF + # If either debugging or symbols are enabled, enable PDBs. +# !IF $(DEBUG)>0 || $(SYMBOLS)!=0 LDFLAGS = /DEBUG !ENDIF # Start with the Tcl related linker options. +# !IF $(NO_TCL)==0 LTLIBPATHS = /LIBPATH:$(TCLLIBDIR) LTLIBS = $(LIBTCL) !ENDIF # If ICU support is enabled, add the linker options for it. +# !IF $(USE_ICU)!=0 LTLIBPATHS = $(LTLIBPATHS) /LIBPATH:$(ICULIBDIR) LTLIBS = $(LTLIBS) $(LIBICU) !ENDIF # nawk compatible awk. +# !IFNDEF NAWK NAWK = gawk.exe !ENDIF @@ -642,8 +720,10 @@ SRC = \ $(TOP)\src\os.c \ $(TOP)\src\os.h \ $(TOP)\src\os_common.h \ + $(TOP)\src\os_setup.h \ $(TOP)\src\os_unix.c \ $(TOP)\src\os_win.c \ + $(TOP)\src\os_win.h \ $(TOP)\src\pager.c \ $(TOP)\src\pager.h \ $(TOP)\src\parse.y \ @@ -875,6 +955,8 @@ HDR = \ opcodes.h \ $(TOP)\src\os.h \ $(TOP)\src\os_common.h \ + $(TOP)\src\os_setup.h \ + $(TOP)\src\os_win.h \ $(TOP)\src\pager.h \ $(TOP)\src\pcache.h \ parse.h \ @@ -974,7 +1056,7 @@ lempar.c: $(TOP)\src\lempar.c copy $(TOP)\src\lempar.c . lemon.exe: $(TOP)\tool\lemon.c lempar.c - $(BCC) -Daccess=_access -Fe$@ $(TOP)\tool\lemon.c /link $(NLTLIBPATHS) + $(BCC) -Daccess=_access -Fe$@ $(TOP)\tool\lemon.c /link $(NLTLINKOPTS) $(NLTLIBPATHS) # Rules to build individual *.lo files from generated *.c files. This # applies to: @@ -1234,7 +1316,7 @@ parse.h: parse.c parse.c: $(TOP)\src\parse.y lemon.exe $(TOP)\addopcodes.awk del /Q parse.y parse.h parse.h.temp copy $(TOP)\src\parse.y . - .\lemon.exe $(OPT_FEATURE_FLAGS) $(OPTS) parse.y + .\lemon.exe $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(OPTS) parse.y move parse.h parse.h.temp $(NAWK) -f $(TOP)\addopcodes.awk parse.h.temp > parse.h @@ -1242,7 +1324,7 @@ sqlite3.h: $(TOP)\src\sqlite.h.in $(TOP)\manifest.uuid $(TOP)\VERSION $(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP:\=/) > sqlite3.h mkkeywordhash.exe: $(TOP)\tool\mkkeywordhash.c - $(BCC) -Fe$@ $(OPT_FEATURE_FLAGS) $(OPTS) $(TOP)\tool\mkkeywordhash.c /link $(NLTLIBPATHS) + $(BCC) -Fe$@ $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(OPTS) $(TOP)\tool\mkkeywordhash.c /link $(NLTLINKOPTS) $(NLTLIBPATHS) keywordhash.h: $(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe .\mkkeywordhash.exe > keywordhash.h @@ -1393,8 +1475,8 @@ clean: del /Q *.lo *.ilk *.lib *.obj *.pdb sqlite3.exe libsqlite3.lib del /Q *.cod *.da *.bb *.bbg gmon.out del /Q sqlite3.h opcodes.c opcodes.h - del /Q lemon.exe lempar.c parse.* - del /Q mkkeywordhash.exe keywordhash.h + del /Q lemon.* lempar.c parse.* + del /Q mkkeywordhash.* keywordhash.h del /Q notasharedlib.* -rmdir /Q/S .deps -rmdir /Q/S .libs diff --git a/Makefile.vxworks b/Makefile.vxworks index c9058da3e5..0d9c27f635 100644 --- a/Makefile.vxworks +++ b/Makefile.vxworks @@ -262,8 +262,10 @@ SRC = \ $(TOP)/src/os.c \ $(TOP)/src/os.h \ $(TOP)/src/os_common.h \ + $(TOP)/src/os_setup.h \ $(TOP)/src/os_unix.c \ $(TOP)/src/os_win.c \ + $(TOP)/src/os_win.h \ $(TOP)/src/pager.c \ $(TOP)/src/pager.h \ $(TOP)/src/parse.y \ @@ -416,6 +418,8 @@ HDR = \ opcodes.h \ $(TOP)/src/os.h \ $(TOP)/src/os_common.h \ + $(TOP)/src/os_setup.h \ + $(TOP)/src/os_win.h \ $(TOP)/src/pager.h \ $(TOP)/src/pcache.h \ parse.h \ diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index f7b2232902..c421aa9229 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -1333,7 +1333,7 @@ static int fts3InitVtab( p->bHasStat = isFts4; p->bFts4 = isFts4; p->bDescIdx = bDescIdx; - p->bAutoincrmerge = 0xff; /* 0xff means setting unknown */ + p->nAutoincrmerge = 0xff; /* 0xff means setting unknown */ p->zContentTbl = zContent; p->zLanguageid = zLanguageid; zContent = 0; @@ -3302,7 +3302,10 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){ Fts3Table *p = (Fts3Table*)pVtab; int rc = sqlite3Fts3PendingTermsFlush(p); - if( rc==SQLITE_OK && p->bAutoincrmerge==1 && p->nLeafAdd>(nMinMerge/16) ){ + if( rc==SQLITE_OK + && p->nLeafAdd>(nMinMerge/16) + && p->nAutoincrmerge && p->nAutoincrmerge!=0xff + ){ int mxLevel = 0; /* Maximum relative level value in db */ int A; /* Incr-merge parameter A */ @@ -3310,7 +3313,7 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){ assert( rc==SQLITE_OK || mxLevel==0 ); A = p->nLeafAdd * mxLevel; A += (A/2); - if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, 8); + if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, p->nAutoincrmerge); } sqlite3Fts3SegmentsClose(p); return rc; diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index a5bb2f0041..1383102f4c 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -210,13 +210,13 @@ struct Fts3Table { sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ char *zContentTbl; /* content=xxx option, or NULL */ char *zLanguageid; /* languageid=xxx option, or NULL */ - u8 bAutoincrmerge; /* True if automerge=1 */ + int nAutoincrmerge; /* Value configured by 'automerge' */ u32 nLeafAdd; /* Number of leaf blocks added this trans */ /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. */ - sqlite3_stmt *aStmt[37]; + sqlite3_stmt *aStmt[40]; char *zReadExprlist; char *zWriteExprlist; diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index 9c71f26ba1..95a9b1aada 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -185,40 +185,23 @@ static int getNextToken( int rc; sqlite3_tokenizer_cursor *pCursor; Fts3Expr *pRet = 0; - int nConsumed = 0; + int i = 0; - rc = sqlite3Fts3OpenTokenizer(pTokenizer, pParse->iLangid, z, n, &pCursor); + /* Set variable i to the maximum number of bytes of input to tokenize. */ + for(i=0; iiLangid, z, i, &pCursor); if( rc==SQLITE_OK ){ const char *zToken; int nToken = 0, iStart = 0, iEnd = 0, iPosition = 0; int nByte; /* total space to allocate */ rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition); - - if( (rc==SQLITE_OK || rc==SQLITE_DONE) && sqlite3_fts3_enable_parentheses ){ - int i; - if( rc==SQLITE_DONE ) iStart = n; - for(i=0; inNest++; - rc = fts3ExprParse(pParse, &z[i+1], n-i-1, &pRet, &nConsumed); - if( rc==SQLITE_OK && !pRet ){ - rc = SQLITE_DONE; - } - nConsumed = (int)(i + 1 + nConsumed); - break; - } - - if( z[i]==')' ){ - rc = SQLITE_DONE; - pParse->nNest--; - nConsumed = i+1; - break; - } - } - } - - if( nConsumed==0 && rc==SQLITE_OK ){ + if( rc==SQLITE_OK ){ nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken; pRet = (Fts3Expr *)fts3MallocZero(nByte); if( !pRet ){ @@ -252,13 +235,14 @@ static int getNextToken( } } - nConsumed = iEnd; + *pnConsumed = iEnd; + }else if( i && rc==SQLITE_DONE ){ + rc = SQLITE_OK; } pModule->xClose(pCursor); } - *pnConsumed = nConsumed; *ppExpr = pRet; return rc; } @@ -508,6 +492,21 @@ static int getNextNode( return getNextString(pParse, &zInput[1], ii-1, ppExpr); } + if( sqlite3_fts3_enable_parentheses ){ + if( *zInput=='(' ){ + int nConsumed = 0; + pParse->nNest++; + rc = fts3ExprParse(pParse, zInput+1, nInput-1, ppExpr, &nConsumed); + if( rc==SQLITE_OK && !*ppExpr ){ rc = SQLITE_DONE; } + *pnConsumed = (int)(zInput - z) + 1 + nConsumed; + return rc; + }else if( *zInput==')' ){ + pParse->nNest--; + *pnConsumed = (zInput - z) + 1; + *ppExpr = 0; + return SQLITE_DONE; + } + } /* If control flows to this point, this must be a regular token, or ** the end of the input. Read a regular token using the sqlite3_tokenizer @@ -626,96 +625,100 @@ static int fts3ExprParse( while( rc==SQLITE_OK ){ Fts3Expr *p = 0; int nByte = 0; + rc = getNextNode(pParse, zIn, nIn, &p, &nByte); + assert( nByte>0 || (rc!=SQLITE_OK && p==0) ); if( rc==SQLITE_OK ){ - int isPhrase; + if( p ){ + int isPhrase; - if( !sqlite3_fts3_enable_parentheses - && p->eType==FTSQUERY_PHRASE && pParse->isNot - ){ - /* Create an implicit NOT operator. */ - Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr)); - if( !pNot ){ - sqlite3Fts3ExprFree(p); - rc = SQLITE_NOMEM; - goto exprparse_out; - } - pNot->eType = FTSQUERY_NOT; - pNot->pRight = p; - p->pParent = pNot; - if( pNotBranch ){ - pNot->pLeft = pNotBranch; - pNotBranch->pParent = pNot; - } - pNotBranch = pNot; - p = pPrev; - }else{ - int eType = p->eType; - isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft); - - /* The isRequirePhrase variable is set to true if a phrase or - ** an expression contained in parenthesis is required. If a - ** binary operator (AND, OR, NOT or NEAR) is encounted when - ** isRequirePhrase is set, this is a syntax error. - */ - if( !isPhrase && isRequirePhrase ){ - sqlite3Fts3ExprFree(p); - rc = SQLITE_ERROR; - goto exprparse_out; - } - - if( isPhrase && !isRequirePhrase ){ - /* Insert an implicit AND operator. */ - Fts3Expr *pAnd; - assert( pRet && pPrev ); - pAnd = fts3MallocZero(sizeof(Fts3Expr)); - if( !pAnd ){ + if( !sqlite3_fts3_enable_parentheses + && p->eType==FTSQUERY_PHRASE && pParse->isNot + ){ + /* Create an implicit NOT operator. */ + Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr)); + if( !pNot ){ sqlite3Fts3ExprFree(p); rc = SQLITE_NOMEM; goto exprparse_out; } - pAnd->eType = FTSQUERY_AND; - insertBinaryOperator(&pRet, pPrev, pAnd); - pPrev = pAnd; - } + pNot->eType = FTSQUERY_NOT; + pNot->pRight = p; + p->pParent = pNot; + if( pNotBranch ){ + pNot->pLeft = pNotBranch; + pNotBranch->pParent = pNot; + } + pNotBranch = pNot; + p = pPrev; + }else{ + int eType = p->eType; + isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft); - /* This test catches attempts to make either operand of a NEAR - ** operator something other than a phrase. For example, either of - ** the following: - ** - ** (bracketed expression) NEAR phrase - ** phrase NEAR (bracketed expression) - ** - ** Return an error in either case. - */ - if( pPrev && ( + /* The isRequirePhrase variable is set to true if a phrase or + ** an expression contained in parenthesis is required. If a + ** binary operator (AND, OR, NOT or NEAR) is encounted when + ** isRequirePhrase is set, this is a syntax error. + */ + if( !isPhrase && isRequirePhrase ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_ERROR; + goto exprparse_out; + } + + if( isPhrase && !isRequirePhrase ){ + /* Insert an implicit AND operator. */ + Fts3Expr *pAnd; + assert( pRet && pPrev ); + pAnd = fts3MallocZero(sizeof(Fts3Expr)); + if( !pAnd ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_NOMEM; + goto exprparse_out; + } + pAnd->eType = FTSQUERY_AND; + insertBinaryOperator(&pRet, pPrev, pAnd); + pPrev = pAnd; + } + + /* This test catches attempts to make either operand of a NEAR + ** operator something other than a phrase. For example, either of + ** the following: + ** + ** (bracketed expression) NEAR phrase + ** phrase NEAR (bracketed expression) + ** + ** Return an error in either case. + */ + if( pPrev && ( (eType==FTSQUERY_NEAR && !isPhrase && pPrev->eType!=FTSQUERY_PHRASE) || (eType!=FTSQUERY_PHRASE && isPhrase && pPrev->eType==FTSQUERY_NEAR) - )){ - sqlite3Fts3ExprFree(p); - rc = SQLITE_ERROR; - goto exprparse_out; - } - - if( isPhrase ){ - if( pRet ){ - assert( pPrev && pPrev->pLeft && pPrev->pRight==0 ); - pPrev->pRight = p; - p->pParent = pPrev; - }else{ - pRet = p; + )){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_ERROR; + goto exprparse_out; } - }else{ - insertBinaryOperator(&pRet, pPrev, p); + + if( isPhrase ){ + if( pRet ){ + assert( pPrev && pPrev->pLeft && pPrev->pRight==0 ); + pPrev->pRight = p; + p->pParent = pPrev; + }else{ + pRet = p; + } + }else{ + insertBinaryOperator(&pRet, pPrev, p); + } + isRequirePhrase = !isPhrase; } - isRequirePhrase = !isPhrase; + pPrev = p; } assert( nByte>0 ); } assert( rc!=SQLITE_OK || (nByte>0 && nByte<=nIn) ); nIn -= nByte; zIn += nByte; - pPrev = p; } if( rc==SQLITE_DONE && pRet && isRequirePhrase ){ diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 90d1609226..ea3c08802a 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -193,6 +193,7 @@ struct SegmentWriter { int nSize; /* Size of allocation at aData */ int nData; /* Bytes of data in aData */ char *aData; /* Pointer to block from malloc() */ + i64 nLeafData; /* Number of bytes of leaf data written */ }; /* @@ -268,6 +269,10 @@ struct SegmentNode { #define SQL_SELECT_INDEXES 35 #define SQL_SELECT_MXLEVEL 36 +#define SQL_SELECT_LEVEL_RANGE2 37 +#define SQL_UPDATE_LEVEL_IDX 38 +#define SQL_UPDATE_LEVEL 39 + /* ** This function is used to obtain an SQLite prepared statement handle ** for the statement identified by the second argument. If successful, @@ -369,7 +374,18 @@ static int fts3SqlStmt( /* SQL_SELECT_MXLEVEL ** Return the largest relative level in the FTS index or indexes. */ -/* 36 */ "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'" +/* 36 */ "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'", + + /* Return segments in order from oldest to newest.*/ +/* 37 */ "SELECT level, idx, end_block " + "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? " + "ORDER BY level DESC, idx ASC", + + /* Update statements used while promoting segments */ +/* 38 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=-1,idx=? " + "WHERE level=? AND idx=?", +/* 39 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=? WHERE level=-1" + }; int rc = SQLITE_OK; sqlite3_stmt *pStmt; @@ -1910,6 +1926,7 @@ static int fts3WriteSegdir( sqlite3_int64 iStartBlock, /* Value for "start_block" field */ sqlite3_int64 iLeafEndBlock, /* Value for "leaves_end_block" field */ sqlite3_int64 iEndBlock, /* Value for "end_block" field */ + sqlite3_int64 nLeafData, /* Bytes of leaf data in segment */ char *zRoot, /* Blob value for "root" field */ int nRoot /* Number of bytes in buffer zRoot */ ){ @@ -1920,7 +1937,13 @@ static int fts3WriteSegdir( sqlite3_bind_int(pStmt, 2, iIdx); sqlite3_bind_int64(pStmt, 3, iStartBlock); sqlite3_bind_int64(pStmt, 4, iLeafEndBlock); - sqlite3_bind_int64(pStmt, 5, iEndBlock); + if( nLeafData==0 ){ + sqlite3_bind_int64(pStmt, 5, iEndBlock); + }else{ + char *zEnd = sqlite3_mprintf("%lld %lld", iEndBlock, nLeafData); + if( !zEnd ) return SQLITE_NOMEM; + sqlite3_bind_text(pStmt, 5, zEnd, -1, sqlite3_free); + } sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC); sqlite3_step(pStmt); rc = sqlite3_reset(pStmt); @@ -2246,6 +2269,9 @@ static int fts3SegWriterAdd( nDoclist; /* Doclist data */ } + /* Increase the total number of bytes written to account for the new entry. */ + pWriter->nLeafData += nReq; + /* If the buffer currently allocated is too small for this entry, realloc ** the buffer to make it large enough. */ @@ -2317,13 +2343,13 @@ static int fts3SegWriterFlush( pWriter->iFirst, pWriter->iFree, &iLast, &zRoot, &nRoot); } if( rc==SQLITE_OK ){ - rc = fts3WriteSegdir( - p, iLevel, iIdx, pWriter->iFirst, iLastLeaf, iLast, zRoot, nRoot); + rc = fts3WriteSegdir(p, iLevel, iIdx, + pWriter->iFirst, iLastLeaf, iLast, pWriter->nLeafData, zRoot, nRoot); } }else{ /* The entire tree fits on the root node. Write it to the segdir table. */ - rc = fts3WriteSegdir( - p, iLevel, iIdx, 0, 0, 0, pWriter->aData, pWriter->nData); + rc = fts3WriteSegdir(p, iLevel, iIdx, + 0, 0, 0, pWriter->nLeafData, pWriter->aData, pWriter->nData); } p->nLeafAdd++; return rc; @@ -2407,6 +2433,37 @@ static int fts3SegmentMaxLevel( return sqlite3_reset(pStmt); } +/* +** iAbsLevel is an absolute level that may be assumed to exist within +** the database. This function checks if it is the largest level number +** within its index. Assuming no error occurs, *pbMax is set to 1 if +** iAbsLevel is indeed the largest level, or 0 otherwise, and SQLITE_OK +** is returned. If an error occurs, an error code is returned and the +** final value of *pbMax is undefined. +*/ +static int fts3SegmentIsMaxLevel(Fts3Table *p, i64 iAbsLevel, int *pbMax){ + + /* Set pStmt to the compiled version of: + ** + ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? + ** + ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). + */ + sqlite3_stmt *pStmt; + int rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_int64(pStmt, 1, iAbsLevel+1); + sqlite3_bind_int64(pStmt, 2, + ((iAbsLevel/FTS3_SEGDIR_MAXLEVEL)+1) * FTS3_SEGDIR_MAXLEVEL + ); + + *pbMax = 0; + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *pbMax = sqlite3_column_type(pStmt, 0)==SQLITE_NULL; + } + return sqlite3_reset(pStmt); +} + /* ** Delete all entries in the %_segments table associated with the segment ** opened with seg-reader pSeg. This function does not affect the contents @@ -2942,6 +2999,140 @@ void sqlite3Fts3SegReaderFinish( } } +/* +** Decode the "end_block" field, selected by column iCol of the SELECT +** statement passed as the first argument. +** +** The "end_block" field may contain either an integer, or a text field +** containing the text representation of two non-negative integers separated +** by one or more space (0x20) characters. In the first case, set *piEndBlock +** to the integer value and *pnByte to zero before returning. In the second, +** set *piEndBlock to the first value and *pnByte to the second. +*/ +static void fts3ReadEndBlockField( + sqlite3_stmt *pStmt, + int iCol, + i64 *piEndBlock, + i64 *pnByte +){ + const unsigned char *zText = sqlite3_column_text(pStmt, iCol); + if( zText ){ + int i; + int iMul = 1; + i64 iVal = 0; + for(i=0; zText[i]>='0' && zText[i]<='9'; i++){ + iVal = iVal*10 + (zText[i] - '0'); + } + *piEndBlock = iVal; + while( zText[i]==' ' ) i++; + iVal = 0; + if( zText[i]=='-' ){ + i++; + iMul = -1; + } + for(/* no-op */; zText[i]>='0' && zText[i]<='9'; i++){ + iVal = iVal*10 + (zText[i] - '0'); + } + *pnByte = (iVal * (i64)iMul); + } +} + + +/* +** A segment of size nByte bytes has just been written to absolute level +** iAbsLevel. Promote any segments that should be promoted as a result. +*/ +static int fts3PromoteSegments( + Fts3Table *p, /* FTS table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level just updated */ + sqlite3_int64 nByte /* Size of new segment at iAbsLevel */ +){ + int rc = SQLITE_OK; + sqlite3_stmt *pRange; + + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE2, &pRange, 0); + + if( rc==SQLITE_OK ){ + int bOk = 0; + i64 iLast = (iAbsLevel/FTS3_SEGDIR_MAXLEVEL + 1) * FTS3_SEGDIR_MAXLEVEL - 1; + i64 nLimit = (nByte*3)/2; + + /* Loop through all entries in the %_segdir table corresponding to + ** segments in this index on levels greater than iAbsLevel. If there is + ** at least one such segment, and it is possible to determine that all + ** such segments are smaller than nLimit bytes in size, they will be + ** promoted to level iAbsLevel. */ + sqlite3_bind_int64(pRange, 1, iAbsLevel+1); + sqlite3_bind_int64(pRange, 2, iLast); + while( SQLITE_ROW==sqlite3_step(pRange) ){ + i64 nSize, dummy; + fts3ReadEndBlockField(pRange, 2, &dummy, &nSize); + if( nSize<=0 || nSize>nLimit ){ + /* If nSize==0, then the %_segdir.end_block field does not not + ** contain a size value. This happens if it was written by an + ** old version of FTS. In this case it is not possible to determine + ** the size of the segment, and so segment promotion does not + ** take place. */ + bOk = 0; + break; + } + bOk = 1; + } + rc = sqlite3_reset(pRange); + + if( bOk ){ + int iIdx = 0; + sqlite3_stmt *pUpdate1; + sqlite3_stmt *pUpdate2; + + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL_IDX, &pUpdate1, 0); + } + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL, &pUpdate2, 0); + } + + if( rc==SQLITE_OK ){ + + /* Loop through all %_segdir entries for segments in this index with + ** levels equal to or greater than iAbsLevel. As each entry is visited, + ** updated it to set (level = -1) and (idx = N), where N is 0 for the + ** oldest segment in the range, 1 for the next oldest, and so on. + ** + ** In other words, move all segments being promoted to level -1, + ** setting the "idx" fields as appropriate to keep them in the same + ** order. The contents of level -1 (which is never used, except + ** transiently here), will be moved back to level iAbsLevel below. */ + sqlite3_bind_int64(pRange, 1, iAbsLevel); + while( SQLITE_ROW==sqlite3_step(pRange) ){ + sqlite3_bind_int(pUpdate1, 1, iIdx++); + sqlite3_bind_int(pUpdate1, 2, sqlite3_column_int(pRange, 0)); + sqlite3_bind_int(pUpdate1, 3, sqlite3_column_int(pRange, 1)); + sqlite3_step(pUpdate1); + rc = sqlite3_reset(pUpdate1); + if( rc!=SQLITE_OK ){ + sqlite3_reset(pRange); + break; + } + } + } + if( rc==SQLITE_OK ){ + rc = sqlite3_reset(pRange); + } + + /* Move level -1 to level iAbsLevel */ + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pUpdate2, 1, iAbsLevel); + sqlite3_step(pUpdate2); + rc = sqlite3_reset(pUpdate2); + } + } + } + + + return rc; +} + /* ** Merge all level iLevel segments in the database into a single ** iLevel+1 segment. Or, if iLevel<0, merge all segments into a @@ -2966,6 +3157,7 @@ static int fts3SegmentMerge( Fts3SegFilter filter; /* Segment term filter condition */ Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */ int bIgnoreEmpty = 0; /* True to ignore empty segments */ + i64 iMaxLevel = 0; /* Max level number for this index/langid */ assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel==FTS3_SEGCURSOR_PENDING @@ -2977,6 +3169,11 @@ static int fts3SegmentMerge( rc = sqlite3Fts3SegReaderCursor(p, iLangid, iIndex, iLevel, 0, 0, 1, 0, &csr); if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished; + if( iLevel!=FTS3_SEGCURSOR_PENDING ){ + rc = fts3SegmentMaxLevel(p, iLangid, iIndex, &iMaxLevel); + if( rc!=SQLITE_OK ) goto finished; + } + if( iLevel==FTS3_SEGCURSOR_ALL ){ /* This call is to merge all segments in the database to a single ** segment. The level of the new segment is equal to the numerically @@ -2986,21 +3183,21 @@ static int fts3SegmentMerge( rc = SQLITE_DONE; goto finished; } - rc = fts3SegmentMaxLevel(p, iLangid, iIndex, &iNewLevel); + iNewLevel = iMaxLevel; bIgnoreEmpty = 1; - }else if( iLevel==FTS3_SEGCURSOR_PENDING ){ - iNewLevel = getAbsoluteLevel(p, iLangid, iIndex, 0); - rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, 0, &iIdx); }else{ /* This call is to merge all segments at level iLevel. find the next ** available segment index at level iLevel+1. The call to ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to ** a single iLevel+2 segment if necessary. */ - rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, iLevel+1, &iIdx); + assert( FTS3_SEGCURSOR_PENDING==-1 ); iNewLevel = getAbsoluteLevel(p, iLangid, iIndex, iLevel+1); + rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, iLevel+1, &iIdx); + bIgnoreEmpty = (iLevel!=FTS3_SEGCURSOR_PENDING) && (iNewLevel>iMaxLevel); } if( rc!=SQLITE_OK ) goto finished; + assert( csr.nSegment>0 ); assert( iNewLevel>=getAbsoluteLevel(p, iLangid, iIndex, 0) ); assert( iNewLevelnLeafData); + } + } + } finished: fts3SegWriterFree(pWriter); @@ -3035,7 +3239,7 @@ static int fts3SegmentMerge( /* -** Flush the contents of pendingTerms to level 0 segments. +** Flush the contents of pendingTerms to level 0 segments. */ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ int rc = SQLITE_OK; @@ -3051,14 +3255,19 @@ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ ** estimate the number of leaf blocks of content to be written */ if( rc==SQLITE_OK && p->bHasStat - && p->bAutoincrmerge==0xff && p->nLeafAdd>0 + && p->nAutoincrmerge==0xff && p->nLeafAdd>0 ){ sqlite3_stmt *pStmt = 0; rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); rc = sqlite3_step(pStmt); - p->bAutoincrmerge = (rc==SQLITE_ROW && sqlite3_column_int(pStmt, 0)); + if( rc==SQLITE_ROW ){ + p->nAutoincrmerge = sqlite3_column_int(pStmt, 0); + if( p->nAutoincrmerge==1 ) p->nAutoincrmerge = 8; + }else if( rc==SQLITE_DONE ){ + p->nAutoincrmerge = 0; + } rc = sqlite3_reset(pStmt); } } @@ -3426,6 +3635,8 @@ struct IncrmergeWriter { int iIdx; /* Index of *output* segment in iAbsLevel+1 */ sqlite3_int64 iStart; /* Block number of first allocated block */ sqlite3_int64 iEnd; /* Block number of last allocated block */ + sqlite3_int64 nLeafData; /* Bytes of leaf page data so far */ + u8 bNoLeafData; /* If true, store 0 for segment size */ NodeWriter aNodeWriter[FTS_MAX_APPENDABLE_HEIGHT]; }; @@ -3764,8 +3975,8 @@ static int fts3IncrmergeAppend( nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; } + pWriter->nLeafData += nSpace; blobGrowBuffer(&pLeaf->block, pLeaf->block.n + nSpace, &rc); - if( rc==SQLITE_OK ){ if( pLeaf->block.n==0 ){ pLeaf->block.n = 1; @@ -3864,6 +4075,7 @@ static void fts3IncrmergeRelease( pWriter->iStart, /* start_block */ pWriter->aNodeWriter[0].iBlock, /* leaves_end_block */ pWriter->iEnd, /* end_block */ + (pWriter->bNoLeafData==0 ? pWriter->nLeafData : 0), /* end_block */ pRoot->block.a, pRoot->block.n /* root */ ); } @@ -3965,7 +4177,11 @@ static int fts3IncrmergeLoad( if( sqlite3_step(pSelect)==SQLITE_ROW ){ iStart = sqlite3_column_int64(pSelect, 1); iLeafEnd = sqlite3_column_int64(pSelect, 2); - iEnd = sqlite3_column_int64(pSelect, 3); + fts3ReadEndBlockField(pSelect, 3, &iEnd, &pWriter->nLeafData); + if( pWriter->nLeafData<0 ){ + pWriter->nLeafData = pWriter->nLeafData * -1; + } + pWriter->bNoLeafData = (pWriter->nLeafData==0); nRoot = sqlite3_column_bytes(pSelect, 4); aRoot = sqlite3_column_blob(pSelect, 4); }else{ @@ -4566,11 +4782,11 @@ static int fts3IncrmergeHintPop(Blob *pHint, i64 *piAbsLevel, int *pnInput){ /* ** Attempt an incremental merge that writes nMerge leaf blocks. ** -** Incremental merges happen nMin segments at a time. The two -** segments to be merged are the nMin oldest segments (the ones with -** the smallest indexes) in the highest level that contains at least -** nMin segments. Multiple merges might occur in an attempt to write the -** quota of nMerge leaf blocks. +** Incremental merges happen nMin segments at a time. The segments +** to be merged are the nMin oldest segments (the ones with the smallest +** values for the _segdir.idx field) in the highest level that contains +** at least nMin segments. Multiple merges might occur in an attempt to +** write the quota of nMerge leaf blocks. */ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ int rc; /* Return code */ @@ -4595,6 +4811,7 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ const i64 nMod = FTS3_SEGDIR_MAXLEVEL * p->nIndex; sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */ int bUseHint = 0; /* True if attempting to append */ + int iIdx = 0; /* Largest idx in level (iAbsLevel+1) */ /* Search the %_segdir table for the absolute level with the smallest ** relative level number that contains at least nMin segments, if any. @@ -4648,6 +4865,19 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ ** to start work on some other level. */ memset(pWriter, 0, nAlloc); pFilter->flags = FTS3_SEGMENT_REQUIRE_POS; + + if( rc==SQLITE_OK ){ + rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx); + assert( bUseHint==1 || bUseHint==0 ); + if( iIdx==0 || (bUseHint && iIdx==1) ){ + int bIgnore; + rc = fts3SegmentIsMaxLevel(p, iAbsLevel+1, &bIgnore); + if( bIgnore ){ + pFilter->flags |= FTS3_SEGMENT_IGNORE_EMPTY; + } + } + } + if( rc==SQLITE_OK ){ rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr); } @@ -4655,16 +4885,12 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ && SQLITE_OK==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter)) && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr)) ){ - int iIdx = 0; /* Largest idx in level (iAbsLevel+1) */ - rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx); - if( rc==SQLITE_OK ){ - if( bUseHint && iIdx>0 ){ - const char *zKey = pCsr->zTerm; - int nKey = pCsr->nTerm; - rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx-1, zKey, nKey, pWriter); - }else{ - rc = fts3IncrmergeWriter(p, iAbsLevel, iIdx, pCsr, pWriter); - } + if( bUseHint && iIdx>0 ){ + const char *zKey = pCsr->zTerm; + int nKey = pCsr->nTerm; + rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx-1, zKey, nKey, pWriter); + }else{ + rc = fts3IncrmergeWriter(p, iAbsLevel, iIdx, pCsr, pWriter); } if( rc==SQLITE_OK && pWriter->nLeafEst ){ @@ -4686,7 +4912,13 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ } } + if( nSeg!=0 ){ + pWriter->nLeafData = pWriter->nLeafData * -1; + } fts3IncrmergeRelease(p, pWriter, &rc); + if( nSeg==0 && pWriter->bNoLeafData==0 ){ + fts3PromoteSegments(p, iAbsLevel+1, pWriter->nLeafData); + } } sqlite3Fts3SegReaderFinish(pCsr); @@ -4773,7 +5005,10 @@ static int fts3DoAutoincrmerge( ){ int rc = SQLITE_OK; sqlite3_stmt *pStmt = 0; - p->bAutoincrmerge = fts3Getint(&zParam)!=0; + p->nAutoincrmerge = fts3Getint(&zParam); + if( p->nAutoincrmerge==1 || p->nAutoincrmerge>FTS3_MERGE_COUNT ){ + p->nAutoincrmerge = 8; + } if( !p->bHasStat ){ assert( p->bFts4==0 ); sqlite3Fts3CreateStatTable(&rc, p); @@ -4782,7 +5017,7 @@ static int fts3DoAutoincrmerge( rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0); if( rc ) return rc; sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); - sqlite3_bind_int(pStmt, 2, p->bAutoincrmerge); + sqlite3_bind_int(pStmt, 2, p->nAutoincrmerge); sqlite3_step(pStmt); rc = sqlite3_reset(pStmt); return rc; diff --git a/main.mk b/main.mk index 57757ccc6a..47c9ae321c 100644 --- a/main.mk +++ b/main.mk @@ -122,8 +122,10 @@ SRC = \ $(TOP)/src/os.c \ $(TOP)/src/os.h \ $(TOP)/src/os_common.h \ + $(TOP)/src/os_setup.h \ $(TOP)/src/os_unix.c \ $(TOP)/src/os_win.c \ + $(TOP)/src/os_win.h \ $(TOP)/src/pager.c \ $(TOP)/src/pager.h \ $(TOP)/src/parse.y \ @@ -209,6 +211,7 @@ SRC += \ $(TOP)/ext/icu/sqliteicu.h \ $(TOP)/ext/icu/icu.c SRC += \ + $(TOP)/ext/rtree/sqlite3rtree.h \ $(TOP)/ext/rtree/rtree.h \ $(TOP)/ext/rtree/rtree.c SRC += \ @@ -345,6 +348,8 @@ HDR = \ opcodes.h \ $(TOP)/src/os.h \ $(TOP)/src/os_common.h \ + $(TOP)/src/os_setup.h \ + $(TOP)/src/os_win.h \ $(TOP)/src/pager.h \ $(TOP)/src/pcache.h \ parse.h \ diff --git a/manifest b/manifest index 551c5d8bd6..dd302e6b0c 100644 --- a/manifest +++ b/manifest @@ -1,10 +1,10 @@ -C Merge\sall\srecent\strunk\senhancements\sand\sfixes\sinto\sthe\ssessions\sbranch. -D 2014-04-28T18:02:21.241 +C Merge\srecent\strunk\schanges\sinto\sthe\ssessions\sbranch. +D 2014-05-20T00:20:23.666 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in e4ee6d36cdf6136aee0158675a3b24dd3bf31a5a +F Makefile.in faf03dca4bd103f3e98b317aeba930dea49cfaa1 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc b448317831d9c364bf501538becce5cc1f0a0f8b -F Makefile.vxworks db21ed42a01d5740e656b16f92cb5d8d5e5dd315 +F Makefile.msc ac23895c1c2928518fa729b3b71bc30a6ab19347 +F Makefile.vxworks 034289efa9d591b04b1a73598623119c306cbba0 F README.md 64f270c43c38c46de749e419c22f0ae2f4499fe8 F VERSION 9f823c026c6a32fc5f84d212a8aae0a221dba45c F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -78,11 +78,11 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 41b1920b9a8657963f09cb93b208c2671c5568db +F ext/fts3/fts3.c e83f894cf1adaf8decd6b1de76bfdcdb79b25507 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h bdeb9015405e8facffb8fc7e09174521a2a780f4 +F ext/fts3/fts3Int.h 16cddf2d7b0e5f3681615ae1d8ca0e45fca44918 F ext/fts3/fts3_aux.c 5c211e17a64885faeb16b9ba7772f9d5445c2365 -F ext/fts3/fts3_expr.c 5165c365cb5a035f5be8bb296f7aa3211d43e4ac +F ext/fts3/fts3_expr.c 2ac35bda474f00c14c19608e49a02c8c7ceb9970 F ext/fts3/fts3_hash.c 29b986e43f4e9dd40110eafa377dc0d63c422c60 F ext/fts3/fts3_hash.h 39cf6874dc239d6b4e30479b1975fe5b22a3caaf F ext/fts3/fts3_icu.c e319e108661147bcca8dd511cd562f33a1ba81b5 @@ -96,7 +96,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_unicode.c 92391b4b4fb043564c6539ea9b8661e3bcba47b9 F ext/fts3/fts3_unicode2.c 0113d3acf13429e6dc38e0647d1bc71211c31a4d -F ext/fts3/fts3_write.c 74c00329006c3ed6325ba4e5ab7c9b5fc99c8934 +F ext/fts3/fts3_write.c 9e4e8579ad26cff3ee7743aaf5c3fe937fc441b4 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197 @@ -158,7 +158,7 @@ F ext/session/test_session.c 7878ac0e2fe9675e8ec24d54f6a538ccc105977f F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt f439556c5ce01ced70987e5ee86549a45165d9ff -F main.mk 82b154dd823b75c2f1247f1bb98893bf94a5af57 +F main.mk 341301be4e8956ad69be7c962e1120b85c0b4a1d F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea F mkopcodeh.awk c6b3fa301db6ef7ac916b14c60868aeaec1337b5 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 @@ -173,7 +173,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c b00900877f766f116f9e16116f1ccacdc21d82f1 -F src/analyze.c 663e0b291d27eb03c9fd6b421e2d61ba348a2389 +F src/analyze.c 3596f863bb80126fe56ba217df5932749271efc8 F src/attach.c 3801129015ef59d76bf23c95ef9b0069d18a0c52 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 @@ -182,7 +182,7 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 6c9b51abd404ce5b78b173b6f2248e8cb824758c F src/btree.h d79306df4ed9181b48916737fe8871a4392c4594 F src/btreeInt.h cf180d86b2e9e418f638d65baa425c4c69c0e0e3 -F src/build.c 5bfeea8f302ec2926c9eea321a61daea92a29fa4 +F src/build.c 02665ca158431a0926b10cbd7d8178a4c9fc4a22 F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a @@ -191,8 +191,8 @@ F src/delete.c 50b74c1dde25d1ebcb4fa5c870762e6470ee46f1 F src/expr.c 4f9e497c66e2f25a4d139357a778c84d5713207c F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 5269ef07b100763134f71b889327c333bd0989cf -F src/func.c 2945bb2c4cdc0ac43733046285a4434310be1811 -F src/global.c 1d7bb7ea8254ae6a68ed9bfaf65fcb3d1690b486 +F src/func.c 2e16316ec3a6365a0dc3e553c586f91b20f7f6c8 +F src/global.c 1e4bd956dc2f608f87d2a929abc4a20db65f30e4 F src/hash.c d139319967164f139c8d1bb8a11b14db9c4ba3cd F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 @@ -201,7 +201,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 867c7b330b740c6c917af9956b13b81d0a048303 -F src/main.c a71db3a3c95e4b435644511cea305f28be73673f +F src/main.c d1fbc849b5a06bfd2e3a1952a68515064a0078ba F src/malloc.c 0203ebce9152c6a0e5de520140b8ba65187350be F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b @@ -213,37 +213,39 @@ F src/mutex.c d3b66a569368015e0fcb1ac15f81c119f504d3bc F src/mutex.h 5bc526e19dccc412b7ff04642f6fdad3fdfdabea F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553 F src/mutex_unix.c c3a4e00f96ba068a8dbef34084465979aaf369cc -F src/mutex_w32.c 6108c88e1cb38d8fbb3534b170793815cbedbf97 +F src/mutex_w32.c 9bcab6e699fdb20eae784237c36b299830fb3316 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace -F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f +F src/os.h 60d419395e32a8029fa380a80a3da2e9030f635e F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 +F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c ae4b5240af4619d711301d7992396e182585269f -F src/os_win.c e71678ac927d0a0fb11d993db20a9748eabf808e -F src/pager.c ab62a24218d87dda1be641f6c5ad291bff78fd94 +F src/os_win.c 485d06a93965f306c7281fca0937829292367234 +F src/os_win.h 057344a6720b4c8405d9bd98f58cb37a6ee46c25 +F src/pager.c f6bb1fa6cdf2062f2d8aec3e64db302bca519ab8 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 F src/pcache.c d8eafac28290d4bb80332005435db44991d07fc2 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 102e6f5a2fbc646154463eb856d1fd716867b64c -F src/pragma.c 21ece94d4f3e76e8e150deecafb9c7abd398ec67 +F src/pragma.c 810ef31ccfaa233201dcf100637a9777cc24e897 F src/prepare.c 677521ab7132615a8a26107a1d1c3132f44ae337 F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be -F src/select.c bc7feff0fb4c4a1b9d655b717bef166846b48e33 +F src/select.c 089c4d46f067a5cccae93524c6377f981ba99bd9 F src/shell.c 2afe7a7154e97be0c74c5feacf09626bda8493be -F src/sqlite.h.in 6e6d3e092ca968abb3e65fbe22ec9220a3b616b6 +F src/sqlite.h.in 78ac857a20a16d03b343320973b7da1736ff8fe9 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 76b4d4f0b9200e58b0a2d5339e2d46b4c7c7c691 +F src/sqliteInt.h 01c7555ded7724ef808b27ccbf9af19da526e0a9 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 7ada527ce7916adcabad7997c10f09680aac7c9b -F src/test1.c 2401eee14a4309a7cfe2aeb2f30ad517a1d9c299 -F src/test2.c 7355101c085304b90024f2261e056cdff13c6c35 +F src/test1.c 899bddeb0c7fb2b8062de6f03af4a4e4f48f9df5 +F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df F src/test5.c 5a34feec76d9b3a86aab30fd4f6cc9c48cbab4c1 @@ -255,7 +257,7 @@ F src/test_async.c 21e11293a2f72080eda70e1124e9102044531cd8 F src/test_autoext.c dea8a01a7153b9adc97bd26161e4226329546e12 F src/test_backup.c 3875e899222b651e18b662f86e0e50daa946344e F src/test_btree.c 2e9978eca99a9a4bfa8cae949efb00886860a64f -F src/test_config.c b131030783f4328beb7008dbfe7392c7f086abc7 +F src/test_config.c f3865829c8add75c49b3cb53f0494650b9292138 F src/test_demovfs.c 69b2085076654ebc18014cbc6386f04409c959a9 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c ced436e3d4b8e4681328409b8081051ce614e28f @@ -271,10 +273,10 @@ F src/test_multiplex.c 9f304bf04170c91c0318238d512df2da039eb1c8 F src/test_multiplex.h 110a8c4d356e0aa464ca8730375608a9a0b61ae1 F src/test_mutex.c 293042d623ebba969160f471a82aa1551626454f F src/test_onefile.c 0396f220561f3b4eedc450cef26d40c593c69a25 -F src/test_osinst.c 90a845c8183013d80eccb1f29e8805608516edba +F src/test_osinst.c 3d0340bc31a9f3d8a3547e0272373e80f78dde25 F src/test_pcache.c a5cd24730cb43c5b18629043314548c9169abb00 -F src/test_quota.c 30c64f0ef84734f2231a686df41ed882b0c59bc0 -F src/test_quota.h 8761e463b25e75ebc078bd67d70e39b9c817a0cb +F src/test_quota.c 65f6348fec0f2b3020c907247fb47556b214abb9 +F src/test_quota.h 2a8ad1952d1d2ca9af0ce0465e56e6c023b5e15d F src/test_rtree.c fdd8d29ca5165c7857987a2ba263fac5c69e231f F src/test_schema.c cd12a2223c3a394f4d07bb93bdf6d344c5c121b6 F src/test_server.c a2615049954cbb9cfb4a62e18e2f0616e4dc38fe @@ -284,16 +286,16 @@ F src/test_superlock.c 2b97936ca127d13962c3605dbc9a4ef269c424cd F src/test_syscall.c 2e21ca7f7dc54a028f1967b63f1e76155c356f9b F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb -F src/test_vfs.c e72f555ef7a59080f898fcf1a233deb9eb704ea9 +F src/test_vfs.c f84075a388527892ff184988f43b69ce69b8083c F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/tokenize.c 6da2de6e12218ccb0aea5184b56727d011f4bee7 F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb F src/update.c 7bb549d61efc6853f5cc708c1de6931179f8a12d F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c -F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf +F src/util.c 049fe1d3c0e2209c1bee107aec2fcff6285f909f F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c d6038559beb49c57f73a79412821d7143fc276ee +F src/vdbe.c 9811665adcf4ad9ad67cf1116e3e6adbe901ece3 F src/vdbe.h d03fcf47890ae1c79a335ca994cb878b302697ca F src/vdbeInt.h c05d4572211384c560b7579e7a299248cf31d6bd F src/vdbeapi.c d3c662762b62e330a03f29de8e2ac7098ef78030 @@ -306,8 +308,8 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 6ae02f1e8b1b29744d9e8cd9b95eac4c5232736d -F src/whereInt.h 929c1349b5355fd44f22cee5c14d72b3329c58a6 +F src/where.c fd4b525cd5ab652cea2fbc71ac15c975271ca461 +F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -320,13 +322,13 @@ F test/alter4.test d6c011fa0d6227abba762498cafbb607c9609e93 F test/altermalloc.test e81ac9657ed25c6c5bb09bebfa5a047cd8e4acfc F test/amatch1.test b5ae7065f042b7f4c1c922933f4700add50cdb9f F test/analyze.test 1772936d66471c65221e437b6d1999c3a03166c4 -F test/analyze3.test 412f690dfe95b337475e3e78a84a85d25f6f125d +F test/analyze3.test bf41f0f680dd1e0d44eed5e769531e93a5320275 F test/analyze4.test eff2df19b8dd84529966420f29ea52edc6b56213 F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4 F test/analyze6.test d31defa011a561b938b4608d3538c1b4e0b5e92c F test/analyze7.test bb1409afc9e8629e414387ef048b8e0e3e0bdc4f F test/analyze8.test 093d15c1c888eed5034304a98c992f7360130b88 -F test/analyze9.test e072a5172d55afcba98d6ca6a219ce8878c2f5c9 +F test/analyze9.test 623e02a99a78fa12fe5def2fd559032d5d887e0f F test/analyzeA.test 1a5c40079894847976d983ca39c707aaa44b6944 F test/analyzeB.test 8bf35ee0a548aea831bf56762cb8e7fdb1db083d F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b @@ -344,11 +346,11 @@ F test/auth.test 5bdf154eb28c0e4bbc0473f335858c0d96171768 F test/auth2.test c3b415b76c033bedb81292118fb7c01f5f10cbcd F test/auth3.test a4755e6a2a2fea547ffe63c874eb569e60a28eb5 F test/autoinc.test c58912526998a39e11f66b533e23cfabea7f25b7 -F test/autoindex1.test d4dfe14001dfcb74cfbd7107f45a79fc1ab6183e +F test/autoindex1.test 762ff3f8e25d852aae55c6462ca166a80c0cde61 F test/autovacuum.test 941892505d2c0f410a0cb5970dfa1c7c4e5f6e74 F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4 F test/avtrans.test 0252654f4295ddda3b2cce0e894812259e655a85 -F test/backcompat.test 5f8ad58b3eaebc78cd2c66c65476a42e6f32b2ad +F test/backcompat.test 19a1f337c68419b020a7481dd272a472c4ad8ef4 F test/backup.test c9cdd23a495864b9edf75a9fa66f5cb7e10fcf62 F test/backup2.test 34986ef926ea522911a51dfdb2f8e99b7b75ebcf F test/backup4.test 2a2e4a64388090b152de753fd9e123f28f6a3bd4 @@ -421,6 +423,7 @@ F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804 F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb F test/corruptI.test b3e4203d420490fc3d3062711597bc1dea06a789 +F test/cost.test 04842adec34311d70c3f3c5fd698b22be54138fd F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -467,7 +470,7 @@ F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea F test/enc2.test 83437a79ba1545a55fb549309175c683fb334473 F test/enc3.test 90683ad0e6ea587b9d5542ca93568af9a9858c40 F test/enc4.test c8f1ce3618508fd0909945beb8b8831feef2c020 -F test/eqp.test 57c6c604c2807fb5531731c5323133453c24afac +F test/eqp.test 90b56d03a93a2e7bb90f88be6083a8ea53f11a0e F test/errmsg.test f31592a594b44ee121371d25ddd5d63497bb3401 F test/eval.test bc269c365ba877554948441e91ad5373f9f91be3 F test/exclusive.test c7ebbc756eacf544c108b15eed64d7d4e5f86b75 @@ -550,15 +553,16 @@ F test/fts3conf.test ee8500c86dd58ec075e8831a1e216a79989436de F test/fts3corrupt.test 2710b77983cc7789295ddbffea52c1d3b7506dbb F test/fts3corrupt2.test 6d96efae2f8a6af3eeaf283aba437e6d0e5447ba F test/fts3cov.test e0fb00d8b715ddae4a94c305992dfc3ef70353d7 -F test/fts3d.test 597b0b76e41f0d672e2731c4d7b631d628efd13f +F test/fts3d.test 95c17d1b67b33a5eac0bf5a0d11116a0c0ac7a3a F test/fts3defer.test 0be4440b73a2e651fc1e472066686d6ada4b9963 -F test/fts3defer2.test a3b6cbeabaf28c9398652a4d101ea224d9358479 +F test/fts3defer2.test e880e3b65bdf999f4746cdaefa65f14a98b9b724 F test/fts3defer3.test dd53fc13223c6d8264a98244e9b19abd35ed71cd F test/fts3drop.test 1b906e293d6773812587b3dc458cb9e8f3f0c297 F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851 -F test/fts3expr.test 06f1a96facc8f3e4b1ad5cc001dc5c8e83e68b9f +F test/fts3expr.test 3401d47b229c4504424caf362cc4ff704cad4162 F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a F test/fts3expr3.test 9e91b8edbcb197bf2e92161aa7696446d96dce5f +F test/fts3expr4.test 0713d94ab951ed88a8c3629a4889a48c55c4067c F test/fts3fault.test cb72dccb0a3b9f730f16c5240f3fcb9303eb1660 F test/fts3fault2.test 3198eef2804deea7cac8403e771d9cbcb752d887 F test/fts3first.test dbdedd20914c8d539aa3206c9b34a23775644641 @@ -580,22 +584,24 @@ F test/fts4aa.test 0c3152322c7f0b548cc942ad763eaba0da87ccca F test/fts4check.test 66fa274cab2b615f2fb338b257713aba8fad88a8 F test/fts4content.test 2e7252557d6d24afa101d9ba1de710d6140e6d06 F test/fts4docid.test e33c383cfbdff0284685604d256f347a18fdbf01 +F test/fts4growth.test df10fde9f47cf5c71861e63fd8efcd573c4f7e53 +F test/fts4growth2.test 2f063be1902a73cd087355837c52fed42ac11a5d F test/fts4incr.test 361960ed3550e781f3f313e17e2182ef9cefc0e9 F test/fts4langid.test 24a6e41063b416bbdf371ff6b4476fa41c194aa7 F test/fts4merge.test c424309743fdd203f8e56a1f1cd7872cd66cc0ee F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 F test/fts4merge3.test aab02a09f50fe6baaddc2e159c3eabc116d45fc7 -F test/fts4merge4.test c19c85ca1faa7b6d536832b49c12e1867235f584 +F test/fts4merge4.test d895b1057a7798b67e03455d0fa50e9ea836c47b F test/fts4noti.test aed33ba44808852dcb24bf70fa132e7bf530f057 F test/fts4unicode.test 01ec3fe2a7c3cfff3b4c0581b83caa11b33efa36 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d -F test/func.test c2cbfc23d554c5bf8678d0fb271aa4f8ef94839c +F test/func.test ae97561957aba6ca9e3a7b8a13aac41830d701ef F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test dbccee9133cfef1473c59ec07b5f0262b9d72f9a F test/func4.test 6beacdfcb0e18c358e6c2dcacf1b65d1fa80955f F test/func5.test cdd224400bc3e48d891827cc913a57051a426fa4 F test/fuzz-oss1.test 4912e528ec9cf2f42134456933659d371c9e0d74 -F test/fuzz.test 77fd50afc12847af50fcf1941679d90adebadde6 +F test/fuzz.test 96083052bf5765e4518c1ba686ce2bab785670d1 F test/fuzz2.test 207d0f9d06db3eaf47a6b7bfc835b8e2fc397167 F test/fuzz3.test efd384b896c647b61a2c1848ba70d42aad60a7b3 F test/fuzz_common.tcl a87dfbb88c2a6b08a38e9a070dabd129e617b45b @@ -625,7 +631,7 @@ F test/index2.test ee83c6b5e3173a3d7137140d945d9a5d4fdfb9d6 F test/index3.test 55a90cff99834305e8141df7afaef39674b57062 F test/index4.test ab92e736d5946840236cd61ac3191f91a7856bf6 F test/index5.test fc07c14193c0430814e7a08b5da46888ee795c33 -F test/index6.test a0a2d286ffa6d35813f5003fdb7be124825b4422 +F test/index6.test fb370966ac3cd0989053dd5385757b5c3e24ab6a F test/index7.test a3baf9a625bda7fd49471e99aeae04095fbfeecf F test/indexedby.test b2f22f3e693a53813aa3f50b812eb609ba6df1ec F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d @@ -708,7 +714,7 @@ F test/minmax.test 42fbad0e81afaa6e0de41c960329f2b2c3526efd F test/minmax2.test b44bae787fc7b227597b01b0ca5575c7cb54d3bc F test/minmax3.test cc1e8b010136db0d01a6f2a29ba5a9f321034354 F test/minmax4.test 536a3360470633a177e42fbc19660d146b51daef -F test/misc1.test 441a0fafc7087f841db09fbfca54e7aea9f5a84c +F test/misc1.test 1201a037c24f982cc0e956cdaa34fcaf6439c417 F test/misc2.test 00d7de54eda90e237fc9a38b9e5ccc769ebf6d4d F test/misc3.test cf3dda47d5dda3e53fc5804a100d3c82be736c9d F test/misc4.test 9c078510fbfff05a9869a0b6d8b86a623ad2c4f6 @@ -726,6 +732,7 @@ F test/multiplex3.test d228f59eac91839a977eac19f21d053f03e4d101 F test/mutex1.test 78b2b9bb320e51d156c4efdb71b99b051e7a4b41 F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660 F test/nan.test e9648b9d007c7045242af35e11a984d4b169443a +F test/nolock.test 0540dd96f39b8876e3ffdd8814fad0ea425efeee F test/notify1.test 669b2b743618efdc18ca4b02f45423d5d2304abf F test/notify2.test ce23eb522c9e1fff6443f96376fe67872202061c F test/notify3.test 10ff25cde502e72a92053a2f215d64bece4ef934 @@ -737,7 +744,7 @@ F test/orderby1.test 9b524aff9147288da43a6d7ddfdcff47fa2303c6 F test/orderby2.test bc11009f7cd99d96b1b11e57b199b00633eb5b04 F test/orderby3.test 8619d06a3debdcd80a27c0fdea5c40b468854b99 F test/orderby4.test 4d39bfbaaa3ae64d026ca2ff166353d2edca4ba4 -F test/orderby5.test 2490183fef54417209d1df253633a605d46bd350 +F test/orderby5.test 8f08a54836d21fb7c70245360751aedd1c2286fb F test/orderby6.test 8b38138ab0972588240b3fca0985d2e400432859 F test/orderby7.test 3d1383d52ade5b9eb3a173b3147fdd296f0202da F test/oserror.test 50417780d0e0d7cd23cf12a8277bb44024765df3 @@ -753,7 +760,7 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54 -F test/permutations.test b7b8b410e3d22e616ff473e588078da6a377c082 +F test/permutations.test f3617ac9b55f63c031c13f5fa9e57116fc597165 F test/pragma.test adb21a90875bc54a880fa939c4d7c46598905aa0 F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -802,7 +809,7 @@ F test/select6.test e76bd10a56988f15726c097a5d5a7966fe82d3b2 F test/select7.test 7fd2ef598cfabb6b9ff6ac13973b91d0527df49d F test/select8.test 391de11bdd52339c30580dabbbbe97e3e9a3c79d F test/select9.test aebc2bb0c3bc44606125033cbcaac2c8d1f33a95 -F test/selectA.test 77adaffe9704cb80e301ebaeff4b107b58d435c5 +F test/selectA.test 64b88a80271c1710966e50e633380696b60a12a4 F test/selectB.test 954e4e49cf1f896d61794e440669e03a27ceea25 F test/selectC.test 871fb55d884d3de5943c4057ebd22c2459e71977 F test/selectD.test b0f02a04ef7737decb24e08be2c39b9664b43394 @@ -831,7 +838,7 @@ F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5 F test/shrink.test 8c70f62b6e8eb4d54533de6d65bd06b1b9a17868 F test/sidedelete.test f0ad71abe6233e3b153100f3b8d679b19a488329 F test/skipscan1.test bed8cbe9d554c8c27afb6c88500f704c86a9196f -F test/skipscan2.test 5a4db0799c338ddbacb154aaa5589c0254b36a8d +F test/skipscan2.test d77f79cdbba25f0f6f35298136cff21a7d7a553a F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test 0e4456e729e5a92a625907c63dcdedfbe72c5dc5 @@ -1016,7 +1023,7 @@ F test/tkt4018.test 7c2c9ba4df489c676a0a7a0e809a1fb9b2185bd1 F test/tokenize.test ce430a7aed48fc98301611429595883fdfcab5d7 F test/tpch01.test 8f4ac52f62f3e9f6bce0889105aecdf0275e331b F test/trace.test 4b36a41a3e9c7842151af6da5998f5080cdad9e5 -F test/trace2.test e7a988fdd982cdec62f1f1f34b0360e6476d01a0 +F test/trace2.test 93b47ca6996c66b47f57224cfb146f34e07df382 F test/trans.test 6e1b4c6a42dba31bd65f8fa5e61a2708e08ddde6 F test/trans2.test 62bd045bfc7a1c14c5ba83ba64d21ade31583f76 F test/trans3.test 373ac5183cc56be69f48ae44090e7f672939f732 @@ -1041,7 +1048,7 @@ F test/types2.test 3555aacf8ed8dc883356e59efc314707e6247a84 F test/types3.test 99e009491a54f4dc02c06bdbc0c5eea56ae3e25a F test/unique.test 93f8b2ef5ea51b9495f8d6493429b1fd0f465264 F test/unixexcl.test cd6c765f75e50e8e2c2ba763149e5d340ea19825 -F test/unordered.test ef85ac8f2f3c93ed2b9e811b684de73175fc464c +F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8 F test/update.test 1b6c488a8f993d090b7ee9ad0e234faa161b3aeb F test/uri.test 23662b7b61958b0f0e47082de7d06341ccf85d5b F test/utf16align.test 54cd35a27c005a9b6e7815d887718780b6a462ae @@ -1071,7 +1078,7 @@ F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test ea8778d5b0df200adef2ca7c00c3c37d4375f772 F test/wal.test 885f32b2b390b30b4aa3dbb0e568f8f78d40f5cc -F test/wal2.test a8e3963abf6b232cf0b852b09b53665ef34007af +F test/wal2.test 1f841d2048080d32f552942e333fd99ce541dada F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c F test/wal5.test 8f888b50f66b78821e61ed0e233ded5de378224b @@ -1098,7 +1105,7 @@ F test/walslow.test e7be6d9888f83aa5d3d3c7c08aa9b5c28b93609a F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e F test/where.test 28b64e93428961b07b0d486778d63fd672948f6b F test/where2.test 455a2eb2666e66c1e84e2cb5815173a85e6237db -F test/where3.test d28c51f257e60be30f74308fa385ceeddfb54a6e +F test/where3.test 1ad55ba900bd7747f98b6082e65bd3e442c5004e F test/where4.test d8420ceeb8323a41ceff1f1841fc528e824e1ecf F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2 F test/where6.test 5da5a98cec820d488e82708301b96cb8c18a258b @@ -1112,7 +1119,7 @@ F test/whereC.test d6f4ecd4fa2d9429681a5b22a25d2bda8e86ab8a F test/whereD.test fd9120e262f9da3c45940f52aefeef4d15b904e5 F test/whereE.test b3a055eef928c992b0a33198a7b8dc10eea5ad2f F test/whereF.test 5b2ba0dbe8074aa13e416b37c753991f0a2492d7 -F test/whereG.test 2533b72ed4a31fd1687230a499b557b911525344 +F test/whereG.test 0ac23e5e8311b69d87245f4a85112de321031658 F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c @@ -1130,7 +1137,7 @@ F test/without_rowid5.test b4a639a367f04d382d20e8f44fc1be4f2d57d107 F test/wordcount.c 9915e06cb33d8ca8109b8700791afe80d305afda F test/zeroblob.test caaecfb4f908f7bc086ed238668049f96774d688 F test/zerodamage.test cf6748bad89553cc1632be51a6f54e487e4039ac -F tool/build-all-msvc.bat e0917e787df675b020d250d60a00de8abaa4e30a x +F tool/build-all-msvc.bat a0534c971b86fe95f1983f445db5b896d3394818 x F tool/build-shell.sh 950f47c6174f1eea171319438b93ba67ff5bf367 F tool/checkSpacing.c 810e51703529a204fc4e1eb060e9ab663e3c06d2 F tool/diffdb.c 7524b1b5df217c20cd0431f6789851a4e0cb191b @@ -1142,17 +1149,17 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/lemon.c 07aba6270d5a5016ba8107b09e431eea4ecdc123 F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc -F tool/logest.c 388c318c7ac8b52b7c08ca1e2de0f4ca9a8f7e81 +F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 F tool/mkautoconfamal.sh f8d8dbf7d62f409ebed5134998bf5b51d7266383 F tool/mkkeywordhash.c c9e05e4a7bcab8fab9f583d5b321fb72f565ad97 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e F tool/mkpragmatab.tcl 78a77b2c554d534c6f2dc903130186ed15715460 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 -F tool/mksqlite3c-noext.tcl 8bce31074e4cbe631bb7676526a048335f4c9f02 -F tool/mksqlite3c.tcl 3a7297ac69e6da8089eeb3d3ba986e0e3d6016f4 +F tool/mksqlite3c-noext.tcl 1712d3d71256ca1f297046619c89e77a4d7c8f6d +F tool/mksqlite3c.tcl d2a3f219da0d86cea1e7e7101f819e187385ea1d F tool/mksqlite3h.tcl 2d0f1b3768f8d000b7881217d5fd4c776eb27467 -F tool/mksqlite3internalh.tcl 3dca7bb5374cee003379b8cbac73714f610ef795 -F tool/mkvsix.tcl 6477fb9dab838b7eea1eed50658ff1cda04850b5 +F tool/mksqlite3internalh.tcl b6514145a7d5321b47e64e19b8116cc44f973eb1 +F tool/mkvsix.tcl 52a4c613707ac34ae9c226e5ccc69cb948556105 F tool/offsets.c fe4262fdfa378e8f5499a42136d17bf3b98f6091 F tool/omittest.tcl 34d7ac01fe4fd18e3637f64abe12c40eca0f6b97 F tool/opcodeDoc.awk b3a2a3d5d3075b8bd90b7afe24283efdd586659c @@ -1179,8 +1186,8 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 -F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 3dca2809352c6c6d56db74447a814f77011c6220 af2cbe64adab5f9e3b0f3da00d06428088589d7f -R 9172deb3091895924031286a7ef97633 +F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f +P e158812c34b01ce516d26636a489509bf61d1c27 8180e320ee4090e41511836678e49a98c0b228e8 +R cb7779728ff23dc0c3ee0e1492ef0498 U drh -Z dedb5680561bda872be1d3df9c89cc1d +Z 89975b84354827e72d595bf54ff41778 diff --git a/manifest.uuid b/manifest.uuid index 0367d488c9..7d4a9bdbde 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e158812c34b01ce516d26636a489509bf61d1c27 \ No newline at end of file +2d33afe0c43cb99caa521c48b721c3b0971e967c \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 2a03e292c8..2952b364c3 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1371,6 +1371,7 @@ static void decodeIntArray( char *zIntArray, /* String containing int array to decode */ int nOut, /* Number of slots in aOut[] */ tRowcnt *aOut, /* Store integers here */ + LogEst *aLog, /* Or, if aOut==0, here */ Index *pIndex /* Handle extra flags for this index, if not NULL */ ){ char *z = zIntArray; @@ -1389,7 +1390,17 @@ static void decodeIntArray( v = v*10 + c - '0'; z++; } - aOut[i] = v; +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + if( aOut ){ + aOut[i] = v; + }else +#else + assert( aOut==0 ); + UNUSED_PARAMETER(aOut); +#endif + { + aLog[i] = sqlite3LogEst(v); + } if( *z==' ' ) z++; } #ifndef SQLITE_ENABLE_STAT3_OR_STAT4 @@ -1445,12 +1456,12 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ z = argv[2]; if( pIndex ){ - decodeIntArray((char*)z, pIndex->nKeyCol+1, pIndex->aiRowEst, pIndex); - if( pIndex->pPartIdxWhere==0 ) pTable->nRowEst = pIndex->aiRowEst[0]; + decodeIntArray((char*)z, pIndex->nKeyCol+1, 0, pIndex->aiRowLogEst, pIndex); + if( pIndex->pPartIdxWhere==0 ) pTable->nRowLogEst = pIndex->aiRowLogEst[0]; }else{ Index fakeIdx; fakeIdx.szIdxRow = pTable->szTabRow; - decodeIntArray((char*)z, 1, &pTable->nRowEst, &fakeIdx); + decodeIntArray((char*)z, 1, 0, &pTable->nRowLogEst, &fakeIdx); pTable->szTabRow = fakeIdx.szIdxRow; } @@ -1642,9 +1653,9 @@ static int loadStatTbl( pPrevIdx = pIdx; } pSample = &pIdx->aSample[pIdx->nSample]; - decodeIntArray((char*)sqlite3_column_text(pStmt,1), nCol, pSample->anEq, 0); - decodeIntArray((char*)sqlite3_column_text(pStmt,2), nCol, pSample->anLt, 0); - decodeIntArray((char*)sqlite3_column_text(pStmt,3), nCol, pSample->anDLt,0); + decodeIntArray((char*)sqlite3_column_text(pStmt,1),nCol,pSample->anEq,0,0); + decodeIntArray((char*)sqlite3_column_text(pStmt,2),nCol,pSample->anLt,0,0); + decodeIntArray((char*)sqlite3_column_text(pStmt,3),nCol,pSample->anDLt,0,0); /* Take a copy of the sample. Add two 0x00 bytes the end of the buffer. ** This is in case the sample record is corrupted. In that case, the diff --git a/src/build.c b/src/build.c index 4d4155ba90..a79abed3b6 100644 --- a/src/build.c +++ b/src/build.c @@ -905,7 +905,7 @@ void sqlite3StartTable( pTable->iPKey = -1; pTable->pSchema = db->aDb[iDb].pSchema; pTable->nRef = 1; - pTable->nRowEst = 1048576; + pTable->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); assert( pParse->pNewTable==0 ); pParse->pNewTable = pTable; @@ -2730,15 +2730,15 @@ Index *sqlite3AllocateIndexObject( nByte = ROUND8(sizeof(Index)) + /* Index structure */ ROUND8(sizeof(char*)*nCol) + /* Index.azColl */ - ROUND8(sizeof(tRowcnt)*(nCol+1) + /* Index.aiRowEst */ + ROUND8(sizeof(LogEst)*(nCol+1) + /* Index.aiRowLogEst */ sizeof(i16)*nCol + /* Index.aiColumn */ sizeof(u8)*nCol); /* Index.aSortOrder */ p = sqlite3DbMallocZero(db, nByte + nExtra); if( p ){ char *pExtra = ((char*)p)+ROUND8(sizeof(Index)); - p->azColl = (char**)pExtra; pExtra += ROUND8(sizeof(char*)*nCol); - p->aiRowEst = (tRowcnt*)pExtra; pExtra += sizeof(tRowcnt)*(nCol+1); - p->aiColumn = (i16*)pExtra; pExtra += sizeof(i16)*nCol; + p->azColl = (char**)pExtra; pExtra += ROUND8(sizeof(char*)*nCol); + p->aiRowLogEst = (LogEst*)pExtra; pExtra += sizeof(LogEst)*(nCol+1); + p->aiColumn = (i16*)pExtra; pExtra += sizeof(i16)*nCol; p->aSortOrder = (u8*)pExtra; p->nColumn = nCol; p->nKeyCol = nCol - 1; @@ -2968,7 +2968,7 @@ Index *sqlite3CreateIndex( if( db->mallocFailed ){ goto exit_create_index; } - assert( EIGHT_BYTE_ALIGNMENT(pIndex->aiRowEst) ); + assert( EIGHT_BYTE_ALIGNMENT(pIndex->aiRowLogEst) ); assert( EIGHT_BYTE_ALIGNMENT(pIndex->azColl) ); pIndex->zName = zExtra; zExtra += nName + 1; @@ -3249,7 +3249,7 @@ exit_create_index: ** Since we do not know, guess 1 million. aiRowEst[1] is an estimate of the ** number of rows in the table that match any particular value of the ** first column of the index. aiRowEst[2] is an estimate of the number -** of rows that match any particular combiniation of the first 2 columns +** of rows that match any particular combination of the first 2 columns ** of the index. And so forth. It must always be the case that * ** aiRowEst[N]<=aiRowEst[N-1] @@ -3260,20 +3260,27 @@ exit_create_index: ** are based on typical values found in actual indices. */ void sqlite3DefaultRowEst(Index *pIdx){ - tRowcnt *a = pIdx->aiRowEst; + /* 10, 9, 8, 7, 6 */ + LogEst aVal[] = { 33, 32, 30, 28, 26 }; + LogEst *a = pIdx->aiRowLogEst; + int nCopy = MIN(ArraySize(aVal), pIdx->nKeyCol); int i; - tRowcnt n; - assert( a!=0 ); - a[0] = pIdx->pTable->nRowEst; - if( a[0]<10 ) a[0] = 10; - n = 10; - for(i=1; i<=pIdx->nKeyCol; i++){ - a[i] = n; - if( n>5 ) n--; - } - if( pIdx->onError!=OE_None ){ - a[pIdx->nKeyCol] = 1; + + /* Set the first entry (number of rows in the index) to the estimated + ** number of rows in the table. Or 10, if the estimated number of rows + ** in the table is less than that. */ + a[0] = pIdx->pTable->nRowLogEst; + if( a[0]<33 ) a[0] = 33; assert( 33==sqlite3LogEst(10) ); + + /* Estimate that a[1] is 10, a[2] is 9, a[3] is 8, a[4] is 7, a[5] is + ** 6 and each subsequent value (if any) is 5. */ + memcpy(&a[1], aVal, nCopy*sizeof(LogEst)); + for(i=nCopy+1; i<=pIdx->nKeyCol; i++){ + a[i] = 23; assert( 23==sqlite3LogEst(5) ); } + + assert( 0==sqlite3LogEst(1) ); + if( pIdx->onError!=OE_None ) a[pIdx->nKeyCol] = 0; } /* diff --git a/src/func.c b/src/func.c index 6be963580c..bc5c78981e 100644 --- a/src/func.c +++ b/src/func.c @@ -1541,7 +1541,7 @@ static void groupConcatStep( } zVal = (char*)sqlite3_value_text(argv[0]); nVal = sqlite3_value_bytes(argv[0]); - if( nVal ) sqlite3StrAccumAppend(pAccum, zVal, nVal); + if( zVal ) sqlite3StrAccumAppend(pAccum, zVal, nVal); } } static void groupConcatFinalize(sqlite3_context *context){ diff --git a/src/global.c b/src/global.c index 1ee3f6436f..2c14b58abd 100644 --- a/src/global.c +++ b/src/global.c @@ -173,15 +173,22 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { 0, /* isMutexInit */ 0, /* isMallocInit */ 0, /* isPCacheInit */ - 0, /* pInitMutex */ 0, /* nRefInitMutex */ + 0, /* pInitMutex */ 0, /* xLog */ 0, /* pLogArg */ - 0, /* bLocaltimeFault */ #ifdef SQLITE_ENABLE_SQLLOG 0, /* xSqllog */ - 0 /* pSqllogArg */ + 0, /* pSqllogArg */ #endif +#ifdef SQLITE_VDBE_COVERAGE + 0, /* xVdbeBranch */ + 0, /* pVbeBranchArg */ +#endif +#ifndef SQLITE_OMIT_BUILTIN_TEST + 0, /* xTestCallback */ +#endif + 0 /* bLocaltimeFault */ }; /* diff --git a/src/main.c b/src/main.c index 65fce475c0..eb90569d30 100644 --- a/src/main.c +++ b/src/main.c @@ -3135,6 +3135,28 @@ int sqlite3_test_control(int op, ...){ break; } + /* + ** sqlite3_test_control(FAULT_INSTALL, xCallback) + ** + ** Arrange to invoke xCallback() whenever sqlite3FaultSim() is called, + ** if xCallback is not NULL. + ** + ** As a test of the fault simulator mechanism itself, sqlite3FaultSim(0) + ** is called immediately after installing the new callback and the return + ** value from sqlite3FaultSim(0) becomes the return from + ** sqlite3_test_control(). + */ + case SQLITE_TESTCTRL_FAULT_INSTALL: { + /* MSVC is picky about pulling func ptrs from va lists. + ** http://support.microsoft.com/kb/47961 + ** sqlite3Config.xTestCallback = va_arg(ap, int(*)(int)); + */ + typedef int(*TESTCALLBACKFUNC_t)(int); + sqlite3Config.xTestCallback = va_arg(ap, TESTCALLBACKFUNC_t); + rc = sqlite3FaultSim(0); + break; + } + /* ** sqlite3_test_control(BENIGN_MALLOC_HOOKS, xBegin, xEnd) ** diff --git a/src/mutex_w32.c b/src/mutex_w32.c index 4b88c17452..f43a152389 100644 --- a/src/mutex_w32.c +++ b/src/mutex_w32.c @@ -13,6 +13,13 @@ */ #include "sqliteInt.h" +#if SQLITE_OS_WIN +/* +** Include the header file for the Windows VFS. +*/ +#include "os_win.h" +#endif + /* ** The code in this file is only used if we are compiling multithreaded ** on a win32 system. diff --git a/src/os.h b/src/os.h index 070a2ddd17..3920a62ee3 100644 --- a/src/os.h +++ b/src/os.h @@ -21,83 +21,10 @@ #define _SQLITE_OS_H_ /* -** Figure out if we are dealing with Unix, Windows, or some other -** operating system. After the following block of preprocess macros, -** all of SQLITE_OS_UNIX, SQLITE_OS_WIN, and SQLITE_OS_OTHER -** will defined to either 1 or 0. One of the four will be 1. The other -** three will be 0. +** Attempt to automatically detect the operating system and setup the +** necessary pre-processor macros for it. */ -#if defined(SQLITE_OS_OTHER) -# if SQLITE_OS_OTHER==1 -# undef SQLITE_OS_UNIX -# define SQLITE_OS_UNIX 0 -# undef SQLITE_OS_WIN -# define SQLITE_OS_WIN 0 -# else -# undef SQLITE_OS_OTHER -# endif -#endif -#if !defined(SQLITE_OS_UNIX) && !defined(SQLITE_OS_OTHER) -# define SQLITE_OS_OTHER 0 -# ifndef SQLITE_OS_WIN -# if defined(_WIN32) || defined(WIN32) || defined(__CYGWIN__) || defined(__MINGW32__) || defined(__BORLANDC__) -# define SQLITE_OS_WIN 1 -# define SQLITE_OS_UNIX 0 -# else -# define SQLITE_OS_WIN 0 -# define SQLITE_OS_UNIX 1 -# endif -# else -# define SQLITE_OS_UNIX 0 -# endif -#else -# ifndef SQLITE_OS_WIN -# define SQLITE_OS_WIN 0 -# endif -#endif - -#if SQLITE_OS_WIN -# include -#endif - -/* -** Determine if we are dealing with Windows NT. -** -** We ought to be able to determine if we are compiling for win98 or winNT -** using the _WIN32_WINNT macro as follows: -** -** #if defined(_WIN32_WINNT) -** # define SQLITE_OS_WINNT 1 -** #else -** # define SQLITE_OS_WINNT 0 -** #endif -** -** However, vs2005 does not set _WIN32_WINNT by default, as it ought to, -** so the above test does not work. We'll just assume that everything is -** winNT unless the programmer explicitly says otherwise by setting -** SQLITE_OS_WINNT to 0. -*/ -#if SQLITE_OS_WIN && !defined(SQLITE_OS_WINNT) -# define SQLITE_OS_WINNT 1 -#endif - -/* -** Determine if we are dealing with WindowsCE - which has a much -** reduced API. -*/ -#if defined(_WIN32_WCE) -# define SQLITE_OS_WINCE 1 -#else -# define SQLITE_OS_WINCE 0 -#endif - -/* -** Determine if we are dealing with WinRT, which provides only a subset of -** the full Win32 API. -*/ -#if !defined(SQLITE_OS_WINRT) -# define SQLITE_OS_WINRT 0 -#endif +#include "os_setup.h" /* If the SET_FULLSYNC macro is not defined above, then make it ** a no-op diff --git a/src/os_setup.h b/src/os_setup.h new file mode 100644 index 0000000000..68de1446ed --- /dev/null +++ b/src/os_setup.h @@ -0,0 +1,57 @@ +/* +** 2013 November 25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains pre-processor directives related to operating system +** detection and/or setup. +*/ +#ifndef _OS_SETUP_H_ +#define _OS_SETUP_H_ + +/* +** Figure out if we are dealing with Unix, Windows, or some other operating +** system. +** +** After the following block of preprocess macros, all of SQLITE_OS_UNIX, +** SQLITE_OS_WIN, and SQLITE_OS_OTHER will defined to either 1 or 0. One of +** the three will be 1. The other two will be 0. +*/ +#if defined(SQLITE_OS_OTHER) +# if SQLITE_OS_OTHER==1 +# undef SQLITE_OS_UNIX +# define SQLITE_OS_UNIX 0 +# undef SQLITE_OS_WIN +# define SQLITE_OS_WIN 0 +# else +# undef SQLITE_OS_OTHER +# endif +#endif +#if !defined(SQLITE_OS_UNIX) && !defined(SQLITE_OS_OTHER) +# define SQLITE_OS_OTHER 0 +# ifndef SQLITE_OS_WIN +# if defined(_WIN32) || defined(WIN32) || defined(__CYGWIN__) || \ + defined(__MINGW32__) || defined(__BORLANDC__) +# define SQLITE_OS_WIN 1 +# define SQLITE_OS_UNIX 0 +# else +# define SQLITE_OS_WIN 0 +# define SQLITE_OS_UNIX 1 +# endif +# else +# define SQLITE_OS_UNIX 0 +# endif +#else +# ifndef SQLITE_OS_WIN +# define SQLITE_OS_WIN 0 +# endif +#endif + +#endif /* _OS_SETUP_H_ */ diff --git a/src/os_win.c b/src/os_win.c index 287dad3b57..fcac665dff 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -15,16 +15,16 @@ #include "sqliteInt.h" #if SQLITE_OS_WIN /* This file is used for Windows only */ -#ifdef __CYGWIN__ -# include -# include /* amalgamator: keep */ -#endif - /* ** Include code that is common to all os_*.c files */ #include "os_common.h" +/* +** Include the header file for the Windows VFS. +*/ +#include "os_win.h" + /* ** Compiling and using WAL mode requires several APIs that are only ** available in Windows platforms based on the NT kernel. @@ -1836,6 +1836,32 @@ static int winLogErrorAtLine( static int winIoerrRetry = SQLITE_WIN32_IOERR_RETRY; static int winIoerrRetryDelay = SQLITE_WIN32_IOERR_RETRY_DELAY; +/* +** The "winIoerrCanRetry1" macro is used to determine if a particular I/O +** error code obtained via GetLastError() is eligible to be retried. It +** must accept the error code DWORD as its only argument and should return +** non-zero if the error code is transient in nature and the operation +** responsible for generating the original error might succeed upon being +** retried. The argument to this macro should be a variable. +** +** Additionally, a macro named "winIoerrCanRetry2" may be defined. If it +** is defined, it will be consulted only when the macro "winIoerrCanRetry1" +** returns zero. The "winIoerrCanRetry2" macro is completely optional and +** may be used to include additional error codes in the set that should +** result in the failing I/O operation being retried by the caller. If +** defined, the "winIoerrCanRetry2" macro must exhibit external semantics +** identical to those of the "winIoerrCanRetry1" macro. +*/ +#if !defined(winIoerrCanRetry1) +#define winIoerrCanRetry1(a) (((a)==ERROR_ACCESS_DENIED) || \ + ((a)==ERROR_SHARING_VIOLATION) || \ + ((a)==ERROR_LOCK_VIOLATION) || \ + ((a)==ERROR_DEV_NOT_EXIST) || \ + ((a)==ERROR_NETNAME_DELETED) || \ + ((a)==ERROR_SEM_TIMEOUT) || \ + ((a)==ERROR_NETWORK_UNREACHABLE)) +#endif + /* ** If a ReadFile() or WriteFile() error occurs, invoke this routine ** to see if it should be retried. Return TRUE to retry. Return FALSE @@ -1849,13 +1875,18 @@ static int winRetryIoerr(int *pnRetry, DWORD *pError){ } return 0; } - if( e==ERROR_ACCESS_DENIED || - e==ERROR_LOCK_VIOLATION || - e==ERROR_SHARING_VIOLATION ){ + if( winIoerrCanRetry1(e) ){ sqlite3_win32_sleep(winIoerrRetryDelay*(1+*pnRetry)); ++*pnRetry; return 1; } +#if defined(winIoerrCanRetry2) + else if( winIoerrCanRetry2(e) ){ + sqlite3_win32_sleep(winIoerrRetryDelay*(1+*pnRetry)); + ++*pnRetry; + return 1; + } +#endif if( pError ){ *pError = e; } diff --git a/src/os_win.h b/src/os_win.h new file mode 100644 index 0000000000..d662cd467d --- /dev/null +++ b/src/os_win.h @@ -0,0 +1,67 @@ +/* +** 2013 November 25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code that is specific to Windows. +*/ +#ifndef _OS_WIN_H_ +#define _OS_WIN_H_ + +/* +** Include the primary Windows SDK header file. +*/ +#include "windows.h" + +#ifdef __CYGWIN__ +# include +# include /* amalgamator: dontcache */ +#endif + +/* +** Determine if we are dealing with Windows NT. +** +** We ought to be able to determine if we are compiling for Windows 9x or +** Windows NT using the _WIN32_WINNT macro as follows: +** +** #if defined(_WIN32_WINNT) +** # define SQLITE_OS_WINNT 1 +** #else +** # define SQLITE_OS_WINNT 0 +** #endif +** +** However, Visual Studio 2005 does not set _WIN32_WINNT by default, as +** it ought to, so the above test does not work. We'll just assume that +** everything is Windows NT unless the programmer explicitly says otherwise +** by setting SQLITE_OS_WINNT to 0. +*/ +#if SQLITE_OS_WIN && !defined(SQLITE_OS_WINNT) +# define SQLITE_OS_WINNT 1 +#endif + +/* +** Determine if we are dealing with Windows CE - which has a much reduced +** API. +*/ +#if defined(_WIN32_WCE) +# define SQLITE_OS_WINCE 1 +#else +# define SQLITE_OS_WINCE 0 +#endif + +/* +** Determine if we are dealing with WinRT, which provides only a subset of +** the full Win32 API. +*/ +#if !defined(SQLITE_OS_WINRT) +# define SQLITE_OS_WINRT 0 +#endif + +#endif /* _OS_WIN_H_ */ diff --git a/src/pager.c b/src/pager.c index b09d6cb930..a77dcecc4c 100644 --- a/src/pager.c +++ b/src/pager.c @@ -626,7 +626,8 @@ struct Pager { u8 ckptSyncFlags; /* SYNC_NORMAL or SYNC_FULL for checkpoint */ u8 walSyncFlags; /* SYNC_NORMAL or SYNC_FULL for wal writes */ u8 syncFlags; /* SYNC_NORMAL or SYNC_FULL otherwise */ - u8 tempFile; /* zFilename is a temporary file */ + u8 tempFile; /* zFilename is a temporary or immutable file */ + u8 noLock; /* Do not lock (except in WAL mode) */ u8 readOnly; /* True for a read-only database */ u8 memDb; /* True to inhibit all file I/O */ @@ -1091,7 +1092,7 @@ static int pagerUnlockDb(Pager *pPager, int eLock){ assert( eLock!=NO_LOCK || pagerUseWal(pPager)==0 ); if( isOpen(pPager->fd) ){ assert( pPager->eLock>=eLock ); - rc = sqlite3OsUnlock(pPager->fd, eLock); + rc = pPager->noLock ? SQLITE_OK : sqlite3OsUnlock(pPager->fd, eLock); if( pPager->eLock!=UNKNOWN_LOCK ){ pPager->eLock = (u8)eLock; } @@ -1115,7 +1116,7 @@ static int pagerLockDb(Pager *pPager, int eLock){ assert( eLock==SHARED_LOCK || eLock==RESERVED_LOCK || eLock==EXCLUSIVE_LOCK ); if( pPager->eLockeLock==UNKNOWN_LOCK ){ - rc = sqlite3OsLock(pPager->fd, eLock); + rc = pPager->noLock ? SQLITE_OK : sqlite3OsLock(pPager->fd, eLock); if( rc==SQLITE_OK && (pPager->eLock!=UNKNOWN_LOCK||eLock==EXCLUSIVE_LOCK) ){ pPager->eLock = (u8)eLock; IOTRACE(("LOCK %p %d\n", pPager, eLock)) @@ -4674,30 +4675,38 @@ int sqlite3PagerOpen( ** + The value returned by sqlite3OsSectorSize() ** + The largest page size that can be written atomically. */ - if( rc==SQLITE_OK && !readOnly ){ - setSectorSize(pPager); - assert(SQLITE_DEFAULT_PAGE_SIZE<=SQLITE_MAX_DEFAULT_PAGE_SIZE); - if( szPageDfltsectorSize ){ - if( pPager->sectorSize>SQLITE_MAX_DEFAULT_PAGE_SIZE ){ - szPageDflt = SQLITE_MAX_DEFAULT_PAGE_SIZE; - }else{ - szPageDflt = (u32)pPager->sectorSize; - } - } -#ifdef SQLITE_ENABLE_ATOMIC_WRITE - { - int iDc = sqlite3OsDeviceCharacteristics(pPager->fd); - int ii; - assert(SQLITE_IOCAP_ATOMIC512==(512>>8)); - assert(SQLITE_IOCAP_ATOMIC64K==(65536>>8)); - assert(SQLITE_MAX_DEFAULT_PAGE_SIZE<=65536); - for(ii=szPageDflt; ii<=SQLITE_MAX_DEFAULT_PAGE_SIZE; ii=ii*2){ - if( iDc&(SQLITE_IOCAP_ATOMIC|(ii>>8)) ){ - szPageDflt = ii; + if( rc==SQLITE_OK ){ + int iDc = sqlite3OsDeviceCharacteristics(pPager->fd); + if( !readOnly ){ + setSectorSize(pPager); + assert(SQLITE_DEFAULT_PAGE_SIZE<=SQLITE_MAX_DEFAULT_PAGE_SIZE); + if( szPageDfltsectorSize ){ + if( pPager->sectorSize>SQLITE_MAX_DEFAULT_PAGE_SIZE ){ + szPageDflt = SQLITE_MAX_DEFAULT_PAGE_SIZE; + }else{ + szPageDflt = (u32)pPager->sectorSize; + } + } +#ifdef SQLITE_ENABLE_ATOMIC_WRITE + { + int ii; + assert(SQLITE_IOCAP_ATOMIC512==(512>>8)); + assert(SQLITE_IOCAP_ATOMIC64K==(65536>>8)); + assert(SQLITE_MAX_DEFAULT_PAGE_SIZE<=65536); + for(ii=szPageDflt; ii<=SQLITE_MAX_DEFAULT_PAGE_SIZE; ii=ii*2){ + if( iDc&(SQLITE_IOCAP_ATOMIC|(ii>>8)) ){ + szPageDflt = ii; + } } } - } #endif + } + pPager->noLock = sqlite3_uri_boolean(zFilename, "nolock", 0); + if( (iDc & SQLITE_IOCAP_IMMUTABLE)!=0 + || sqlite3_uri_boolean(zFilename, "immutable", 0) ){ + vfsFlags |= SQLITE_OPEN_READONLY; + goto act_like_temp_file; + } } }else{ /* If a temporary file is requested, it is not opened immediately. @@ -4707,10 +4716,14 @@ int sqlite3PagerOpen( ** This branch is also run for an in-memory database. An in-memory ** database is the same as a temp-file that is never written out to ** disk and uses an in-memory rollback journal. + ** + ** This branch also runs for files marked as immutable. */ +act_like_temp_file: tempFile = 1; - pPager->eState = PAGER_READER; - pPager->eLock = EXCLUSIVE_LOCK; + pPager->eState = PAGER_READER; /* Pretend we already have a lock */ + pPager->eLock = EXCLUSIVE_LOCK; /* Pretend we are in EXCLUSIVE locking mode */ + pPager->noLock = 1; /* Do no locking */ readOnly = (vfsFlags&SQLITE_OPEN_READONLY); } @@ -4751,9 +4764,6 @@ int sqlite3PagerOpen( /* pPager->nPage = 0; */ pPager->mxPgno = SQLITE_MAX_PAGE_COUNT; /* pPager->state = PAGER_UNLOCK; */ -#if 0 - assert( pPager->state == (tempFile ? PAGER_EXCLUSIVE : PAGER_UNLOCK) ); -#endif /* pPager->errMask = 0; */ pPager->tempFile = (u8)tempFile; assert( tempFile==PAGER_LOCKINGMODE_NORMAL diff --git a/src/pragma.c b/src/pragma.c index 66d0c3328e..4c69ceb4fd 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1488,13 +1488,15 @@ void sqlite3Pragma( sqlite3VdbeAddOp2(v, OP_Null, 0, 2); sqlite3VdbeAddOp2(v, OP_Integer, (int)sqlite3LogEstToInt(pTab->szTabRow), 3); - sqlite3VdbeAddOp2(v, OP_Integer, (int)pTab->nRowEst, 4); + sqlite3VdbeAddOp2(v, OP_Integer, + (int)sqlite3LogEstToInt(pTab->nRowLogEst), 4); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pIdx->zName, 0); sqlite3VdbeAddOp2(v, OP_Integer, (int)sqlite3LogEstToInt(pIdx->szIdxRow), 3); - sqlite3VdbeAddOp2(v, OP_Integer, (int)pIdx->aiRowEst[0], 4); + sqlite3VdbeAddOp2(v, OP_Integer, + (int)sqlite3LogEstToInt(pIdx->aiRowLogEst[0]), 4); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); } } diff --git a/src/select.c b/src/select.c index 6efdde4b72..5fff010f34 100644 --- a/src/select.c +++ b/src/select.c @@ -466,15 +466,17 @@ static void pushOntoSorter( ){ Vdbe *v = pParse->pVdbe; int nExpr = pSort->pOrderBy->nExpr; - int regBase = sqlite3GetTempRange(pParse, nExpr+2); - int regRecord = sqlite3GetTempReg(pParse); + int regRecord = ++pParse->nMem; + int regBase = pParse->nMem+1; int nOBSat = pSort->nOBSat; int op; + + pParse->nMem += nExpr+2; /* nExpr+2 registers allocated at regBase */ sqlite3ExprCacheClear(pParse); sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, 0); sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+1, 1); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nExpr+2-nOBSat, regRecord); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nExpr+2-nOBSat,regRecord); if( nOBSat>0 ){ int regPrevKey; /* The first nOBSat columns of the previous row */ int addrFirst; /* Address of the OP_IfNot opcode */ @@ -511,10 +513,6 @@ static void pushOntoSorter( op = OP_IdxInsert; } sqlite3VdbeAddOp2(v, op, pSort->iECursor, regRecord); - if( nOBSat==0 ){ - sqlite3ReleaseTempReg(pParse, regRecord); - sqlite3ReleaseTempRange(pParse, regBase, nExpr+2); - } if( pSelect->iLimit ){ int addr1, addr2; int iLimit; @@ -1690,7 +1688,7 @@ Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){ assert( db->lookaside.bEnabled==0 ); pTab->nRef = 1; pTab->zName = 0; - pTab->nRowEst = 1048576; + pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); selectColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol); selectAddColumnTypeAndCollation(pParse, pTab, pSelect); pTab->iPKey = -1; @@ -3829,7 +3827,7 @@ static int withExpand( pTab->nRef = 1; pTab->zName = sqlite3DbStrDup(db, pCte->zName); pTab->iPKey = -1; - pTab->nRowEst = 1048576; + pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); pTab->tabFlags |= TF_Ephemeral; pFrom->pSelect = sqlite3SelectDup(db, pCte->pSelect, 0); if( db->mallocFailed ) return SQLITE_NOMEM; @@ -4005,7 +4003,7 @@ static int selectExpander(Walker *pWalker, Select *p){ while( pSel->pPrior ){ pSel = pSel->pPrior; } selectColumnsFromExprList(pParse, pSel->pEList, &pTab->nCol, &pTab->aCol); pTab->iPKey = -1; - pTab->nRowEst = 1048576; + pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); pTab->tabFlags |= TF_Ephemeral; #endif }else{ @@ -4655,7 +4653,7 @@ int sqlite3Select( sqlite3SelectDestInit(&dest, SRT_Coroutine, pItem->regReturn); explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); sqlite3Select(pParse, pSub, &dest); - pItem->pTab->nRowEst = (unsigned)pSub->nSelectRow; + pItem->pTab->nRowLogEst = sqlite3LogEst(pSub->nSelectRow); pItem->viaCoroutine = 1; pItem->regResult = dest.iSdst; sqlite3VdbeAddOp1(v, OP_EndCoroutine, pItem->regReturn); @@ -4686,7 +4684,7 @@ int sqlite3Select( sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor); explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); sqlite3Select(pParse, pSub, &dest); - pItem->pTab->nRowEst = (unsigned)pSub->nSelectRow; + pItem->pTab->nRowLogEst = sqlite3LogEst(pSub->nSelectRow); if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr); retAddr = sqlite3VdbeAddOp1(v, OP_Return, pItem->regReturn); VdbeComment((v, "end %s", pItem->pTab->zName)); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index ef78794408..2f2d2a96f3 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -555,7 +555,10 @@ int sqlite3_exec( ** file that were written at the application level might have changed ** and that adjacent bytes, even bytes within the same sector are ** guaranteed to be unchanged. The SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN -** flag indicate that a file cannot be deleted when open. +** flag indicate that a file cannot be deleted when open. The +** SQLITE_IOCAP_IMMUTABLE flag indicates that the file is on +** read-only media and cannot be changed even by processes with +** elevated privileges. */ #define SQLITE_IOCAP_ATOMIC 0x00000001 #define SQLITE_IOCAP_ATOMIC512 0x00000002 @@ -570,6 +573,7 @@ int sqlite3_exec( #define SQLITE_IOCAP_SEQUENTIAL 0x00000400 #define SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN 0x00000800 #define SQLITE_IOCAP_POWERSAFE_OVERWRITE 0x00001000 +#define SQLITE_IOCAP_IMMUTABLE 0x00002000 /* ** CAPI3REF: File Locking Levels @@ -2774,6 +2778,30 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** ^If sqlite3_open_v2() is used and the "cache" parameter is present in ** a URI filename, its value overrides any behavior requested by setting ** SQLITE_OPEN_PRIVATECACHE or SQLITE_OPEN_SHAREDCACHE flag. +** +**
  • psow: ^The psow parameter may be "true" (or "on" or "yes" or +** "1") or "false" (or "off" or "no" or "0") to indicate that the +** [powersafe overwrite] property does or does not apply to the +** storage media on which the database file resides. ^The psow query +** parameter only works for the built-in unix and Windows VFSes. +** +**
  • nolock: ^The nolock parameter is a boolean query parameter +** which if set disables file locking in rollback journal modes. This +** is useful for accessing a database on a filesystem that does not +** support locking. Caution: Database corruption might result if two +** or more processes write to the same database and any one of those +** processes uses nolock=1. +** +**
  • immutable: ^The immutable parameter is a boolean query +** parameter that indicates that the database file is stored on +** read-only media. ^When immutable is set, SQLite assumes that the +** database file cannot be changed, even by a process with higher +** privilege, and so the database is opened read-only and all locking +** and change detection is disabled. Caution: Setting the immutable +** property on a database file that does in fact change can result +** in incorrect query results and/or [SQLITE_CORRUPT] errors. +** See also: [SQLITE_IOCAP_IMMUTABLE]. +** ** ** ** ^Specifying an unknown parameter in the query component of a URI is not an @@ -2803,8 +2831,9 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** Open file "data.db" in the current directory for read-only access. ** Regardless of whether or not shared-cache mode is enabled by ** default, use a private cache. -** file:/home/fred/data.db?vfs=unix-nolock -** Open file "/home/fred/data.db". Use the special VFS "unix-nolock". +** file:/home/fred/data.db?vfs=unix-dotfile +** Open file "/home/fred/data.db". Use the special VFS "unix-dotfile" +** that uses dot-files in place of posix advisory locking. ** file:data.db?mode=readonly ** An error. "readonly" is not a valid option for the "mode" parameter. ** diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 3ab4281ebf..9b4215dd81 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -525,10 +525,10 @@ typedef INT8_TYPE i8; /* 1-byte signed integer */ ** gives a possible range of values of approximately 1.0e986 to 1e-986. ** But the allowed values are "grainy". Not every value is representable. ** For example, quantities 16 and 17 are both represented by a LogEst -** of 40. However, since LogEst quantatites are suppose to be estimates, +** of 40. However, since LogEst quantaties are suppose to be estimates, ** not exact values, this imprecision is not a problem. ** -** "LogEst" is short for "Logarithimic Estimate". +** "LogEst" is short for "Logarithmic Estimate". ** ** Examples: ** 1 -> 0 20 -> 43 10000 -> 132 @@ -1479,7 +1479,7 @@ struct Table { #ifndef SQLITE_OMIT_CHECK ExprList *pCheck; /* All CHECK constraints */ #endif - tRowcnt nRowEst; /* Estimated rows in table - from sqlite_stat1 table */ + LogEst nRowLogEst; /* Estimated rows in table - from sqlite_stat1 table */ int tnum; /* Root BTree node for this table (see note above) */ i16 iPKey; /* If not negative, use aCol[iPKey] as the primary key */ i16 nCol; /* Number of columns in this table */ @@ -1688,7 +1688,7 @@ struct UnpackedRecord { struct Index { char *zName; /* Name of this index */ i16 *aiColumn; /* Which columns are used by this index. 1st is 0 */ - tRowcnt *aiRowEst; /* From ANALYZE: Est. rows selected by each column */ + LogEst *aiRowLogEst; /* From ANALYZE: Est. rows selected by each column */ Table *pTable; /* The SQL table being indexed */ char *zColAff; /* String defining the affinity of each column */ Index *pNext; /* The next index associated with the same table */ @@ -2721,11 +2721,10 @@ struct Sqlite3Config { int isMutexInit; /* True after mutexes are initialized */ int isMallocInit; /* True after malloc is initialized */ int isPCacheInit; /* True after malloc is initialized */ - sqlite3_mutex *pInitMutex; /* Mutex used by sqlite3_initialize() */ int nRefInitMutex; /* Number of users of pInitMutex */ + sqlite3_mutex *pInitMutex; /* Mutex used by sqlite3_initialize() */ void (*xLog)(void*,int,const char*); /* Function for logging */ void *pLogArg; /* First argument to xLog() */ - int bLocaltimeFault; /* True to fail localtime() calls */ #ifdef SQLITE_ENABLE_SQLLOG void(*xSqllog)(void*,sqlite3*,const char*, int); void *pSqllogArg; @@ -2737,6 +2736,10 @@ struct Sqlite3Config { void (*xVdbeBranch)(void*,int iSrcLine,u8 eThis,u8 eMx); /* Callback */ void *pVdbeBranchArg; /* 1st argument */ #endif +#ifndef SQLITE_OMIT_BUILTIN_TEST + int (*xTestCallback)(int); /* Invoked by sqlite3FaultSim() */ +#endif + int bLocaltimeFault; /* True to fail localtime() calls */ }; /* @@ -3038,6 +3041,12 @@ int sqlite3ParseUri(const char*,const char*,unsigned int*, Btree *sqlite3DbNameToBtree(sqlite3*,const char*); int sqlite3CodeOnce(Parse *); +#ifdef SQLITE_OMIT_BUILTIN_TEST +# define sqlite3FaultSim(X) SQLITE_OK +#else + int sqlite3FaultSim(int); +#endif + Bitvec *sqlite3BitvecCreate(u32); int sqlite3BitvecTest(Bitvec*, u32); int sqlite3BitvecSet(Bitvec*, u32); diff --git a/src/test1.c b/src/test1.c index 3000288c7d..44f56a6c84 100644 --- a/src/test1.c +++ b/src/test1.c @@ -14,6 +14,10 @@ ** testing of the SQLite library. */ #include "sqliteInt.h" +#if SQLITE_OS_WIN +# include "os_win.h" +#endif + #include "vdbeInt.h" #include "tcl.h" #include diff --git a/src/test2.c b/src/test2.c index d130e9d01b..58f271ff27 100644 --- a/src/test2.c +++ b/src/test2.c @@ -568,7 +568,90 @@ static int testPendingByte( rc = sqlite3_test_control(SQLITE_TESTCTRL_PENDING_BYTE, pbyte); Tcl_SetObjResult(interp, Tcl_NewIntObj(rc)); return TCL_OK; -} +} + +/* +** The sqlite3FaultSim() callback: +*/ +static Tcl_Interp *faultSimInterp = 0; +static int faultSimScriptSize = 0; +static char *faultSimScript; +static int faultSimCallback(int x){ + char zInt[30]; + int i; + int isNeg; + int rc; + if( x==0 ){ + memcpy(faultSimScript+faultSimScriptSize, "0", 2); + }else{ + /* Convert x to text without using any sqlite3 routines */ + if( x<0 ){ + isNeg = 1; + x = -x; + }else{ + isNeg = 0; + } + zInt[sizeof(zInt)-1] = 0; + for(i=sizeof(zInt)-2; i>0 && x>0; i--, x /= 10){ + zInt[i] = (x%10) + '0'; + } + if( isNeg ) zInt[i--] = '-'; + memcpy(faultSimScript+faultSimScriptSize, zInt+i+1, sizeof(zInt)-i); + } + rc = Tcl_Eval(faultSimInterp, faultSimScript); + if( rc ){ + fprintf(stderr, "fault simulator script failed: [%s]", faultSimScript); + rc = SQLITE_ERROR; + }else{ + rc = atoi(Tcl_GetStringResult(faultSimInterp)); + } + Tcl_ResetResult(faultSimInterp); + return rc; +} + +/* +** sqlite3_test_control_fault_install SCRIPT +** +** Arrange to invoke SCRIPT with the integer argument to sqlite3FaultSim() +** appended, whenever sqlite3FaultSim() is called. Or, if SCRIPT is the +** empty string, cancel the sqlite3FaultSim() callback. +*/ +static int faultInstallCmd( + void *NotUsed, + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + const char **argv /* Text of each argument */ +){ + const char *zScript; + int nScript; + int rc; + if( argc!=1 && argc!=2 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], + " SCRIPT\"", (void*)0); + } + zScript = argc==2 ? argv[1] : ""; + nScript = (int)strlen(zScript); + if( faultSimScript ){ + free(faultSimScript); + faultSimScript = 0; + } + if( nScript==0 ){ + rc = sqlite3_test_control(SQLITE_TESTCTRL_FAULT_INSTALL, 0); + }else{ + faultSimScript = malloc( nScript+100 ); + if( faultSimScript==0 ){ + Tcl_AppendResult(interp, "out of memory", (void*)0); + return SQLITE_ERROR; + } + memcpy(faultSimScript, zScript, nScript); + faultSimScript[nScript] = ' '; + faultSimScriptSize = nScript+1; + faultSimInterp = interp; + rc = sqlite3_test_control(SQLITE_TESTCTRL_FAULT_INSTALL, faultSimCallback); + } + Tcl_SetObjResult(interp, Tcl_NewIntObj(rc)); + return SQLITE_OK; +} /* ** sqlite3BitvecBuiltinTest SIZE PROGRAM @@ -638,7 +721,8 @@ int Sqlitetest2_Init(Tcl_Interp *interp){ { "fake_big_file", (Tcl_CmdProc*)fake_big_file }, #endif { "sqlite3BitvecBuiltinTest",(Tcl_CmdProc*)testBitvecBuiltinTest }, - { "sqlite3_test_control_pending_byte", (Tcl_CmdProc*)testPendingByte }, + { "sqlite3_test_control_pending_byte", (Tcl_CmdProc*)testPendingByte }, + { "sqlite3_test_control_fault_install", (Tcl_CmdProc*)faultInstallCmd }, }; int i; for(i=0; i #include diff --git a/src/test_osinst.c b/src/test_osinst.c index 531433313e..1701def159 100644 --- a/src/test_osinst.c +++ b/src/test_osinst.c @@ -70,6 +70,12 @@ */ #include "sqlite3.h" + +#include "os_setup.h" +#if SQLITE_OS_WIN +# include "os_win.h" +#endif + #include #include @@ -221,7 +227,6 @@ static sqlite3_uint64 vfslog_time(){ return sTime.tv_usec + (sqlite3_uint64)sTime.tv_sec * 1000000; } #elif SQLITE_OS_WIN -#include #include static sqlite3_uint64 vfslog_time(){ FILETIME ft; diff --git a/src/test_quota.c b/src/test_quota.c index e590996ca4..80ebd0589e 100644 --- a/src/test_quota.c +++ b/src/test_quota.c @@ -44,49 +44,13 @@ #define sqlite3_mutex_notheld(X) ((void)(X),1) #endif /* SQLITE_THREADSAFE==0 */ - -/* -** Figure out if we are dealing with Unix, Windows, or some other -** operating system. After the following block of preprocess macros, -** all of SQLITE_OS_UNIX, SQLITE_OS_WIN, and SQLITE_OS_OTHER -** will defined to either 1 or 0. One of the four will be 1. The other -** three will be 0. -*/ -#if defined(SQLITE_OS_OTHER) -# if SQLITE_OS_OTHER==1 -# undef SQLITE_OS_UNIX -# define SQLITE_OS_UNIX 0 -# undef SQLITE_OS_WIN -# define SQLITE_OS_WIN 0 -# else -# undef SQLITE_OS_OTHER -# endif -#endif -#if !defined(SQLITE_OS_UNIX) && !defined(SQLITE_OS_OTHER) -# define SQLITE_OS_OTHER 0 -# ifndef SQLITE_OS_WIN -# if defined(_WIN32) || defined(WIN32) || defined(__CYGWIN__) \ - || defined(__MINGW32__) || defined(__BORLANDC__) -# define SQLITE_OS_WIN 1 -# define SQLITE_OS_UNIX 0 -# else -# define SQLITE_OS_WIN 0 -# define SQLITE_OS_UNIX 1 -# endif -# else -# define SQLITE_OS_UNIX 0 -# endif -#else -# ifndef SQLITE_OS_WIN -# define SQLITE_OS_WIN 0 -# endif -#endif +#include "os_setup.h" #if SQLITE_OS_UNIX # include #endif #if SQLITE_OS_WIN -# include +# include "os_win.h" # include #endif diff --git a/src/test_quota.h b/src/test_quota.h index 2d0767a19a..c17e15adca 100644 --- a/src/test_quota.h +++ b/src/test_quota.h @@ -31,12 +31,6 @@ #include #include #include -#if SQLITE_OS_UNIX -# include -#endif -#if SQLITE_OS_WIN -# include -#endif /* Make this callable from C++ */ #ifdef __cplusplus diff --git a/src/test_vfs.c b/src/test_vfs.c index 613b0fce77..7ee2a93453 100644 --- a/src/test_vfs.c +++ b/src/test_vfs.c @@ -127,8 +127,10 @@ struct Testvfs { #define TESTVFS_FULLPATHNAME_MASK 0x00008000 #define TESTVFS_READ_MASK 0x00010000 #define TESTVFS_UNLOCK_MASK 0x00020000 +#define TESTVFS_LOCK_MASK 0x00040000 +#define TESTVFS_CKLOCK_MASK 0x00080000 -#define TESTVFS_ALL_MASK 0x0003FFFF +#define TESTVFS_ALL_MASK 0x000FFFFF #define TESTVFS_MAX_PAGES 1024 @@ -466,8 +468,15 @@ static int tvfsFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ ** Lock an tvfs-file. */ static int tvfsLock(sqlite3_file *pFile, int eLock){ - TestvfsFd *p = tvfsGetFd(pFile); - return sqlite3OsLock(p->pReal, eLock); + TestvfsFd *pFd = tvfsGetFd(pFile); + Testvfs *p = (Testvfs *)pFd->pVfs->pAppData; + if( p->pScript && p->mask&TESTVFS_LOCK_MASK ){ + char zLock[30]; + sqlite3_snprintf(sizeof(zLock),zLock,"%d",eLock); + tvfsExecTcl(p, "xLock", Tcl_NewStringObj(pFd->zFilename, -1), + Tcl_NewStringObj(zLock, -1), 0, 0); + } + return sqlite3OsLock(pFd->pReal, eLock); } /* @@ -476,6 +485,12 @@ static int tvfsLock(sqlite3_file *pFile, int eLock){ static int tvfsUnlock(sqlite3_file *pFile, int eLock){ TestvfsFd *pFd = tvfsGetFd(pFile); Testvfs *p = (Testvfs *)pFd->pVfs->pAppData; + if( p->pScript && p->mask&TESTVFS_UNLOCK_MASK ){ + char zLock[30]; + sqlite3_snprintf(sizeof(zLock),zLock,"%d",eLock); + tvfsExecTcl(p, "xUnlock", Tcl_NewStringObj(pFd->zFilename, -1), + Tcl_NewStringObj(zLock, -1), 0, 0); + } if( p->mask&TESTVFS_WRITE_MASK && tvfsInjectIoerr(p) ){ return SQLITE_IOERR_UNLOCK; } @@ -486,8 +501,13 @@ static int tvfsUnlock(sqlite3_file *pFile, int eLock){ ** Check if another file-handle holds a RESERVED lock on an tvfs-file. */ static int tvfsCheckReservedLock(sqlite3_file *pFile, int *pResOut){ - TestvfsFd *p = tvfsGetFd(pFile); - return sqlite3OsCheckReservedLock(p->pReal, pResOut); + TestvfsFd *pFd = tvfsGetFd(pFile); + Testvfs *p = (Testvfs *)pFd->pVfs->pAppData; + if( p->pScript && p->mask&TESTVFS_CKLOCK_MASK ){ + tvfsExecTcl(p, "xCheckReservedLock", Tcl_NewStringObj(pFd->zFilename, -1), + 0, 0, 0); + } + return sqlite3OsCheckReservedLock(pFd->pReal, pResOut); } /* @@ -1111,26 +1131,32 @@ static int testvfs_obj_cmd( break; } + /* TESTVFS filter METHOD-LIST + ** + ** Activate special processing for those methods contained in the list + */ case CMD_FILTER: { static struct VfsMethod { char *zName; int mask; } vfsmethod [] = { - { "xShmOpen", TESTVFS_SHMOPEN_MASK }, - { "xShmLock", TESTVFS_SHMLOCK_MASK }, - { "xShmBarrier", TESTVFS_SHMBARRIER_MASK }, - { "xShmUnmap", TESTVFS_SHMCLOSE_MASK }, - { "xShmMap", TESTVFS_SHMMAP_MASK }, - { "xSync", TESTVFS_SYNC_MASK }, - { "xDelete", TESTVFS_DELETE_MASK }, - { "xWrite", TESTVFS_WRITE_MASK }, - { "xRead", TESTVFS_READ_MASK }, - { "xTruncate", TESTVFS_TRUNCATE_MASK }, - { "xOpen", TESTVFS_OPEN_MASK }, - { "xClose", TESTVFS_CLOSE_MASK }, - { "xAccess", TESTVFS_ACCESS_MASK }, - { "xFullPathname", TESTVFS_FULLPATHNAME_MASK }, - { "xUnlock", TESTVFS_UNLOCK_MASK }, + { "xShmOpen", TESTVFS_SHMOPEN_MASK }, + { "xShmLock", TESTVFS_SHMLOCK_MASK }, + { "xShmBarrier", TESTVFS_SHMBARRIER_MASK }, + { "xShmUnmap", TESTVFS_SHMCLOSE_MASK }, + { "xShmMap", TESTVFS_SHMMAP_MASK }, + { "xSync", TESTVFS_SYNC_MASK }, + { "xDelete", TESTVFS_DELETE_MASK }, + { "xWrite", TESTVFS_WRITE_MASK }, + { "xRead", TESTVFS_READ_MASK }, + { "xTruncate", TESTVFS_TRUNCATE_MASK }, + { "xOpen", TESTVFS_OPEN_MASK }, + { "xClose", TESTVFS_CLOSE_MASK }, + { "xAccess", TESTVFS_ACCESS_MASK }, + { "xFullPathname", TESTVFS_FULLPATHNAME_MASK }, + { "xUnlock", TESTVFS_UNLOCK_MASK }, + { "xLock", TESTVFS_LOCK_MASK }, + { "xCheckReservedLock", TESTVFS_CKLOCK_MASK }, }; Tcl_Obj **apElem = 0; int nElem = 0; @@ -1162,6 +1188,12 @@ static int testvfs_obj_cmd( break; } + /* + ** TESTVFS script ?SCRIPT? + ** + ** Query or set the script to be run when filtered VFS events + ** occur. + */ case CMD_SCRIPT: { if( objc==3 ){ int nByte; @@ -1248,6 +1280,7 @@ static int testvfs_obj_cmd( { "safe_append", SQLITE_IOCAP_SAFE_APPEND }, { "undeletable_when_open", SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN }, { "powersafe_overwrite", SQLITE_IOCAP_POWERSAFE_OVERWRITE }, + { "immutable", SQLITE_IOCAP_IMMUTABLE }, { 0, 0 } }; Tcl_Obj *pRet; diff --git a/src/util.c b/src/util.c index d88c17b759..4fe07a862a 100644 --- a/src/util.c +++ b/src/util.c @@ -31,6 +31,24 @@ void sqlite3Coverage(int x){ } #endif +/* +** Give a callback to the test harness that can be used to simulate faults +** in places where it is difficult or expensive to do so purely by means +** of inputs. +** +** The intent of the integer argument is to let the fault simulator know +** which of multiple sqlite3FaultSim() calls has been hit. +** +** Return whatever integer value the test callback returns, or return +** SQLITE_OK if no test callback is installed. +*/ +#ifndef SQLITE_OMIT_BUILTIN_TEST +int sqlite3FaultSim(int iTest){ + int (*xCallback)(int) = sqlite3GlobalConfig.xTestCallback; + return xCallback ? xCallback(iTest) : SQLITE_OK; +} +#endif + #ifndef SQLITE_OMIT_FLOATING_POINT /* ** Return true if the floating point value is Not a Number (NaN). @@ -1246,8 +1264,8 @@ LogEst sqlite3LogEstAdd(LogEst a, LogEst b){ } /* -** Convert an integer into a LogEst. In other words, compute a -** good approximatation for 10*log2(x). +** Convert an integer into a LogEst. In other words, compute an +** approximation for 10*log2(x). */ LogEst sqlite3LogEst(u64 x){ static LogEst a[] = { 0, 2, 3, 5, 6, 7, 8, 9 }; diff --git a/src/vdbe.c b/src/vdbe.c index 7cf341d938..95760b6da7 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -4293,6 +4293,7 @@ case OP_SorterData: { pC = p->apCsr[pOp->p1]; assert( isSorter(pC) ); rc = sqlite3VdbeSorterRowkey(pC, pOut); + assert( rc!=SQLITE_OK || (pOut->flags & MEM_Blob) ); break; } diff --git a/src/where.c b/src/where.c index 6342b7b50b..2daa9f1536 100644 --- a/src/where.c +++ b/src/where.c @@ -227,7 +227,7 @@ static int whereClauseInsert(WhereClause *pWC, Expr *p, u8 wtFlags){ if( p && ExprHasProperty(p, EP_Unlikely) ){ pTerm->truthProb = sqlite3LogEst(p->iTable) - 99; }else{ - pTerm->truthProb = -1; + pTerm->truthProb = 1; } pTerm->pExpr = sqlite3ExprSkipCollate(p); pTerm->wtFlags = wtFlags; @@ -1956,7 +1956,8 @@ static void whereKeyStats( iLower = 0; iUpper = aSample[0].anLt[iCol]; }else{ - iUpper = i>=pIdx->nSample ? pIdx->aiRowEst[0] : aSample[i].anLt[iCol]; + i64 nRow0 = sqlite3LogEstToInt(pIdx->aiRowLogEst[0]); + iUpper = i>=pIdx->nSample ? nRow0 : aSample[i].anLt[iCol]; iLower = aSample[i-1].anEq[iCol] + aSample[i-1].anLt[iCol]; } aStat[1] = (pIdx->nKeyCol>iCol ? pIdx->aAvgEq[iCol] : 1); @@ -1975,6 +1976,29 @@ static void whereKeyStats( } #endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ +/* +** If it is not NULL, pTerm is a term that provides an upper or lower +** bound on a range scan. Without considering pTerm, it is estimated +** that the scan will visit nNew rows. This function returns the number +** estimated to be visited after taking pTerm into account. +** +** If the user explicitly specified a likelihood() value for this term, +** then the return value is the likelihood multiplied by the number of +** input rows. Otherwise, this function assumes that an "IS NOT NULL" term +** has a likelihood of 0.50, and any other term a likelihood of 0.25. +*/ +static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){ + LogEst nRet = nNew; + if( pTerm ){ + if( pTerm->truthProb<=0 ){ + nRet += pTerm->truthProb; + }else if( (pTerm->wtFlags & TERM_VNULL)==0 ){ + nRet -= 20; assert( 20==sqlite3LogEst(4) ); + } + } + return nRet; +} + /* ** This function is used to estimate the number of rows that will be visited ** by scanning an index for a range of values. The range may have an upper @@ -2067,7 +2091,7 @@ static int whereRangeScanEst( /* Determine iLower and iUpper using ($P) only. */ if( nEq==0 ){ iLower = 0; - iUpper = p->aiRowEst[0]; + iUpper = sqlite3LogEstToInt(p->aiRowLogEst[0]); }else{ /* Note: this call could be optimized away - since the same values must ** have been requested when testing key $P in whereEqualScanEst(). */ @@ -2127,17 +2151,18 @@ static int whereRangeScanEst( UNUSED_PARAMETER(pBuilder); #endif assert( pLower || pUpper ); - /* TUNING: Each inequality constraint reduces the search space 4-fold. - ** A BETWEEN operator, therefore, reduces the search space 16-fold */ - nNew = nOut; - if( pLower && (pLower->wtFlags & TERM_VNULL)==0 ){ - nNew -= 20; assert( 20==sqlite3LogEst(4) ); - nOut--; - } - if( pUpper ){ - nNew -= 20; assert( 20==sqlite3LogEst(4) ); - nOut--; - } + assert( pUpper==0 || (pUpper->wtFlags & TERM_VNULL)==0 ); + nNew = whereRangeAdjust(pLower, nOut); + nNew = whereRangeAdjust(pUpper, nNew); + + /* TUNING: If there is both an upper and lower limit, assume the range is + ** reduced by an additional 75%. This means that, by default, an open-ended + ** range query (e.g. col > ?) is assumed to match 1/4 of the rows in the + ** index. While a closed range (e.g. col BETWEEN ? AND ?) is estimated to + ** match 1/64 of the index. */ + if( pLower && pUpper ) nNew -= 20; + + nOut -= (pLower!=0) + (pUpper!=0); if( nNew<10 ) nNew = 10; if( nNewnOut = (LogEst)nOut; @@ -2234,6 +2259,7 @@ static int whereInScanEst( tRowcnt *pnRow /* Write the revised row estimate here */ ){ Index *p = pBuilder->pNew->u.btree.pIndex; + i64 nRow0 = sqlite3LogEstToInt(p->aiRowLogEst[0]); int nRecValid = pBuilder->nRecValid; int rc = SQLITE_OK; /* Subfunction return code */ tRowcnt nEst; /* Number of rows for a single term */ @@ -2242,14 +2268,14 @@ static int whereInScanEst( assert( p->aSample!=0 ); for(i=0; rc==SQLITE_OK && inExpr; i++){ - nEst = p->aiRowEst[0]; + nEst = nRow0; rc = whereEqualScanEst(pParse, pBuilder, pList->a[i].pExpr, &nEst); nRowEst += nEst; pBuilder->nRecValid = nRecValid; } if( rc==SQLITE_OK ){ - if( nRowEst > p->aiRowEst[0] ) nRowEst = p->aiRowEst[0]; + if( nRowEst > nRow0 ) nRowEst = nRow0; *pnRow = nRowEst; WHERETRACE(0x10,("IN row estimate: est=%g\n", nRowEst)); } @@ -3735,7 +3761,7 @@ static int whereLoopCheaperProperSubset( if( pX->rRun > pY->rRun ) return 0; /* X costs more than Y */ if( pX->nOut > pY->nOut ) return 0; /* X costs more than Y */ } - for(j=0, i=pX->nLTerm-1; i>=0; i--){ + for(i=pX->nLTerm-1; i>=0; i--){ for(j=pY->nLTerm-1; j>=0; j--){ if( pY->aLTerm[j]==pX->aLTerm[i] ) break; } @@ -3757,12 +3783,25 @@ static int whereLoopCheaperProperSubset( ** To say "WhereLoop X is a proper subset of Y" means that X uses fewer ** WHERE clause terms than Y and that every WHERE clause term used by X is ** also used by Y. +** +** This adjustment is omitted for SKIPSCAN loops. In a SKIPSCAN loop, the +** WhereLoop.nLTerm field is not an accurate measure of the number of WHERE +** clause terms covered, since some of the first nLTerm entries in aLTerm[] +** will be NULL (because they are skipped). That makes it more difficult +** to compare the loops. We could add extra code to do the comparison, and +** perhaps we will someday. But SKIPSCAN is sufficiently uncommon, and this +** adjustment is sufficient minor, that it is very difficult to construct +** a test case where the extra code would improve the query plan. Better +** to avoid the added complexity and just omit cost adjustments to SKIPSCAN +** loops. */ static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ if( (pTemplate->wsFlags & WHERE_INDEXED)==0 ) return; + if( (pTemplate->wsFlags & WHERE_SKIPSCAN)!=0 ) return; for(; p; p=p->pNextLoop){ if( p->iTab!=pTemplate->iTab ) continue; if( (p->wsFlags & WHERE_INDEXED)==0 ) continue; + if( (p->wsFlags & WHERE_SKIPSCAN)!=0 ) continue; if( whereLoopCheaperProperSubset(p, pTemplate) ){ /* Adjust pTemplate cost downward so that it is cheaper than its ** subset p */ @@ -3987,13 +4026,20 @@ static void whereLoopOutputAdjust(WhereClause *pWC, WhereLoop *pLoop){ if( pX==pTerm ) break; if( pX->iParent>=0 && (&pWC->a[pX->iParent])==pTerm ) break; } - if( j<0 ) pLoop->nOut += pTerm->truthProb; + if( j<0 ){ + pLoop->nOut += (pTerm->truthProb<=0 ? pTerm->truthProb : -1); + } } } /* -** We have so far matched pBuilder->pNew->u.btree.nEq terms of the index pIndex. -** Try to match one more. +** We have so far matched pBuilder->pNew->u.btree.nEq terms of the +** index pIndex. Try to match one more. +** +** When this function is called, pBuilder->pNew->nOut contains the +** number of rows expected to be visited by filtering using the nEq +** terms only. If it is modified, this value is restored before this +** function returns. ** ** If pProbe->tnum==0, that means pIndex is a fake index used for the ** INTEGER PRIMARY KEY. @@ -4019,7 +4065,6 @@ static int whereLoopAddBtreeIndex( LogEst saved_nOut; /* Original value of pNew->nOut */ int iCol; /* Index of the column in the table */ int rc = SQLITE_OK; /* Return code */ - LogEst nRowEst; /* Estimated index selectivity */ LogEst rLogSize; /* Logarithm of table size */ WhereTerm *pTop = 0, *pBtm = 0; /* Top and bottom range constraints */ @@ -4040,11 +4085,8 @@ static int whereLoopAddBtreeIndex( assert( pNew->u.btree.nEq<=pProbe->nKeyCol ); if( pNew->u.btree.nEq < pProbe->nKeyCol ){ iCol = pProbe->aiColumn[pNew->u.btree.nEq]; - nRowEst = sqlite3LogEst(pProbe->aiRowEst[pNew->u.btree.nEq+1]); - if( nRowEst==0 && pProbe->onError==OE_None ) nRowEst = 1; }else{ iCol = -1; - nRowEst = 0; } pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, iCol, opMask, pProbe); @@ -4055,18 +4097,23 @@ static int whereLoopAddBtreeIndex( saved_prereq = pNew->prereq; saved_nOut = pNew->nOut; pNew->rSetup = 0; - rLogSize = estLog(sqlite3LogEst(pProbe->aiRowEst[0])); + rLogSize = estLog(pProbe->aiRowLogEst[0]); /* Consider using a skip-scan if there are no WHERE clause constraints ** available for the left-most terms of the index, and if the average - ** number of repeats in the left-most terms is at least 18. The magic - ** number 18 was found by experimentation to be the payoff point where - ** skip-scan become faster than a full-scan. - */ + ** number of repeats in the left-most terms is at least 18. + ** + ** The magic number 18 is selected on the basis that scanning 17 rows + ** is almost always quicker than an index seek (even though if the index + ** contains fewer than 2^17 rows we assume otherwise in other parts of + ** the code). And, even if it is not, it should not be too much slower. + ** On the other hand, the extra seeks could end up being significantly + ** more expensive. */ + assert( 42==sqlite3LogEst(18) ); if( pTerm==0 && saved_nEq==saved_nSkip && saved_nEq+1nKeyCol - && pProbe->aiRowEst[saved_nEq+1]>=18 /* TUNING: Minimum for skip-scan */ + && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */ && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK ){ LogEst nIter; @@ -4074,34 +4121,40 @@ static int whereLoopAddBtreeIndex( pNew->u.btree.nSkip++; pNew->aLTerm[pNew->nLTerm++] = 0; pNew->wsFlags |= WHERE_SKIPSCAN; - nIter = sqlite3LogEst(pProbe->aiRowEst[0]/pProbe->aiRowEst[saved_nEq+1]); - pNew->rRun = rLogSize + nIter; - pNew->nOut += nIter; - whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter); + nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1]; + pNew->nOut -= nIter; + whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul); pNew->nOut = saved_nOut; } for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ + u16 eOp = pTerm->eOperator; /* Shorthand for pTerm->eOperator */ + LogEst rCostIdx; + LogEst nOutUnadjusted; /* nOut before IN() and WHERE adjustments */ int nIn = 0; #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 int nRecValid = pBuilder->nRecValid; #endif - if( (pTerm->eOperator==WO_ISNULL || (pTerm->wtFlags&TERM_VNULL)!=0) + if( (eOp==WO_ISNULL || (pTerm->wtFlags&TERM_VNULL)!=0) && (iCol<0 || pSrc->pTab->aCol[iCol].notNull) ){ continue; /* ignore IS [NOT] NULL constraints on NOT NULL columns */ } if( pTerm->prereqRight & pNew->maskSelf ) continue; - assert( pNew->nOut==saved_nOut ); - pNew->wsFlags = saved_wsFlags; pNew->u.btree.nEq = saved_nEq; pNew->nLTerm = saved_nLTerm; if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */ pNew->aLTerm[pNew->nLTerm++] = pTerm; pNew->prereq = (saved_prereq | pTerm->prereqRight) & ~pNew->maskSelf; - pNew->rRun = rLogSize; /* Baseline cost is log2(N). Adjustments below */ - if( pTerm->eOperator & WO_IN ){ + + assert( nInMul==0 + || (pNew->wsFlags & WHERE_COLUMN_NULL)!=0 + || (pNew->wsFlags & WHERE_COLUMN_IN)!=0 + || (pNew->wsFlags & WHERE_SKIPSCAN)!=0 + ); + + if( eOp & WO_IN ){ Expr *pExpr = pTerm->pExpr; pNew->wsFlags |= WHERE_COLUMN_IN; if( ExprHasProperty(pExpr, EP_xIsSelect) ){ @@ -4113,83 +4166,118 @@ static int whereLoopAddBtreeIndex( } assert( nIn>0 ); /* RHS always has 2 or more terms... The parser ** changes "x IN (?)" into "x=?". */ - pNew->rRun += nIn; - pNew->u.btree.nEq++; - pNew->nOut = nRowEst + nInMul + nIn; - }else if( pTerm->eOperator & (WO_EQ) ){ - assert( - (pNew->wsFlags & (WHERE_COLUMN_NULL|WHERE_COLUMN_IN|WHERE_SKIPSCAN))!=0 - || nInMul==0 - ); + + }else if( eOp & (WO_EQ) ){ pNew->wsFlags |= WHERE_COLUMN_EQ; - if( iCol<0 || (nInMul==0 && pNew->u.btree.nEq==pProbe->nKeyCol-1)){ - assert( (pNew->wsFlags & WHERE_COLUMN_IN)==0 || iCol<0 ); + if( iCol<0 || (nInMul==0 && pNew->u.btree.nEq==pProbe->nKeyCol-1) ){ if( iCol>=0 && pProbe->onError==OE_None ){ pNew->wsFlags |= WHERE_UNQ_WANTED; }else{ pNew->wsFlags |= WHERE_ONEROW; } } - pNew->u.btree.nEq++; - pNew->nOut = nRowEst + nInMul; - }else if( pTerm->eOperator & (WO_ISNULL) ){ + }else if( eOp & WO_ISNULL ){ pNew->wsFlags |= WHERE_COLUMN_NULL; - pNew->u.btree.nEq++; - /* TUNING: IS NULL selects 2 rows */ - nIn = 10; assert( 10==sqlite3LogEst(2) ); - pNew->nOut = nRowEst + nInMul + nIn; - }else if( pTerm->eOperator & (WO_GT|WO_GE) ){ - testcase( pTerm->eOperator & WO_GT ); - testcase( pTerm->eOperator & WO_GE ); + }else if( eOp & (WO_GT|WO_GE) ){ + testcase( eOp & WO_GT ); + testcase( eOp & WO_GE ); pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; pBtm = pTerm; pTop = 0; }else{ - assert( pTerm->eOperator & (WO_LT|WO_LE) ); - testcase( pTerm->eOperator & WO_LT ); - testcase( pTerm->eOperator & WO_LE ); + assert( eOp & (WO_LT|WO_LE) ); + testcase( eOp & WO_LT ); + testcase( eOp & WO_LE ); pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT; pTop = pTerm; pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ? pNew->aLTerm[pNew->nLTerm-2] : 0; } + + /* At this point pNew->nOut is set to the number of rows expected to + ** be visited by the index scan before considering term pTerm, or the + ** values of nIn and nInMul. In other words, assuming that all + ** "x IN(...)" terms are replaced with "x = ?". This block updates + ** the value of pNew->nOut to account for pTerm (but not nIn/nInMul). */ + assert( pNew->nOut==saved_nOut ); if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ - /* Adjust nOut and rRun for STAT3 range values */ - assert( pNew->nOut==saved_nOut ); + /* Adjust nOut using stat3/stat4 data. Or, if there is no stat3/stat4 + ** data, using some other estimate. */ whereRangeScanEst(pParse, pBuilder, pBtm, pTop, pNew); - } + }else{ + int nEq = ++pNew->u.btree.nEq; + assert( eOp & (WO_ISNULL|WO_EQ|WO_IN) ); + + assert( pNew->nOut==saved_nOut ); + if( pTerm->truthProb<=0 && iCol>=0 ){ + assert( (eOp & WO_IN) || nIn==0 ); + testcase( eOp & WO_IN ); + pNew->nOut += pTerm->truthProb; + pNew->nOut -= nIn; + pNew->wsFlags |= WHERE_LIKELIHOOD; + }else{ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - if( nInMul==0 - && pProbe->nSample - && pNew->u.btree.nEq<=pProbe->nSampleCol - && OptimizationEnabled(db, SQLITE_Stat3) - ){ - Expr *pExpr = pTerm->pExpr; - tRowcnt nOut = 0; - if( (pTerm->eOperator & (WO_EQ|WO_ISNULL))!=0 ){ - testcase( pTerm->eOperator & WO_EQ ); - testcase( pTerm->eOperator & WO_ISNULL ); - rc = whereEqualScanEst(pParse, pBuilder, pExpr->pRight, &nOut); - }else if( (pTerm->eOperator & WO_IN) - && !ExprHasProperty(pExpr, EP_xIsSelect) ){ - rc = whereInScanEst(pParse, pBuilder, pExpr->x.pList, &nOut); - } - assert( nOut==0 || rc==SQLITE_OK ); - if( nOut ){ - pNew->nOut = sqlite3LogEst(nOut); - if( pNew->nOut>saved_nOut ) pNew->nOut = saved_nOut; - } - } + tRowcnt nOut = 0; + if( nInMul==0 + && pProbe->nSample + && pNew->u.btree.nEq<=pProbe->nSampleCol + && OptimizationEnabled(db, SQLITE_Stat3) + && ((eOp & WO_IN)==0 || !ExprHasProperty(pTerm->pExpr, EP_xIsSelect)) + && (pNew->wsFlags & WHERE_LIKELIHOOD)==0 + ){ + Expr *pExpr = pTerm->pExpr; + if( (eOp & (WO_EQ|WO_ISNULL))!=0 ){ + testcase( eOp & WO_EQ ); + testcase( eOp & WO_ISNULL ); + rc = whereEqualScanEst(pParse, pBuilder, pExpr->pRight, &nOut); + }else{ + rc = whereInScanEst(pParse, pBuilder, pExpr->x.pList, &nOut); + } + assert( rc!=SQLITE_OK || nOut>0 ); + if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK; + if( rc!=SQLITE_OK ) break; /* Jump out of the pTerm loop */ + if( nOut ){ + pNew->nOut = sqlite3LogEst(nOut); + if( pNew->nOut>saved_nOut ) pNew->nOut = saved_nOut; + pNew->nOut -= nIn; + } + } + if( nOut==0 ) #endif - if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){ - /* Each row involves a step of the index, then a binary search of - ** the main table */ - pNew->rRun = sqlite3LogEstAdd(pNew->rRun,rLogSize>27 ? rLogSize-17 : 10); + { + pNew->nOut += (pProbe->aiRowLogEst[nEq] - pProbe->aiRowLogEst[nEq-1]); + if( eOp & WO_ISNULL ){ + /* TUNING: If there is no likelihood() value, assume that a + ** "col IS NULL" expression matches twice as many rows + ** as (col=?). */ + pNew->nOut += 10; + } + } + } } - /* Step cost for each output row */ - pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut); + + /* Set rCostIdx to the cost of visiting selected rows in index. Add + ** it to pNew->rRun, which is currently set to the cost of the index + ** seek only. Then, if this is a non-covering index, add the cost of + ** visiting the rows in the main table. */ + rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow; + pNew->rRun = sqlite3LogEstAdd(rLogSize, rCostIdx); + if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){ + pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16); + } + + nOutUnadjusted = pNew->nOut; + pNew->rRun += nInMul + nIn; + pNew->nOut += nInMul + nIn; whereLoopOutputAdjust(pBuilder->pWC, pNew); rc = whereLoopInsert(pBuilder, pNew); + + if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ + pNew->nOut = saved_nOut; + }else{ + pNew->nOut = nOutUnadjusted; + } + if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 && pNew->u.btree.nEq<(pProbe->nKeyCol + (pProbe->zName!=0)) ){ @@ -4273,6 +4361,29 @@ static int whereUsablePartialIndex(int iTab, WhereClause *pWC, Expr *pWhere){ ** Add all WhereLoop objects for a single table of the join where the table ** is idenfied by pBuilder->pNew->iTab. That table is guaranteed to be ** a b-tree table, not a virtual table. +** +** The costs (WhereLoop.rRun) of the b-tree loops added by this function +** are calculated as follows: +** +** For a full scan, assuming the table (or index) contains nRow rows: +** +** cost = nRow * 3.0 // full-table scan +** cost = nRow * K // scan of covering index +** cost = nRow * (K+3.0) // scan of non-covering index +** +** where K is a value between 1.1 and 3.0 set based on the relative +** estimated average size of the index and table records. +** +** For an index scan, where nVisit is the number of index rows visited +** by the scan, and nSeek is the number of seek operations required on +** the index b-tree: +** +** cost = nSeek * (log(nRow) + K * nVisit) // covering index +** cost = nSeek * (log(nRow) + (K+3.0) * nVisit) // non-covering index +** +** Normally, nSeek is 1. nSeek values greater than 1 come about if the +** WHERE clause includes "x IN (....)" terms used in place of "x=?". Or when +** implicit "x IN (SELECT x FROM tbl)" terms are added for skip-scans. */ static int whereLoopAddBtree( WhereLoopBuilder *pBuilder, /* WHERE clause information */ @@ -4281,7 +4392,7 @@ static int whereLoopAddBtree( WhereInfo *pWInfo; /* WHERE analysis context */ Index *pProbe; /* An index we are evaluating */ Index sPk; /* A fake index object for the primary key */ - tRowcnt aiRowEstPk[2]; /* The aiRowEst[] value for the sPk index */ + LogEst aiRowEstPk[2]; /* The aiRowLogEst[] value for the sPk index */ i16 aiColumnPk = -1; /* The aColumn[] value for the sPk index */ SrcList *pTabList; /* The FROM clause */ struct SrcList_item *pSrc; /* The FROM clause btree term to add */ @@ -4316,11 +4427,12 @@ static int whereLoopAddBtree( memset(&sPk, 0, sizeof(Index)); sPk.nKeyCol = 1; sPk.aiColumn = &aiColumnPk; - sPk.aiRowEst = aiRowEstPk; + sPk.aiRowLogEst = aiRowEstPk; sPk.onError = OE_Replace; sPk.pTable = pTab; - aiRowEstPk[0] = pTab->nRowEst; - aiRowEstPk[1] = 1; + sPk.szIdxRow = pTab->szTabRow; + aiRowEstPk[0] = pTab->nRowLogEst; + aiRowEstPk[1] = 0; pFirst = pSrc->pTab->pIndex; if( pSrc->notIndexed==0 ){ /* The real indices of the table are only considered if the @@ -4329,7 +4441,7 @@ static int whereLoopAddBtree( } pProbe = &sPk; } - rSize = sqlite3LogEst(pTab->nRowEst); + rSize = pTab->nRowLogEst; rLogSize = estLog(rSize); #ifndef SQLITE_OMIT_AUTOMATIC_INDEX @@ -4379,6 +4491,7 @@ static int whereLoopAddBtree( && !whereUsablePartialIndex(pNew->iTab, pWC, pProbe->pPartIdxWhere) ){ continue; /* Partial index inappropriate for this query */ } + rSize = pProbe->aiRowLogEst[0]; pNew->u.btree.nEq = 0; pNew->u.btree.nSkip = 0; pNew->nLTerm = 0; @@ -4396,10 +4509,8 @@ static int whereLoopAddBtree( /* Full table scan */ pNew->iSortIdx = b ? iSortIdx : 0; - /* TUNING: Cost of full table scan is 3*(N + log2(N)). - ** + The extra 3 factor is to encourage the use of indexed lookups - ** over full scans. FIXME */ - pNew->rRun = sqlite3LogEstAdd(rSize,rLogSize) + 16; + /* TUNING: Cost of full table scan is (N*3.0). */ + pNew->rRun = rSize + 16; whereLoopOutputAdjust(pWC, pNew); rc = whereLoopInsert(pBuilder, pNew); pNew->nOut = rSize; @@ -4426,35 +4537,16 @@ static int whereLoopAddBtree( ) ){ pNew->iSortIdx = b ? iSortIdx : 0; - /* TUNING: The base cost of an index scan is N + log2(N). - ** The log2(N) is for the initial seek to the beginning and the N - ** is for the scan itself. */ - pNew->rRun = sqlite3LogEstAdd(rSize, rLogSize); - if( m==0 ){ - /* TUNING: Cost of a covering index scan is K*(N + log2(N)). - ** + The extra factor K of between 1.1 and 3.0 that depends - ** on the relative sizes of the table and the index. K - ** is smaller for smaller indices, thus favoring them. - ** The upper bound on K (3.0) matches the penalty factor - ** on a full table scan that tries to encourage the use of - ** indexed lookups over full scans. - */ - pNew->rRun += 1 + (15*pProbe->szIdxRow)/pTab->szTabRow; - }else{ - /* TUNING: The cost of scanning a non-covering index is multiplied - ** by log2(N) to account for the binary search of the main table - ** that must happen for each row of the index. - ** TODO: Should there be a multiplier here, analogous to the 3x - ** multiplier for a fulltable scan or covering index scan, to - ** further discourage the use of an index scan? Or is the log2(N) - ** term sufficient discouragement? - ** TODO: What if some or all of the WHERE clause terms can be - ** computed without reference to the original table. Then the - ** penality should reduce to logK where K is the number of output - ** rows. - */ - pNew->rRun += rLogSize; + + /* The cost of visiting the index rows is N*K, where K is + ** between 1.1 and 3.0, depending on the relative sizes of the + ** index and table rows. If this is a non-covering index scan, + ** also add the cost of visiting table rows (N*3.0). */ + pNew->rRun = rSize + 1 + (15*pProbe->szIdxRow)/pTab->szTabRow; + if( m!=0 ){ + pNew->rRun = sqlite3LogEstAdd(pNew->rRun, rSize+16); } + whereLoopOutputAdjust(pWC, pNew); rc = whereLoopInsert(pBuilder, pNew); pNew->nOut = rSize; @@ -4658,7 +4750,7 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ int iCur; WhereClause tempWC; WhereLoopBuilder sSubBuild; - WhereOrSet sSum, sCur, sPrev; + WhereOrSet sSum, sCur; struct SrcList_item *pItem; pWC = pBuilder->pWC; @@ -4714,6 +4806,7 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ whereOrMove(&sSum, &sCur); once = 0; }else{ + WhereOrSet sPrev; whereOrMove(&sPrev, &sSum); sSum.n = 0; for(i=0; iiSortIdx = 0; memset(&pNew->u, 0, sizeof(pNew->u)); for(i=0; rc==SQLITE_OK && irRun = sSum.a[i].rRun + 18; + /* TUNING: Currently sSum.a[i].rRun is set to the sum of the costs + ** of all sub-scans required by the OR-scan. However, due to rounding + ** errors, it may be that the cost of the OR-scan is equal to its + ** most expensive sub-scan. Add the smallest possible penalty + ** (equivalent to multiplying the cost by 1.07) to ensure that + ** this does not happen. Otherwise, for WHERE clauses such as the + ** following where there is an index on "y": + ** + ** WHERE likelihood(x=?, 0.99) OR y=? + ** + ** the planner may elect to "OR" together a full-table scan and an + ** index lookup. And other similarly odd results. */ + pNew->rRun = sSum.a[i].rRun + 1; pNew->nOut = sSum.a[i].nOut; pNew->prereq = sSum.a[i].prereq; rc = whereLoopInsert(pBuilder, pNew); @@ -5179,22 +5283,27 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags, iLoop, pWLoop, &revMask); if( isOrdered>=0 && isOrdered0 ); + assert( nOrderBy>0 && 66==sqlite3LogEst(100) ); rScale = sqlite3LogEst((nOrderBy-isOrdered)*100/nOrderBy) - 66; - rSortCost = nRowEst + estLog(nRowEst) + rScale; + rSortCost = nRowEst + estLog(nRowEst) + rScale + 16; + /* TUNING: The cost of implementing DISTINCT using a B-TREE is - ** also N*log(N) but it has a larger constant of proportionality. - ** Multiply by 3.0. */ + ** similar but with a larger constant of proportionality. + ** Multiply by an additional factor of 3.0. */ if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){ rSortCost += 16; } diff --git a/src/whereInt.h b/src/whereInt.h index 72e7530db9..010cd6e8ae 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -458,3 +458,4 @@ struct WhereInfo { #define WHERE_AUTO_INDEX 0x00004000 /* Uses an ephemeral index */ #define WHERE_SKIPSCAN 0x00008000 /* Uses the skip-scan algorithm */ #define WHERE_UNQ_WANTED 0x00010000 /* WHERE_ONEROW would have been helpful*/ +#define WHERE_LIKELIHOOD 0x00020000 /* A likelihood() is affecting nOut */ diff --git a/test/analyze3.test b/test/analyze3.test index fb26303ee3..e7416d5730 100644 --- a/test/analyze3.test +++ b/test/analyze3.test @@ -103,12 +103,21 @@ do_test analyze3-1.1.1 { } } {1} +do_execsql_test analyze3-1.1.x { + SELECT count(*) FROM t1 WHERE x>200 AND x<300; + SELECT count(*) FROM t1 WHERE x>0 AND x<1100; +} {99 1000} + +# The first of the following two SELECT statements visits 99 rows. So +# it is better to use the index. But the second visits every row in +# the table (1000 in total) so it is better to do a full-table scan. +# do_eqp_test analyze3-1.1.2 { SELECT sum(y) FROM t1 WHERE x>200 AND x<300 } {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (x>? AND x0 AND x<1100 -} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (x>? AND x200 AND x<300 } @@ -125,17 +134,17 @@ do_test analyze3-1.1.6 { } {199 0 14850} do_test analyze3-1.1.7 { sf_execsql { SELECT sum(y) FROM t1 WHERE x>0 AND x<1100 } -} {2000 0 499500} +} {999 999 499500} do_test analyze3-1.1.8 { set l [string range "0" 0 end] set u [string range "1100" 0 end] sf_execsql { SELECT sum(y) FROM t1 WHERE x>$l AND x<$u } -} {2000 0 499500} +} {999 999 499500} do_test analyze3-1.1.9 { set l [expr int(0)] set u [expr int(1100)] sf_execsql { SELECT sum(y) FROM t1 WHERE x>$l AND x<$u } -} {2000 0 499500} +} {999 999 499500} # The following tests are similar to the block above. The difference is @@ -152,12 +161,17 @@ do_test analyze3-1.2.1 { ANALYZE; } } {} +do_execsql_test analyze3-2.1.x { + SELECT count(*) FROM t2 WHERE x>1 AND x<2; + SELECT count(*) FROM t2 WHERE x>0 AND x<99; +} {200 990} do_eqp_test analyze3-1.2.2 { SELECT sum(y) FROM t2 WHERE x>1 AND x<2 } {0 0 0 {SEARCH TABLE t2 USING INDEX i2 (x>? AND x0 AND x<99 -} {0 0 0 {SEARCH TABLE t2 USING INDEX i2 (x>? AND x12 AND x<20 } } {161 0 4760} @@ -173,17 +187,17 @@ do_test analyze3-1.2.6 { } {161 0 integer integer 4760} do_test analyze3-1.2.7 { sf_execsql { SELECT sum(y) FROM t2 WHERE x>0 AND x<99 } -} {1981 0 490555} +} {999 999 490555} do_test analyze3-1.2.8 { set l [string range "0" 0 end] set u [string range "99" 0 end] sf_execsql {SELECT typeof($l), typeof($u), sum(y) FROM t2 WHERE x>$l AND x<$u} -} {1981 0 text text 490555} +} {999 999 text text 490555} do_test analyze3-1.2.9 { set l [expr int(0)] set u [expr int(99)] sf_execsql {SELECT typeof($l), typeof($u), sum(y) FROM t2 WHERE x>$l AND x<$u} -} {1981 0 integer integer 490555} +} {999 999 integer integer 490555} # Same tests a third time. This time, column x has INTEGER affinity and # is not the leftmost column of the table. This triggered a bug causing @@ -199,12 +213,16 @@ do_test analyze3-1.3.1 { ANALYZE; } } {} +do_execsql_test analyze3-1.3.x { + SELECT count(*) FROM t3 WHERE x>200 AND x<300; + SELECT count(*) FROM t3 WHERE x>0 AND x<1100 +} {99 1000} do_eqp_test analyze3-1.3.2 { SELECT sum(y) FROM t3 WHERE x>200 AND x<300 } {0 0 0 {SEARCH TABLE t3 USING INDEX i3 (x>? AND x0 AND x<1100 -} {0 0 0 {SEARCH TABLE t3 USING INDEX i3 (x>? AND x200 AND x<300 } @@ -221,17 +239,17 @@ do_test analyze3-1.3.6 { } {199 0 14850} do_test analyze3-1.3.7 { sf_execsql { SELECT sum(y) FROM t3 WHERE x>0 AND x<1100 } -} {2000 0 499500} +} {999 999 499500} do_test analyze3-1.3.8 { set l [string range "0" 0 end] set u [string range "1100" 0 end] sf_execsql { SELECT sum(y) FROM t3 WHERE x>$l AND x<$u } -} {2000 0 499500} +} {999 999 499500} do_test analyze3-1.3.9 { set l [expr int(0)] set u [expr int(1100)] sf_execsql { SELECT sum(y) FROM t3 WHERE x>$l AND x<$u } -} {2000 0 499500} +} {999 999 499500} #------------------------------------------------------------------------- # Test that the values of bound SQL variables may be used for the LIKE diff --git a/test/analyze9.test b/test/analyze9.test index 820bcdb0e7..f25e5924e6 100644 --- a/test/analyze9.test +++ b/test/analyze9.test @@ -566,7 +566,7 @@ foreach {tn schema} { drop_all_tables do_test 13.1 { execsql { - CREATE TABLE t1(a, b, c); + CREATE TABLE t1(a, b, c, d); CREATE INDEX i1 ON t1(a); CREATE INDEX i2 ON t1(b, c); } @@ -577,16 +577,16 @@ do_test 13.1 { execsql ANALYZE } {} do_eqp_test 13.2.1 { - SELECT * FROM t1 WHERE a='abc' AND rowid<15 AND b<20 + SELECT * FROM t1 WHERE a='abc' AND rowid<15 AND b<12 } {/SEARCH TABLE t1 USING INDEX i1/} do_eqp_test 13.2.2 { - SELECT * FROM t1 WHERE a='abc' AND rowid<'15' AND b<20 + SELECT * FROM t1 WHERE a='abc' AND rowid<'15' AND b<12 } {/SEARCH TABLE t1 USING INDEX i1/} do_eqp_test 13.3.1 { - SELECT * FROM t1 WHERE a='abc' AND rowid<100 AND b<20 + SELECT * FROM t1 WHERE a='abc' AND rowid<100 AND b<12 } {/SEARCH TABLE t1 USING INDEX i2/} do_eqp_test 13.3.2 { - SELECT * FROM t1 WHERE a='abc' AND rowid<'100' AND b<20 + SELECT * FROM t1 WHERE a='abc' AND rowid<'100' AND b<12 } {/SEARCH TABLE t1 USING INDEX i2/} #------------------------------------------------------------------------- diff --git a/test/autoindex1.test b/test/autoindex1.test index 0e5032b9b9..6cb0ab146a 100644 --- a/test/autoindex1.test +++ b/test/autoindex1.test @@ -97,6 +97,8 @@ do_test autoindex1-210 { PRAGMA automatic_index=ON; ANALYZE; UPDATE sqlite_stat1 SET stat='10000' WHERE tbl='t1'; + -- Table t2 actually contains 8 rows. + UPDATE sqlite_stat1 SET stat='16' WHERE tbl='t2'; ANALYZE sqlite_master; SELECT b, (SELECT d FROM t2 WHERE c=a) FROM t1; } diff --git a/test/backcompat.test b/test/backcompat.test index fdc2410b7a..ea7e6a9eed 100644 --- a/test/backcompat.test +++ b/test/backcompat.test @@ -58,12 +58,24 @@ proc do_backcompat_test {rv bin1 bin2 script} { code1 { sqlite3 db test.db } code2 { sqlite3 db test.db } + foreach c {code1 code2} { + $c { + set v [split [db version] .] + if {[llength $v]==3} {lappend v 0} + set ::sqlite_libversion [format \ + "%d%.2d%.2d%2d" [lindex $v 0] [lindex $v 1] [lindex $v 2] [lindex $v 3] + ] + } + } + uplevel $script catch { code1 { db close } } catch { code2 { db close } } catch { close $::bc_chan2 } catch { close $::bc_chan1 } + + } array set ::incompatible [list] @@ -381,6 +393,48 @@ ifcapable fts3 { } { do_test backcompat-3.7 [list sql1 $q] [sql2 $q] } + + # Now test that an incremental merge can be started by one version + # and finished by another. And that the integrity-check still + # passes. + do_test backcompat-3.8 { + sql1 { + DROP TABLE IF EXISTS t1; + DROP TABLE IF EXISTS t2; + CREATE TABLE t1(docid, words); + CREATE VIRTUAL TABLE t2 USING fts3(words); + } + code1 [list source $testdir/genesis.tcl] + code1 { fts_kjv_genesis } + sql1 { + INSERT INTO t2 SELECT words FROM t1; + INSERT INTO t2 SELECT words FROM t1; + INSERT INTO t2 SELECT words FROM t1; + INSERT INTO t2 SELECT words FROM t1; + INSERT INTO t2 SELECT words FROM t1; + INSERT INTO t2 SELECT words FROM t1; + SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level; + } + } {0 {0 1 2 3 4 5}} + + if {[code1 { set ::sqlite_libversion }] >=3071200 + && [code2 { set ::sqlite_libversion }] >=3071200 + } { + do_test backcompat-3.9 { + sql1 { INSERT INTO t2(t2) VALUES('merge=100,4'); } + sql2 { INSERT INTO t2(t2) VALUES('merge=100,4'); } + sql1 { INSERT INTO t2(t2) VALUES('merge=100,4'); } + sql2 { INSERT INTO t2(t2) VALUES('merge=2500,4'); } + sql2 { + SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level; + } + } {0 {0 1} 1 0} + + do_test backcompat-3.10 { + sql1 { INSERT INTO t2(t2) VALUES('integrity-check') } + sql2 { INSERT INTO t2(t2) VALUES('integrity-check') } + } {} + } } } } diff --git a/test/cost.test b/test/cost.test new file mode 100644 index 0000000000..c413c3cdfa --- /dev/null +++ b/test/cost.test @@ -0,0 +1,251 @@ +# 2014-04-26 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix cost + + +do_execsql_test 1.1 { + CREATE TABLE t3(id INTEGER PRIMARY KEY, b NOT NULL); + CREATE TABLE t4(c, d, e); + CREATE UNIQUE INDEX i3 ON t3(b); + CREATE UNIQUE INDEX i4 ON t4(c, d); +} +do_eqp_test 1.2 { + SELECT e FROM t3, t4 WHERE b=c ORDER BY b, d; +} { + 0 0 0 {SCAN TABLE t3 USING COVERING INDEX i3} + 0 1 1 {SEARCH TABLE t4 USING INDEX i4 (c=?)} +} + + +do_execsql_test 2.1 { + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a); +} + +# It is better to use an index for ORDER BY than sort externally, even +# if the index is a non-covering index. +do_eqp_test 2.2 { + SELECT * FROM t1 ORDER BY a; +} { + 0 0 0 {SCAN TABLE t1 USING INDEX i1} +} + +do_execsql_test 3.1 { + CREATE TABLE t5(a INTEGER PRIMARY KEY,b,c,d,e,f,g); + CREATE INDEX t5b ON t5(b); + CREATE INDEX t5c ON t5(c); + CREATE INDEX t5d ON t5(d); + CREATE INDEX t5e ON t5(e); + CREATE INDEX t5f ON t5(f); + CREATE INDEX t5g ON t5(g); +} + +do_eqp_test 3.2 { + SELECT a FROM t5 + WHERE b IS NULL OR c IS NULL OR d IS NULL + ORDER BY a; +} { + 0 0 0 {SEARCH TABLE t5 USING INDEX t5b (b=?)} + 0 0 0 {SEARCH TABLE t5 USING INDEX t5c (c=?)} + 0 0 0 {SEARCH TABLE t5 USING INDEX t5d (d=?)} + 0 0 0 {USE TEMP B-TREE FOR ORDER BY} +} + +#------------------------------------------------------------------------- +# If there is no likelihood() or stat3 data, SQLite assumes that a closed +# range scan (e.g. one constrained by "col BETWEEN ? AND ?" constraint) +# visits 1/64 of the rows in a table. +# +# Note: 1/63 =~ 0.016 +# Note: 1/65 =~ 0.015 +# +reset_db +do_execsql_test 4.1 { + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a); + CREATE INDEX i2 ON t1(b); +} +do_eqp_test 4.2 { + SELECT * FROM t1 WHERE likelihood(a=?, 0.014) AND b BETWEEN ? AND ?; +} { + 0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?)} +} +do_eqp_test 4.3 { + SELECT * FROM t1 WHERE likelihood(a=?, 0.016) AND b BETWEEN ? AND ?; +} { + 0 0 0 {SEARCH TABLE t1 USING INDEX i2 (b>? AND b? AND x? AND b=950 AND b<=1010) OR (b IS NULL AND c NOT NULL) + ORDER BY a +} { + 0 0 0 {SEARCH TABLE t1 USING INDEX t1b (b>? AND b16; + DROP TABLE IF EXISTS x1; + CREATE VIRTUAL TABLE x1 USING fts4; +} + +db func second second +proc second {L} {lindex $L 1} + +for {set tn 0} {$tn < 40} {incr tn} { + do_test 2.2.$tn { + for {set i 0} {$i < 100} {incr i} { + tt 0 ; tt 1 ; tt 2 ; tt 3 + } + execsql { SELECT max(level) FROM x1_segdir } + } {1} +} + + +finish_test + diff --git a/test/fts4merge4.test b/test/fts4merge4.test index 8e2119de2b..038e460d0e 100644 --- a/test/fts4merge4.test +++ b/test/fts4merge4.test @@ -53,6 +53,50 @@ do_execsql_test 2.2 { SELECT count(*) FROM t1_segdir; } 35 do_execsql_test 2.3 { INSERT INTO t1(t1) VALUES('optimize') } {} do_execsql_test 2.4 { SELECT count(*) FROM t1_segdir; } 1 +#------------------------------------------------------------------------- +# Now test that the automerge=? option appears to work. +# +do_execsql_test 2.1 { CREATE VIRTUAL TABLE t2 USING fts4; } + +set doc "" +foreach c1 "a b c d e f g h i j" { + foreach c2 "a b c d e f g h i j" { + foreach c3 "a b c d e f g h i j" { + lappend doc "$c1$c2$c3" + } + } +} +set doc [string repeat $doc 10] + +foreach {tn am expected} { + 1 {automerge=2} {1 1 2 1 4 1 6 1} + 2 {automerge=4} {1 2 2 1 3 1} + 3 {automerge=8} {0 4 1 3 2 1} + 4 {automerge=1} {0 4 1 3 2 1} +} { + foreach {tn2 openclose} {1 {} 2 { db close ; sqlite3 db test.db }} { + do_test 2.2.$tn.$tn2 { + execsql { DELETE FROM t2 } + execsql { INSERT INTO t2(t2) VALUES($am) }; + + eval $openclose + + for {set i 0} {$i < 100} {incr i} { + execsql { + BEGIN; + INSERT INTO t2 VALUES($doc); + INSERT INTO t2 VALUES($doc); + INSERT INTO t2 VALUES($doc); + INSERT INTO t2 VALUES($doc); + INSERT INTO t2 VALUES($doc); + COMMIT; + } + } + + execsql { SELECT level, count(*) FROM t2_segdir GROUP BY level } + } [list {*}$expected] + } +} sqlite3_enable_shared_cache $::enable_shared_cache finish_test diff --git a/test/func.test b/test/func.test index e3cec32153..98ae8ddeb5 100644 --- a/test/func.test +++ b/test/func.test @@ -1194,6 +1194,18 @@ do_test func-24.12 { WHEN 'program' THEN null ELSE t1 END) FROM tbl1 } } {,is,free,software} +# Tests to verify ticket http://www.sqlite.org/src/tktview/55746f9e65f8587c0 +do_test func-24.13 { + execsql { + SELECT typeof(group_concat(x)) FROM (SELECT '' AS x); + } +} {text} +do_test func-24.14 { + execsql { + SELECT typeof(group_concat(x,'')) + FROM (SELECT '' AS x UNION ALL SELECT ''); + } +} {text} # Use the test_isolation function to make sure that type conversions diff --git a/test/fuzz.test b/test/fuzz.test index e1b22ae804..0deed3b636 100644 --- a/test/fuzz.test +++ b/test/fuzz.test @@ -285,7 +285,7 @@ do_test fuzz-1.18 { ) )) } -} {0 -4294967298} +} {0 {{}}} # At one point the following INSERT statement caused an assert() to fail. # diff --git a/test/index6.test b/test/index6.test index 3451e5c1df..68bdd06c14 100644 --- a/test/index6.test +++ b/test/index6.test @@ -145,11 +145,11 @@ do_test index6-2.1 { execsql { CREATE TABLE t2(a,b); INSERT INTO t2(a,b) SELECT value, value FROM nums WHERE value<1000; - UPDATE t2 SET a=NULL WHERE b%5==0; + UPDATE t2 SET a=NULL WHERE b%2==0; CREATE INDEX t2a1 ON t2(a) WHERE a IS NOT NULL; SELECT count(*) FROM t2 WHERE a IS NOT NULL; } -} {800} +} {500} do_test index6-2.2 { execsql { EXPLAIN QUERY PLAN @@ -157,6 +157,7 @@ do_test index6-2.2 { } } {/.* TABLE t2 USING INDEX t2a1 .*/} ifcapable stat4||stat3 { + execsql ANALYZE do_test index6-2.3stat4 { execsql { EXPLAIN QUERY PLAN diff --git a/test/misc1.test b/test/misc1.test index f886f896e2..173b77d637 100644 --- a/test/misc1.test +++ b/test/misc1.test @@ -604,5 +604,21 @@ do_execsql_test misc1-19.2 { SELECT * FROM t19b; } {4 5 6} +# 2014-05-16: Tests for the SQLITE_TESTCTRL_FAULT_INSTALL feature. +# +unset -nocomplain fault_callbacks +set fault_callbacks {} +proc fault_callback {n} { + lappend ::fault_callbacks $n + return 0 +} +do_test misc1-19.1 { + sqlite3_test_control_fault_install fault_callback + set fault_callbacks +} {0} +do_test misc1-19.2 { + sqlite3_test_control_fault_install + set fault_callbacks +} {0} finish_test diff --git a/test/nolock.test b/test/nolock.test new file mode 100644 index 0000000000..331af08ad7 --- /dev/null +++ b/test/nolock.test @@ -0,0 +1,185 @@ +# 2014-05-07 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file implements regression tests for SQLite library. The +# focus of this file is testing the nolock=1 and immutable=1 query +# parameters and the SQLITE_IOCAP_IMMUTABLE device characteristic. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +unset -nocomplain tvfs_calls +proc tvfs_reset {} { + global tvfs_calls + array set tvfs_calls {xLock 0 xUnlock 0 xCheckReservedLock 0 xAccess 0} +} +proc tvfs_callback {op args} { + global tvfs_calls + incr tvfs_calls($op) + return SQLITE_OK +} +tvfs_reset + +testvfs tvfs +tvfs script tvfs_callback +tvfs filter {xLock xUnlock xCheckReservedLock xAccess} + +############################################################################ +# Verify that the nolock=1 query parameter for URI filenames disables all +# calls to xLock and xUnlock for rollback databases. +# +do_test nolock-1.0 { + db close + forcedelete test.db + tvfs_reset + sqlite db test.db -vfs tvfs + db eval {CREATE TABLE t1(a,b,c); INSERT INTO t1 VALUES(1,2,3);} + list xLock $::tvfs_calls(xLock) xUnlock $::tvfs_calls(xUnlock) \ + xCheckReservedLock $::tvfs_calls(xCheckReservedLock) +} {xLock 7 xUnlock 5 xCheckReservedLock 0} + +do_test nolock-1.1 { + db close + forcedelete test.db + tvfs_reset + sqlite db file:test.db?nolock=0 -vfs tvfs -uri 1 + db eval {CREATE TABLE t1(a,b,c); INSERT INTO t1 VALUES(1,2,3);} + list xLock $::tvfs_calls(xLock) xUnlock $::tvfs_calls(xUnlock) \ + xCheckReservedLock $::tvfs_calls(xCheckReservedLock) +} {xLock 7 xUnlock 5 xCheckReservedLock 0} + +do_test nolock-1.2 { + db close + forcedelete test.db + tvfs_reset + sqlite db file:test.db?nolock=1 -vfs tvfs -uri 1 + db eval {CREATE TABLE t1(a,b,c); INSERT INTO t1 VALUES(1,2,3);} + list xLock $::tvfs_calls(xLock) xUnlock $::tvfs_calls(xUnlock) \ + xCheckReservedLock $::tvfs_calls(xCheckReservedLock) +} {xLock 0 xUnlock 0 xCheckReservedLock 0} + +do_test nolock-1.3 { + db close + tvfs_reset + sqlite db file:test.db?nolock=0 -vfs tvfs -uri 1 -readonly 1 + db eval {SELECT * FROM t1} + list xLock $::tvfs_calls(xLock) xUnlock $::tvfs_calls(xUnlock) \ + xCheckReservedLock $::tvfs_calls(xCheckReservedLock) +} {xLock 2 xUnlock 2 xCheckReservedLock 0} + +do_test nolock-1.4 { + db close + tvfs_reset + sqlite db file:test.db?nolock=1 -vfs tvfs -uri 1 -readonly 1 + db eval {SELECT * FROM t1} + list xLock $::tvfs_calls(xLock) xUnlock $::tvfs_calls(xUnlock) \ + xCheckReservedLock $::tvfs_calls(xCheckReservedLock) +} {xLock 0 xUnlock 0 xCheckReservedLock 0} + +############################################################################# +# Verify that immutable=1 disables both locking and xAccess calls to the +# journal files. +# +do_test nolock-2.0 { + db close + forcedelete test.db + # begin by creating a test database + sqlite3 db test.db + db eval { + CREATE TABLE t1(a,b); + INSERT INTO t1 VALUES('hello','world'); + CREATE TABLE t2(x,y); + INSERT INTO t2 VALUES(12345,67890); + SELECT * FROM t1, t2; + } +} {hello world 12345 67890} +do_test nolock-2.1 { + tvfs_reset + sqlite3 db2 test.db -vfs tvfs + db2 eval {SELECT * FROM t1, t2} +} {hello world 12345 67890} +do_test nolock-2.2 { + list xLock $::tvfs_calls(xLock) xUnlock $::tvfs_calls(xUnlock) \ + xCheckReservedLock $::tvfs_calls(xCheckReservedLock) \ + xAccess $::tvfs_calls(xAccess) +} {xLock 2 xUnlock 2 xCheckReservedLock 0 xAccess 4} + + +do_test nolock-2.11 { + db2 close + tvfs_reset + sqlite3 db2 file:test.db?immutable=0 -vfs tvfs -uri 1 + db2 eval {SELECT * FROM t1, t2} +} {hello world 12345 67890} +do_test nolock-2.12 { + list xLock $::tvfs_calls(xLock) xUnlock $::tvfs_calls(xUnlock) \ + xCheckReservedLock $::tvfs_calls(xCheckReservedLock) \ + xAccess $::tvfs_calls(xAccess) +} {xLock 2 xUnlock 2 xCheckReservedLock 0 xAccess 4} + + +do_test nolock-2.21 { + db2 close + tvfs_reset + sqlite3 db2 file:test.db?immutable=1 -vfs tvfs -uri 1 + db2 eval {SELECT * FROM t1, t2} +} {hello world 12345 67890} +do_test nolock-2.22 { + list xLock $::tvfs_calls(xLock) xUnlock $::tvfs_calls(xUnlock) \ + xCheckReservedLock $::tvfs_calls(xCheckReservedLock) \ + xAccess $::tvfs_calls(xAccess) +} {xLock 0 xUnlock 0 xCheckReservedLock 0 xAccess 0} + +do_test nolock-2.31 { + db2 close + tvfs_reset + sqlite3 db2 file:test.db?immutable=1 -vfs tvfs -uri 1 -readonly 1 + db2 eval {SELECT * FROM t1, t2} +} {hello world 12345 67890} +do_test nolock-2.32 { + list xLock $::tvfs_calls(xLock) xUnlock $::tvfs_calls(xUnlock) \ + xCheckReservedLock $::tvfs_calls(xCheckReservedLock) \ + xAccess $::tvfs_calls(xAccess) +} {xLock 0 xUnlock 0 xCheckReservedLock 0 xAccess 0} + +############################################################################ +# Verify that the SQLITE_IOCAP_IMMUTABLE flag works +# +do_test nolock-3.1 { + db2 close + tvfs devchar immutable + tvfs_reset + sqlite3 db2 test.db -vfs tvfs + db2 eval {SELECT * FROM t1, t2} +} {hello world 12345 67890} +do_test nolock-3.2 { + list xLock $::tvfs_calls(xLock) xUnlock $::tvfs_calls(xUnlock) \ + xCheckReservedLock $::tvfs_calls(xCheckReservedLock) \ + xAccess $::tvfs_calls(xAccess) +} {xLock 0 xUnlock 0 xCheckReservedLock 0 xAccess 0} + +do_test nolock-3.11 { + db2 close + tvfs_reset + sqlite3 db2 test.db -vfs tvfs -readonly 1 + db2 eval {SELECT * FROM t1, t2} +} {hello world 12345 67890} +do_test nolock-3.12 { + list xLock $::tvfs_calls(xLock) xUnlock $::tvfs_calls(xUnlock) \ + xCheckReservedLock $::tvfs_calls(xCheckReservedLock) \ + xAccess $::tvfs_calls(xAccess) +} {xLock 0 xUnlock 0 xCheckReservedLock 0 xAccess 0} + +db2 close +db close +tvfs delete +finish_test diff --git a/test/orderby5.test b/test/orderby5.test index bccd469f25..c9cce703ba 100644 --- a/test/orderby5.test +++ b/test/orderby5.test @@ -80,12 +80,12 @@ do_execsql_test 2.1a { EXPLAIN QUERY PLAN SELECT * FROM t2 WHERE a=0 ORDER BY a, b, c; } {~/B-TREE/} + do_execsql_test 2.1b { EXPLAIN QUERY PLAN - SELECT * FROM t1 WHERE a=0 ORDER BY a, b, c; + SELECT * FROM t1 WHERE likelihood(a=0, 0.05) ORDER BY a, b, c; } {/B-TREE/} - do_execsql_test 2.2 { EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE +a=0 ORDER BY a, b, c; diff --git a/test/permutations.test b/test/permutations.test index f7b955f5c5..ecfc8e532e 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -115,6 +115,7 @@ set allquicktests [test_set $alltests -exclude { incrvacuum_ioerr.test autovacuum_crash.test btree8.test shared_err.test vtab_err.test walslow.test walcrash.test walcrash3.test walthread.test rtree3.test indexfault.test securedel2.test + fts4growth.test fts4growth2.test }] if {[info exists ::env(QUICKTEST_INCLUDE)]} { set allquicktests [concat $allquicktests $::env(QUICKTEST_INCLUDE)] @@ -199,6 +200,7 @@ test_suite "fts3" -prefix "" -description { fts3corrupt2.test fts3first.test fts4langid.test fts4merge.test fts4check.test fts4unicode.test fts4noti.test fts3varint.test + fts4growth.test fts4growth2.test } test_suite "nofaultsim" -prefix "" -description { diff --git a/test/selectA.test b/test/selectA.test index ca2ec38da4..6e593e8e22 100644 --- a/test/selectA.test +++ b/test/selectA.test @@ -21,6 +21,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +set testprefix selectA ifcapable !compound { finish_test @@ -1310,4 +1311,68 @@ do_execsql_test selectA-3.98 { SELECT n FROM xyz ORDER BY +n; } {MAD MAD+ MAD++} +#------------------------------------------------------------------------- +# At one point the following code exposed a temp register reuse problem. +# +proc f {args} { return 1 } +db func f f + +do_execsql_test 4.1.1 { + CREATE TABLE t4(a, b); + CREATE TABLE t5(c, d); + + INSERT INTO t5 VALUES(1, 'x'); + INSERT INTO t5 VALUES(2, 'x'); + INSERT INTO t4 VALUES(3, 'x'); + INSERT INTO t4 VALUES(4, 'x'); + + CREATE INDEX i1 ON t4(a); + CREATE INDEX i2 ON t5(c); +} + +do_eqp_test 4.1.2 { + SELECT c, d FROM t5 + UNION ALL + SELECT a, b FROM t4 WHERE f()==f() + ORDER BY 1,2 +} { + 1 0 0 {SCAN TABLE t5 USING INDEX i2} + 1 0 0 {USE TEMP B-TREE FOR RIGHT PART OF ORDER BY} + 2 0 0 {SCAN TABLE t4 USING INDEX i1} + 2 0 0 {USE TEMP B-TREE FOR RIGHT PART OF ORDER BY} + 0 0 0 {COMPOUND SUBQUERIES 1 AND 2 (UNION ALL)} +} + +do_execsql_test 4.1.3 { + SELECT c, d FROM t5 + UNION ALL + SELECT a, b FROM t4 WHERE f()==f() + ORDER BY 1,2 +} { + 1 x 2 x 3 x 4 x +} + +do_execsql_test 4.2.1 { + CREATE TABLE t6(a, b); + CREATE TABLE t7(c, d); + + INSERT INTO t7 VALUES(2, 9); + INSERT INTO t6 VALUES(3, 0); + INSERT INTO t6 VALUES(4, 1); + INSERT INTO t7 VALUES(5, 6); + INSERT INTO t6 VALUES(6, 0); + INSERT INTO t7 VALUES(7, 6); + + CREATE INDEX i6 ON t6(a); + CREATE INDEX i7 ON t7(c); +} + +do_execsql_test 4.2.2 { + SELECT c, f(d,c,d,c,d) FROM t7 + UNION ALL + SELECT a, b FROM t6 + ORDER BY 1,2 +} {/2 . 3 . 4 . 5 . 6 . 7 ./} + + finish_test diff --git a/test/skipscan2.test b/test/skipscan2.test index e39b16ed27..27d193e94d 100644 --- a/test/skipscan2.test +++ b/test/skipscan2.test @@ -74,6 +74,7 @@ do_execsql_test skipscan2-1.4 { -- of a skip-scan. So make a manual adjustment to the stat1 table -- to make it seem like there are many more. UPDATE sqlite_stat1 SET stat='10000 5000 20' WHERE idx='people_idx1'; + UPDATE sqlite_stat1 SET stat='10000 1' WHERE idx='sqlite_autoindex_people_1'; ANALYZE sqlite_master; } db cache flush diff --git a/test/trace2.test b/test/trace2.test index 8f68d87585..562c70c538 100644 --- a/test/trace2.test +++ b/test/trace2.test @@ -136,6 +136,7 @@ ifcapable fts3 { "-- SELECT (SELECT max(idx) FROM 'main'.'x1_segdir' WHERE level = ?) + 1" "-- SELECT coalesce((SELECT max(blockid) FROM 'main'.'x1_segments') + 1, 1)" "-- REPLACE INTO 'main'.'x1_segdir' VALUES(?,?,?,?,?,?)" + "-- SELECT level, idx, end_block FROM 'main'.'x1_segdir' WHERE level BETWEEN ? AND ? ORDER BY level DESC, idx ASC" } do_trace_test 2.3 { diff --git a/test/unordered.test b/test/unordered.test index d8f7aa6add..147e91f0d9 100644 --- a/test/unordered.test +++ b/test/unordered.test @@ -42,7 +42,7 @@ foreach idxmode {ordered unordered} { 1 "SELECT * FROM t1 ORDER BY a" {0 0 0 {SCAN TABLE t1 USING INDEX i1}} {0 0 0 {SCAN TABLE t1} 0 0 0 {USE TEMP B-TREE FOR ORDER BY}} - 2 "SELECT * FROM t1 WHERE a >?" + 2 "SELECT * FROM t1 WHERE a > 100" {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a>?)}} {0 0 0 {SCAN TABLE t1}} 3 "SELECT * FROM t1 WHERE a = ? ORDER BY rowid" diff --git a/test/wal2.test b/test/wal2.test index b331d5ed10..9d45444d6a 100644 --- a/test/wal2.test +++ b/test/wal2.test @@ -811,7 +811,13 @@ do_test wal2-7.1.1 { do_test wal2-7.1.2 { forcecopy test.db test2.db forcecopy test.db-wal test2.db-wal - hexio_write test2.db-wal 48 FF + # The first 32 bytes of the WAL file contain the WAL header. Offset 48 + # is the first byte of the checksum for the first frame in the WAL. + # The following three lines replaces the contents of that byte with + # a different value. + set newval FF + if {$newval == [hexio_read test2.db-wal 48 1]} { set newval 00 } + hexio_write test2.db-wal 48 $newval } {1} do_test wal2-7.1.3 { sqlite3 db2 test2.db diff --git a/test/where3.test b/test/where3.test index 8fa9fa7840..c2804b5579 100644 --- a/test/where3.test +++ b/test/where3.test @@ -231,6 +231,7 @@ do_execsql_test where3-3.0 { CREATE TABLE t301(a INTEGER PRIMARY KEY,b,c); CREATE INDEX t301c ON t301(c); INSERT INTO t301 VALUES(1,2,3); + INSERT INTO t301 VALUES(2,2,3); CREATE TABLE t302(x, y); INSERT INTO t302 VALUES(4,5); ANALYZE; @@ -251,7 +252,7 @@ do_execsql_test where3-3.2 { } {} do_execsql_test where3-3.3 { SELECT * FROM t301 WHERE c=3 AND a IS NOT NULL; -} {1 2 3} +} {1 2 3 2 2 3} if 0 { # Query planner no longer does this # Verify that when there are multiple tables in a join which must be diff --git a/test/whereG.test b/test/whereG.test index 17d5653223..66918a35fa 100644 --- a/test/whereG.test +++ b/test/whereG.test @@ -14,6 +14,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +set testprefix whereG do_execsql_test whereG-1.0 { CREATE TABLE composer( @@ -179,5 +180,46 @@ do_execsql_test whereG-4.0 { ORDER BY x; } {right} +#------------------------------------------------------------------------- +# Test that likelihood() specifications on indexed terms are taken into +# account by various forms of loops. +# +# 5.1.*: open ended range scans +# 5.2.*: skip-scans +# +reset_db + +do_execsql_test 5.1 { + CREATE TABLE t1(a, b, c); + CREATE INDEX i1 ON t1(a, b); +} +do_eqp_test 5.1.2 { + SELECT * FROM t1 WHERE a>? +} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a>?)}} +do_eqp_test 5.1.3 { + SELECT * FROM t1 WHERE likelihood(a>?, 0.9) +} {0 0 0 {SCAN TABLE t1}} + +do_test 5.2 { + for {set i 0} {$i < 100} {incr i} { + execsql { INSERT INTO t1 VALUES('abc', $i, $i); } + } + execsql { INSERT INTO t1 SELECT 'def', b, c FROM t1; } + execsql { ANALYZE } +} {} +do_eqp_test 5.2.2 { + SELECT * FROM t1 WHERE likelihood(b>?, 0.01) +} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (ANY(a) AND b>?)}} +do_eqp_test 5.2.3 { + SELECT * FROM t1 WHERE likelihood(b>?, 0.9) +} {0 0 0 {SCAN TABLE t1}} + +do_eqp_test 5.3.1 { + SELECT * FROM t1 WHERE a=? +} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?)}} +do_eqp_test 5.3.2 { + SELECT * FROM t1 WHERE likelihood(a=?, 0.9) +} {0 0 0 {SCAN TABLE t1}} finish_test + diff --git a/tool/build-all-msvc.bat b/tool/build-all-msvc.bat index 6e0aeb572c..1fb61d4df8 100755 --- a/tool/build-all-msvc.bat +++ b/tool/build-all-msvc.bat @@ -146,6 +146,17 @@ IF NOT DEFINED CONFIGURATIONS ( %_VECHO% Configurations = '%CONFIGURATIONS%' +REM +REM NOTE: If the command used to invoke NMAKE is not already set, use the +REM default. +REM +IF NOT DEFINED NMAKE_CMD ( + SET NMAKE_CMD=nmake -B -f Makefile.msc +) + +%_VECHO% NmakeCmd = '%NMAKE_CMD%' +%_VECHO% NmakeArgs = '%NMAKE_ARGS%' + REM REM NOTE: Setup environment variables to translate between the MSVC platform REM names and the names to be used for the platform-specific binary @@ -238,6 +249,7 @@ GOTO set_vcvarsall_done :set_vcvarsall_phone SET VCVARSALL=%VCINSTALLDIR%\WPSDK\WP80\vcvarsphoneall.bat :set_vcvarsall_done +SET VCVARSALL=%VCVARSALL:\\=\% REM REM NOTE: This is the outer loop. There should be exactly one iteration per @@ -265,9 +277,11 @@ FOR %%P IN (%PLATFORMS%) DO ( REM and/or Visual Studio. This block may need to be updated in the REM future to account for additional environment variables. REM + CALL :fn_UnsetVariable CommandPromptType CALL :fn_UnsetVariable DevEnvDir CALL :fn_UnsetVariable ExtensionSdkDir CALL :fn_UnsetVariable Framework35Version + CALL :fn_UnsetVariable Framework40Version CALL :fn_UnsetVariable FrameworkDir CALL :fn_UnsetVariable FrameworkDir32 CALL :fn_UnsetVariable FrameworkVersion @@ -283,6 +297,8 @@ FOR %%P IN (%PLATFORMS%) DO ( CALL :fn_UnsetVariable WindowsSdkDir CALL :fn_UnsetVariable WindowsSdkDir_35 CALL :fn_UnsetVariable WindowsSdkDir_old + CALL :fn_UnsetVariable WindowsSDK_ExecutablePath_x86 + CALL :fn_UnsetVariable WindowsSDK_ExecutablePath_x64 REM REM NOTE: Reset the PATH here to the absolute bare minimum required. @@ -299,6 +315,8 @@ FOR %%P IN (%PLATFORMS%) DO ( REM environment variables to be picked up by the MSVC makefile REM itself. REM + %_AECHO% Building the %%B configuration for platform %%P with name %%D... + IF /I "%%B" == "Debug" ( SET DEBUG=2 SET MEMDEBUG=1 @@ -374,11 +392,12 @@ FOR %%P IN (%PLATFORMS%) DO ( REM REM NOTE: The Windows 8.1 SDK has a slightly different directory - REM naming convention. Currently, this tool assumes that - REM the Windows 8.1 SDK should only be used with MSVC 2013. + REM naming convention. REM - IF "%VisualStudioVersion%" == "12.0" ( + IF DEFINED USE_WINV63_NSDKLIBPATH ( CALL :fn_AppendVariable NSDKLIBPATH \lib\winv6.3\um\x86 + ) ELSE IF "%VisualStudioVersion%" == "12.0" ( + CALL :fn_AppendVariable NSDKLIBPATH \..\8.0\lib\win8\um\x86 ) ELSE ( CALL :fn_AppendVariable NSDKLIBPATH \lib\win8\um\x86 ) @@ -392,7 +411,7 @@ FOR %%P IN (%PLATFORMS%) DO ( REM file, etc. REM IF NOT DEFINED NOCLEAN ( - %__ECHO% nmake -f Makefile.msc clean + %__ECHO% %NMAKE_CMD% clean IF ERRORLEVEL 1 ( ECHO Failed to clean for platform %%P. @@ -404,6 +423,7 @@ FOR %%P IN (%PLATFORMS%) DO ( REM need to remove the build output for the files we are REM specifically wanting to build for each platform. REM + %_AECHO% Cleaning final output files only... %__ECHO% DEL /Q *.lo sqlite3.dll sqlite3.lib sqlite3.pdb ) @@ -414,7 +434,7 @@ FOR %%P IN (%PLATFORMS%) DO ( REM Also, disable looking for and/or linking to the native Tcl REM runtime library. REM - %__ECHO% nmake -f Makefile.msc sqlite3.dll XCOMPILE=1 USE_NATIVE_LIBPATHS=1 NO_TCL=1 %NMAKE_ARGS% + %__ECHO% %NMAKE_CMD% sqlite3.dll XCOMPILE=1 USE_NATIVE_LIBPATHS=1 NO_TCL=1 %NMAKE_ARGS% IF ERRORLEVEL 1 ( ECHO Failed to build %%B "sqlite3.dll" for platform %%P. diff --git a/tool/logest.c b/tool/logest.c index 1ac337d36c..347fa68a4f 100644 --- a/tool/logest.c +++ b/tool/logest.c @@ -83,7 +83,8 @@ static LogEst logEstFromDouble(double x){ LogEst e; assert( sizeof(x)==8 && sizeof(a)==8 ); if( x<=0.0 ) return -32768; - if( x<1.0 ) return -logEstFromDouble(1/x); + if( x<0.01 ) return -logEstFromDouble(1.0/x); + if( x<1.0 ) return logEstFromDouble(100.0*x) - 66; if( x<1024.0 ) return logEstFromInteger((sqlite3_uint64)(1024.0*x)) - 100; if( x<=2000000000.0 ) return logEstFromInteger((sqlite3_uint64)x); memcpy(&a, &x, 8); @@ -156,8 +157,10 @@ int main(int argc, char **argv){ } } for(i=n-1; i>=0; i--){ - if( a[i]<0 ){ + if( a[i]<-40 ){ printf("%5d (%f)\n", a[i], 1.0/(double)logEstToInt(-a[i])); + }else if( a[i]<10 ){ + printf("%5d (%f)\n", a[i], logEstToInt(a[i]+100)/1024.0); }else{ sqlite3_uint64 x = logEstToInt(a[i]+100)*100/1024; printf("%5d (%lld.%02lld)\n", a[i], x/100, x%100); diff --git a/tool/mksqlite3c-noext.tcl b/tool/mksqlite3c-noext.tcl index 017ad6292f..ecb9cb0439 100644 --- a/tool/mksqlite3c-noext.tcl +++ b/tool/mksqlite3c-noext.tcl @@ -99,6 +99,8 @@ foreach hdr { mutex.h opcodes.h os_common.h + os_setup.h + os_win.h os.h pager.h parse.h diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl index 96497c8609..11e0f05f7d 100644 --- a/tool/mksqlite3c.tcl +++ b/tool/mksqlite3c.tcl @@ -103,6 +103,8 @@ foreach hdr { mutex.h opcodes.h os_common.h + os_setup.h + os_win.h os.h pager.h parse.h @@ -170,7 +172,9 @@ proc copy_file {filename} { if {$linemacros} {puts $out "#line [expr {$ln+1}] \"$filename\""} } } elseif {![info exists seen_hdr($hdr)]} { - set seen_hdr($hdr) 1 + if {![regexp {/\*\s+amalgamator:\s+dontcache\s+\*/} $line]} { + set seen_hdr($hdr) 1 + } puts $out $line } elseif {[regexp {/\*\s+amalgamator:\s+keep\s+\*/} $line]} { # This include file must be kept because there was a "keep" diff --git a/tool/mksqlite3internalh.tcl b/tool/mksqlite3internalh.tcl index 406ef5c457..7e92b3ad7d 100644 --- a/tool/mksqlite3internalh.tcl +++ b/tool/mksqlite3internalh.tcl @@ -60,6 +60,8 @@ foreach hdr { keywordhash.h opcodes.h os_common.h + os_setup.h + os_win.h os.h pager.h parse.h diff --git a/tool/mkvsix.tcl b/tool/mkvsix.tcl index 65fa731245..208ce2b142 100644 --- a/tool/mkvsix.tcl +++ b/tool/mkvsix.tcl @@ -65,12 +65,16 @@ # argument is optional and if present must contain the name of the directory # containing the root of the source tree for SQLite. The third argument is # optional and if present must contain the flavor the VSIX package to build. -# Currently, the only supported package flavors are "WinRT", "WinRT81", and -# "WP80". The fourth argument is optional and if present must be a string -# containing a list of platforms to include in the VSIX package. The format -# of the platform list string is "platform1,platform2,platform3". Typically, -# when on Windows, this script is executed using commands similar to the -# following from a normal Windows command prompt: +# Currently, the only supported package flavors are "WinRT", "WinRT81", "WP80", +# "WP81", and "Win32". The fourth argument is optional and if present must be +# a string containing a list of platforms to include in the VSIX package. The +# platform list is "platform1,platform2,platform3". The fifth argument is +# optional and if present must contain the version of Visual Studio required by +# the package. Currently, the only supported versions are "2012" and "2013". +# The package flavors "WinRT81" and "WP81" are only supported when the Visual +# Studio version is "2013". Typically, when on Windows, this script is +# executed using commands similar to the following from a normal Windows +# command prompt: # # CD /D C:\dev\sqlite\core # tclsh85 tool\mkvsix.tcl C:\Temp @@ -100,7 +104,7 @@ proc fail { {error ""} {usage false} } { puts stdout "usage:\ [file tail [info nameofexecutable]]\ [file tail [info script]] \[sourceDirectory\]\ -\[packageFlavor\] \[platformNames\]" +\[packageFlavor\] \[platformNames\] \[vsVersion\]" exit 1 } @@ -170,13 +174,81 @@ proc writeFile { fileName data } { return "" } -proc substFile { fileName } { +proc getMinVsVersionXmlChunk { vsVersion } { + switch -exact $vsVersion { + 2012 { + return [appendArgs \ + "\r\n " {MinVSVersion="11.0"}] + } + 2013 { + return [appendArgs \ + "\r\n " {MinVSVersion="12.0"}] + } + default { + return "" + } + } +} + +proc getMaxPlatformVersionXmlChunk { packageFlavor vsVersion } { # - # NOTE: Performs all Tcl command, variable, and backslash substitutions in - # the specified file and then rewrites the contents of that same file - # with the substituted data. + # NOTE: Only Visual Studio 2013 supports this SDK manifest attribute. # - return [writeFile $fileName [uplevel 1 [list subst [readFile $fileName]]]] + if {![string equal $vsVersion 2013]} then { + return "" + } + + switch -exact $packageFlavor { + WinRT { + return [appendArgs \ + "\r\n " {MaxPlatformVersion="8.0"}] + } + WinRT81 { + return [appendArgs \ + "\r\n " {MaxPlatformVersion="8.1"}] + } + WP80 { + return [appendArgs \ + "\r\n " {MaxPlatformVersion="8.0"}] + } + WP81 { + return [appendArgs \ + "\r\n " {MaxPlatformVersion="8.1"}] + } + default { + return "" + } + } +} + +proc getExtraFileListXmlChunk { packageFlavor vsVersion } { + # + # NOTE: Windows Phone 8.0 does not require any extra attributes in its VSIX + # package SDK manifests; however, it appears that Windows Phone 8.1 + # does. + # + if {[string equal $packageFlavor WP80]} then { + return "" + } + + set appliesTo [expr {[string equal $packageFlavor Win32] ? \ + "VisualC" : "WindowsAppContainer"}] + + switch -exact $vsVersion { + 2012 { + return [appendArgs \ + "\r\n " AppliesTo=\" $appliesTo \" \ + "\r\n " {DependsOn="Microsoft.VCLibs, version=11.0"}] + } + 2013 { + return [appendArgs \ + "\r\n " AppliesTo=\" $appliesTo \" \ + "\r\n " {DependsOn="Microsoft.VCLibs, version=12.0"}] + } + default { + return "" + } + } } proc replaceFileNameTokens { fileName name buildName platformName } { @@ -188,6 +260,15 @@ proc replaceFileNameTokens { fileName name buildName platformName } { $name] $fileName] } +proc substFile { fileName } { + # + # NOTE: Performs all Tcl command, variable, and backslash substitutions in + # the specified file and then rewrites the contents of that same file + # with the substituted data. + # + return [writeFile $fileName [uplevel 1 [list subst [readFile $fileName]]]] +} + # # NOTE: This is the entry point for this script. # @@ -206,7 +287,7 @@ set rootName [file rootname [file tail $script]] # NOTE: Process and verify all the command line arguments. # set argc [llength $argv] -if {$argc < 1 || $argc > 4} then {fail} +if {$argc < 1 || $argc > 5} then {fail} set binaryDirectory [lindex $argv 0] @@ -251,58 +332,123 @@ if {[string length $packageFlavor] == 0} then { fail "invalid package flavor" } -if {[string equal -nocase $packageFlavor WinRT]} then { - set shortName SQLite.WinRT - set displayName "SQLite for Windows Runtime" - set targetPlatformIdentifier Windows - set targetPlatformVersion v8.0 - set minVsVersion 11.0 - set extraSdkPath "" - set extraFileListAttributes [appendArgs \ - "\r\n " {AppliesTo="WindowsAppContainer"} \ - "\r\n " {DependsOn="Microsoft.VCLibs, version=11.0"}] -} elseif {[string equal -nocase $packageFlavor WinRT81]} then { - set shortName SQLite.WinRT81 - set displayName "SQLite for Windows Runtime (Windows 8.1)" - set targetPlatformIdentifier Windows - set targetPlatformVersion v8.1 - set minVsVersion 12.0 - set extraSdkPath "" - set extraFileListAttributes [appendArgs \ - "\r\n " {AppliesTo="WindowsAppContainer"} \ - "\r\n " {DependsOn="Microsoft.VCLibs, version=12.0"}] -} elseif {[string equal -nocase $packageFlavor WP80]} then { - set shortName SQLite.WP80 - set displayName "SQLite for Windows Phone" - set targetPlatformIdentifier "Windows Phone" - set targetPlatformVersion v8.0 - set minVsVersion 11.0 - set extraSdkPath "\\..\\$targetPlatformIdentifier" - set extraFileListAttributes "" -} elseif {[string equal -nocase $packageFlavor Win32]} then { - set shortName SQLite.Win32 - set displayName "SQLite for Windows" - set targetPlatformIdentifier Windows - set targetPlatformVersion v8.0 - set minVsVersion 11.0 - set extraSdkPath "" - set extraFileListAttributes [appendArgs \ - "\r\n " {AppliesTo="VisualC"} \ - "\r\n " {DependsOn="Microsoft.VCLibs, version=11.0"}] -} else { - fail "unsupported package flavor, must be one of: WinRT WinRT81 WP80 Win32" -} - if {$argc >= 4} then { set platformNames [list] foreach platformName [split [lindex $argv 3] ", "] { + set platformName [string trim $platformName] + if {[string length $platformName] > 0} then { lappend platformNames $platformName } } } +if {$argc >= 5} then { + set vsVersion [lindex $argv 4] +} else { + set vsVersion 2012 +} + +if {[string length $vsVersion] == 0} then { + fail "invalid Visual Studio version" +} + +if {![string equal $vsVersion 2012] && ![string equal $vsVersion 2013]} then { + fail [appendArgs \ + "unsupported Visual Studio version, must be one of: " \ + [list 2012 2013]] +} + +set shortNames(WinRT,2012) SQLite.WinRT +set shortNames(WinRT,2013) SQLite.WinRT.2013 +set shortNames(WinRT81,2013) SQLite.WinRT81 +set shortNames(WP80,2012) SQLite.WP80 +set shortNames(WP80,2013) SQLite.WP80.2013 +set shortNames(WP81,2013) SQLite.WP81 +set shortNames(Win32,2012) SQLite.Win32 +set shortNames(Win32,2013) SQLite.Win32.2013 + +set displayNames(WinRT,2012) "SQLite for Windows Runtime" +set displayNames(WinRT,2013) "SQLite for Windows Runtime" +set displayNames(WinRT81,2013) "SQLite for Windows Runtime (Windows 8.1)" +set displayNames(WP80,2012) "SQLite for Windows Phone" +set displayNames(WP80,2013) "SQLite for Windows Phone" +set displayNames(WP81,2013) "SQLite for Windows Phone 8.1" +set displayNames(Win32,2012) "SQLite for Windows" +set displayNames(Win32,2013) "SQLite for Windows" + +if {[string equal $packageFlavor WinRT]} then { + set shortName $shortNames($packageFlavor,$vsVersion) + set displayName $displayNames($packageFlavor,$vsVersion) + set targetPlatformIdentifier Windows + set targetPlatformVersion v8.0 + set minVsVersion [getMinVsVersionXmlChunk $vsVersion] + set maxPlatformVersion \ + [getMaxPlatformVersionXmlChunk $packageFlavor $vsVersion] + set extraSdkPath "" + set extraFileListAttributes \ + [getExtraFileListXmlChunk $packageFlavor $vsVersion] +} elseif {[string equal $packageFlavor WinRT81]} then { + if {$vsVersion ne "2013"} then { + fail [appendArgs \ + "unsupported combination, package flavor " $packageFlavor \ + " is only supported with Visual Studio 2013"] + } + set shortName $shortNames($packageFlavor,$vsVersion) + set displayName $displayNames($packageFlavor,$vsVersion) + set targetPlatformIdentifier Windows + set targetPlatformVersion v8.1 + set minVsVersion [getMinVsVersionXmlChunk $vsVersion] + set maxPlatformVersion \ + [getMaxPlatformVersionXmlChunk $packageFlavor $vsVersion] + set extraSdkPath "" + set extraFileListAttributes \ + [getExtraFileListXmlChunk $packageFlavor $vsVersion] +} elseif {[string equal $packageFlavor WP80]} then { + set shortName $shortNames($packageFlavor,$vsVersion) + set displayName $displayNames($packageFlavor,$vsVersion) + set targetPlatformIdentifier "Windows Phone" + set targetPlatformVersion v8.0 + set minVsVersion [getMinVsVersionXmlChunk $vsVersion] + set maxPlatformVersion \ + [getMaxPlatformVersionXmlChunk $packageFlavor $vsVersion] + set extraSdkPath "\\..\\$targetPlatformIdentifier" + set extraFileListAttributes \ + [getExtraFileListXmlChunk $packageFlavor $vsVersion] +} elseif {[string equal $packageFlavor WP81]} then { + if {$vsVersion ne "2013"} then { + fail [appendArgs \ + "unsupported combination, package flavor " $packageFlavor \ + " is only supported with Visual Studio 2013"] + } + set shortName $shortNames($packageFlavor,$vsVersion) + set displayName $displayNames($packageFlavor,$vsVersion) + set targetPlatformIdentifier WindowsPhoneApp + set targetPlatformVersion v8.1 + set minVsVersion [getMinVsVersionXmlChunk $vsVersion] + set maxPlatformVersion \ + [getMaxPlatformVersionXmlChunk $packageFlavor $vsVersion] + set extraSdkPath "\\..\\$targetPlatformIdentifier" + set extraFileListAttributes \ + [getExtraFileListXmlChunk $packageFlavor $vsVersion] +} elseif {[string equal $packageFlavor Win32]} then { + set shortName $shortNames($packageFlavor,$vsVersion) + set displayName $displayNames($packageFlavor,$vsVersion) + set targetPlatformIdentifier Windows + set targetPlatformVersion v8.0 + set minVsVersion [getMinVsVersionXmlChunk $vsVersion] + set maxPlatformVersion \ + [getMaxPlatformVersionXmlChunk $packageFlavor $vsVersion] + set extraSdkPath "" + set extraFileListAttributes \ + [getExtraFileListXmlChunk $packageFlavor $vsVersion] +} else { + fail [appendArgs \ + "unsupported package flavor, must be one of: " \ + [list WinRT WinRT81 WP80 WP81 Win32]] +} + ############################################################################### # @@ -490,7 +636,7 @@ if {![info exists buildNames]} then { # overridden via the command line or the user-specific customizations # file. # -if {![info exists platformNames]} then { +if {![info exists platformNames] || [llength $platformNames] == 0} then { set platformNames [list x86 x64 ARM] } diff --git a/tool/win/sqlite.vsix b/tool/win/sqlite.vsix index ac4afb3f4b..1450011266 100644 Binary files a/tool/win/sqlite.vsix and b/tool/win/sqlite.vsix differ