From 8bc52629150345f7b05a781944f911c18e311749 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 25 Nov 2013 09:36:07 +0000 Subject: [PATCH 01/71] Initial work on isolating usage of the Windows header file. FossilOrigin-Name: 0d42c6b830ee50f45539d73e3948ed3bc50a11d1 --- Makefile.in | 2 ++ Makefile.msc | 2 ++ Makefile.vxworks | 2 ++ main.mk | 2 ++ manifest | 36 +++++++++++--------- manifest.uuid | 2 +- src/mutex_w32.c | 5 +++ src/os.h | 43 ------------------------ src/os_win.c | 10 +++--- src/os_win.h | 67 +++++++++++++++++++++++++++++++++++++ tool/mksqlite3c-noext.tcl | 1 + tool/mksqlite3c.tcl | 1 + tool/mksqlite3internalh.tcl | 1 + 13 files changed, 109 insertions(+), 65 deletions(-) create mode 100644 src/os_win.h diff --git a/Makefile.in b/Makefile.in index a0217c4343..d96d30ae86 100644 --- a/Makefile.in +++ b/Makefile.in @@ -241,6 +241,7 @@ SRC = \ $(TOP)/src/os_common.h \ $(TOP)/src/os_unix.c \ $(TOP)/src/os_win.c \ + $(TOP)/src/os_win.h \ $(TOP)/src/pager.c \ $(TOP)/src/pager.h \ $(TOP)/src/parse.y \ @@ -457,6 +458,7 @@ HDR = \ opcodes.h \ $(TOP)/src/os.h \ $(TOP)/src/os_common.h \ + $(TOP)/src/os_win.h \ $(TOP)/src/pager.h \ $(TOP)/src/pcache.h \ parse.h \ diff --git a/Makefile.msc b/Makefile.msc index 437cb0eac3..afe66baad8 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -628,6 +628,7 @@ SRC = \ $(TOP)\src\os_common.h \ $(TOP)\src\os_unix.c \ $(TOP)\src\os_win.c \ + $(TOP)\src\os_win.h \ $(TOP)\src\pager.c \ $(TOP)\src\pager.h \ $(TOP)\src\parse.y \ @@ -847,6 +848,7 @@ HDR = \ opcodes.h \ $(TOP)\src\os.h \ $(TOP)\src\os_common.h \ + $(TOP)\src\os_win.h \ $(TOP)\src\pager.h \ $(TOP)\src\pcache.h \ parse.h \ diff --git a/Makefile.vxworks b/Makefile.vxworks index c9058da3e5..70b9bdde6b 100644 --- a/Makefile.vxworks +++ b/Makefile.vxworks @@ -264,6 +264,7 @@ SRC = \ $(TOP)/src/os_common.h \ $(TOP)/src/os_unix.c \ $(TOP)/src/os_win.c \ + $(TOP)/src/os_win.h \ $(TOP)/src/pager.c \ $(TOP)/src/pager.h \ $(TOP)/src/parse.y \ @@ -416,6 +417,7 @@ HDR = \ opcodes.h \ $(TOP)/src/os.h \ $(TOP)/src/os_common.h \ + $(TOP)/src/os_win.h \ $(TOP)/src/pager.h \ $(TOP)/src/pcache.h \ parse.h \ diff --git a/main.mk b/main.mk index 41d1743832..4992de549b 100644 --- a/main.mk +++ b/main.mk @@ -123,6 +123,7 @@ SRC = \ $(TOP)/src/os_common.h \ $(TOP)/src/os_unix.c \ $(TOP)/src/os_win.c \ + $(TOP)/src/os_win.h \ $(TOP)/src/pager.c \ $(TOP)/src/pager.h \ $(TOP)/src/parse.y \ @@ -339,6 +340,7 @@ HDR = \ opcodes.h \ $(TOP)/src/os.h \ $(TOP)/src/os_common.h \ + $(TOP)/src/os_win.h \ $(TOP)/src/pager.h \ $(TOP)/src/pcache.h \ parse.h \ diff --git a/manifest b/manifest index aa2b98cdb9..41499ad4d0 100644 --- a/manifest +++ b/manifest @@ -1,10 +1,10 @@ -C Add\sthe\s--scratch\sparameter\sto\sspeedtest1.\s\sImproved\serror\smessages\swhen\nmisconfiguring\smemory\sparameters\sin\sspeedtest1. -D 2013-11-24T01:14:14.123 +C Initial\swork\son\sisolating\susage\sof\sthe\sWindows\sheader\sfile. +D 2013-11-25T09:36:07.707 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in e1a9b4258bbde53f5636f4e238c65b7e11459e2b +F Makefile.in 16254fb023b7e69788013f0da5ccdc2063ba2328 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc a6247094079f3ef97dcd7a69d0e13ff718c7073e -F Makefile.vxworks db21ed42a01d5740e656b16f92cb5d8d5e5dd315 +F Makefile.msc 7e91dccedae82c45dbf91a4e8240c642c2b90c83 +F Makefile.vxworks f921bd6a0600e36d67da080d820ae712ab393f30 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 F VERSION 52f7e22bfcec71a462e34194b4ae1671380fde59 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -142,7 +142,7 @@ F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt f439556c5ce01ced70987e5ee86549a45165d9ff -F main.mk 82fd90375561d7b66287ae5a8b09e1e027394019 +F main.mk c9b6d61c55f461543244d63992c63037c3f11dac F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh d2a981497b404d6498f5ff3e3b1f3816bdfcb338 @@ -200,13 +200,14 @@ F src/mutex.c d3b66a569368015e0fcb1ac15f81c119f504d3bc F src/mutex.h 5bc526e19dccc412b7ff04642f6fdad3fdfdabea F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553 F src/mutex_unix.c c3a4e00f96ba068a8dbef34084465979aaf369cc -F src/mutex_w32.c 6108c88e1cb38d8fbb3534b170793815cbedbf97 +F src/mutex_w32.c ab08c0fc54b71979370ca7c8f42fc64a9f211ebb F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c b4ad71336fd96f97776f75587cd9e8218288f5be -F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f +F src/os.h cd8fd6545d6a936bae05026136324ec8e620c7f8 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 143624d9eabb3b997c59cf594e0d06c56edd43e9 -F src/os_win.c ef091b347d682cb24fc575ac9a6290341af62e2b +F src/os_win.c 423d8f815067f4980cb84abab501374afc9bd039 +F src/os_win.h db1d1d18f56ea2931ba1c2ec25ff4616ae1e581b F src/pager.c 2aa4444ffe86e9282d03bc349a4a5e49bd77c0e8 F src/pager.h f094af9f6ececfaa8a1e93876905a4f34233fb0c F src/parse.y acee1a9958539e21263362b194594c5255ad2fca @@ -1111,10 +1112,10 @@ F tool/mkkeywordhash.c 189d76644e373c7d0864c628deb8ce7b4f403591 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e F tool/mkpragmatab.tcl 78a77b2c554d534c6f2dc903130186ed15715460 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 -F tool/mksqlite3c-noext.tcl 8bce31074e4cbe631bb7676526a048335f4c9f02 -F tool/mksqlite3c.tcl e2ba20d3f690990079d17f3e5a7417dfb7ada543 +F tool/mksqlite3c-noext.tcl 1407684f6781496421be53841f1259303f0c8be9 +F tool/mksqlite3c.tcl c6b22da4d4ee272afda33f0aeb896d9c2e26dc95 F tool/mksqlite3h.tcl ba24038056f51fde07c0079c41885ab85e2cff12 -F tool/mksqlite3internalh.tcl 3dca7bb5374cee003379b8cbac73714f610ef795 +F tool/mksqlite3internalh.tcl ba0532989220eaef044cbb8c0108eca623d23bbc F tool/mkvsix.tcl 6477fb9dab838b7eea1eed50658ff1cda04850b5 F tool/offsets.c fe4262fdfa378e8f5499a42136d17bf3b98f6091 F tool/omittest.tcl 4665982e95a6e5c1bd806cf7bc3dea95be422d77 @@ -1142,7 +1143,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 12e612e8e7c4a6f83acf0daf5608151fb5ec1575 -R d231270eb8540c71c1fe43ad7f945ecf -U drh -Z 904197ecb3cb4ac430a652007d0a071c +P 8f3c767a30c552548ead104ca125f182ce4849ad +R e592d18e44da45424fc95ed6d55c4c01 +T *branch * winHdr +T *sym-winHdr * +T -sym-trunk * +U mistachkin +Z 4118ce5841a2a285e11b8bcc9d8fe99e diff --git a/manifest.uuid b/manifest.uuid index 7967f97b83..586356026f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8f3c767a30c552548ead104ca125f182ce4849ad \ No newline at end of file +0d42c6b830ee50f45539d73e3948ed3bc50a11d1 \ No newline at end of file diff --git a/src/mutex_w32.c b/src/mutex_w32.c index 4b88c17452..2eb852413e 100644 --- a/src/mutex_w32.c +++ b/src/mutex_w32.c @@ -19,6 +19,11 @@ */ #ifdef SQLITE_MUTEX_W32 +/* +** Include the header file for the Windows VFS. +*/ +#include "os_win.h" + /* ** Each recursive mutex is an instance of the following structure. */ diff --git a/src/os.h b/src/os.h index 070a2ddd17..1d438d22cd 100644 --- a/src/os.h +++ b/src/os.h @@ -56,49 +56,6 @@ # endif #endif -#if SQLITE_OS_WIN -# include -#endif - -/* -** Determine if we are dealing with Windows NT. -** -** We ought to be able to determine if we are compiling for win98 or winNT -** using the _WIN32_WINNT macro as follows: -** -** #if defined(_WIN32_WINNT) -** # define SQLITE_OS_WINNT 1 -** #else -** # define SQLITE_OS_WINNT 0 -** #endif -** -** However, vs2005 does not set _WIN32_WINNT by default, as it ought to, -** so the above test does not work. We'll just assume that everything is -** winNT unless the programmer explicitly says otherwise by setting -** SQLITE_OS_WINNT to 0. -*/ -#if SQLITE_OS_WIN && !defined(SQLITE_OS_WINNT) -# define SQLITE_OS_WINNT 1 -#endif - -/* -** Determine if we are dealing with WindowsCE - which has a much -** reduced API. -*/ -#if defined(_WIN32_WCE) -# define SQLITE_OS_WINCE 1 -#else -# define SQLITE_OS_WINCE 0 -#endif - -/* -** Determine if we are dealing with WinRT, which provides only a subset of -** the full Win32 API. -*/ -#if !defined(SQLITE_OS_WINRT) -# define SQLITE_OS_WINRT 0 -#endif - /* If the SET_FULLSYNC macro is not defined above, then make it ** a no-op */ diff --git a/src/os_win.c b/src/os_win.c index a2f5513a2b..343f0f01f0 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -15,16 +15,16 @@ #include "sqliteInt.h" #if SQLITE_OS_WIN /* This file is used for Windows only */ -#ifdef __CYGWIN__ -# include -# include /* amalgamator: keep */ -#endif - /* ** Include code that is common to all os_*.c files */ #include "os_common.h" +/* +** Include the header file for the Windows VFS. +*/ +#include "os_win.h" + /* ** Compiling and using WAL mode requires several APIs that are only ** available in Windows platforms based on the NT kernel. diff --git a/src/os_win.h b/src/os_win.h new file mode 100644 index 0000000000..15e53b1789 --- /dev/null +++ b/src/os_win.h @@ -0,0 +1,67 @@ +/* +** 2013 November 25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code that is specific to Windows. +*/ +#ifndef _OS_WIN_H_ +#define _OS_WIN_H_ + +/* +** Include the Windows SDK header file. +*/ +#include "windows.h" + +#ifdef __CYGWIN__ +# include +# include /* amalgamator: keep */ +#endif + +/* +** Determine if we are dealing with Windows NT. +** +** We ought to be able to determine if we are compiling for win98 or winNT +** using the _WIN32_WINNT macro as follows: +** +** #if defined(_WIN32_WINNT) +** # define SQLITE_OS_WINNT 1 +** #else +** # define SQLITE_OS_WINNT 0 +** #endif +** +** However, vs2005 does not set _WIN32_WINNT by default, as it ought to, +** so the above test does not work. We'll just assume that everything is +** winNT unless the programmer explicitly says otherwise by setting +** SQLITE_OS_WINNT to 0. +*/ +#if SQLITE_OS_WIN && !defined(SQLITE_OS_WINNT) +# define SQLITE_OS_WINNT 1 +#endif + +/* +** Determine if we are dealing with WindowsCE - which has a much +** reduced API. +*/ +#if defined(_WIN32_WCE) +# define SQLITE_OS_WINCE 1 +#else +# define SQLITE_OS_WINCE 0 +#endif + +/* +** Determine if we are dealing with WinRT, which provides only a subset of +** the full Win32 API. +*/ +#if !defined(SQLITE_OS_WINRT) +# define SQLITE_OS_WINRT 0 +#endif + +#endif /* _OS_WIN_H_ */ diff --git a/tool/mksqlite3c-noext.tcl b/tool/mksqlite3c-noext.tcl index 017ad6292f..02d9ae5469 100644 --- a/tool/mksqlite3c-noext.tcl +++ b/tool/mksqlite3c-noext.tcl @@ -99,6 +99,7 @@ foreach hdr { mutex.h opcodes.h os_common.h + os_win.h os.h pager.h parse.h diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl index de9b48b97b..e565bcbfe3 100644 --- a/tool/mksqlite3c.tcl +++ b/tool/mksqlite3c.tcl @@ -103,6 +103,7 @@ foreach hdr { mutex.h opcodes.h os_common.h + os_win.h os.h pager.h parse.h diff --git a/tool/mksqlite3internalh.tcl b/tool/mksqlite3internalh.tcl index 406ef5c457..3938316eec 100644 --- a/tool/mksqlite3internalh.tcl +++ b/tool/mksqlite3internalh.tcl @@ -60,6 +60,7 @@ foreach hdr { keywordhash.h opcodes.h os_common.h + os_win.h os.h pager.h parse.h From f3a4c7c2c7159bd7e2d438babc73bb46f821c724 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 25 Nov 2013 09:37:24 +0000 Subject: [PATCH 02/71] Correct line-endings in the new header file. FossilOrigin-Name: 94219b9f1ff59ea408c7ded167ee28b00c0f38c1 --- manifest | 15 +++--- manifest.uuid | 2 +- src/os_win.h | 134 +++++++++++++++++++++++++------------------------- 3 files changed, 74 insertions(+), 77 deletions(-) diff --git a/manifest b/manifest index 41499ad4d0..e8fa1b9203 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Initial\swork\son\sisolating\susage\sof\sthe\sWindows\sheader\sfile. -D 2013-11-25T09:36:07.707 +C Correct\sline-endings\sin\sthe\snew\sheader\sfile. +D 2013-11-25T09:37:24.023 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 16254fb023b7e69788013f0da5ccdc2063ba2328 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -207,7 +207,7 @@ F src/os.h cd8fd6545d6a936bae05026136324ec8e620c7f8 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 143624d9eabb3b997c59cf594e0d06c56edd43e9 F src/os_win.c 423d8f815067f4980cb84abab501374afc9bd039 -F src/os_win.h db1d1d18f56ea2931ba1c2ec25ff4616ae1e581b +F src/os_win.h ac253253d080dc3dd261efe5e98626c4b95e5a26 F src/pager.c 2aa4444ffe86e9282d03bc349a4a5e49bd77c0e8 F src/pager.h f094af9f6ececfaa8a1e93876905a4f34233fb0c F src/parse.y acee1a9958539e21263362b194594c5255ad2fca @@ -1143,10 +1143,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 8f3c767a30c552548ead104ca125f182ce4849ad -R e592d18e44da45424fc95ed6d55c4c01 -T *branch * winHdr -T *sym-winHdr * -T -sym-trunk * +P 0d42c6b830ee50f45539d73e3948ed3bc50a11d1 +R 0e6472b71f09f9a7891e918b85232304 U mistachkin -Z 4118ce5841a2a285e11b8bcc9d8fe99e +Z c441f2886dac757b12ddb73cf4ef6f57 diff --git a/manifest.uuid b/manifest.uuid index 586356026f..910f42306d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0d42c6b830ee50f45539d73e3948ed3bc50a11d1 \ No newline at end of file +94219b9f1ff59ea408c7ded167ee28b00c0f38c1 \ No newline at end of file diff --git a/src/os_win.h b/src/os_win.h index 15e53b1789..6ccd9f4828 100644 --- a/src/os_win.h +++ b/src/os_win.h @@ -1,67 +1,67 @@ -/* -** 2013 November 25 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This file contains code that is specific to Windows. -*/ -#ifndef _OS_WIN_H_ -#define _OS_WIN_H_ - -/* -** Include the Windows SDK header file. -*/ -#include "windows.h" - -#ifdef __CYGWIN__ -# include -# include /* amalgamator: keep */ -#endif - -/* -** Determine if we are dealing with Windows NT. -** -** We ought to be able to determine if we are compiling for win98 or winNT -** using the _WIN32_WINNT macro as follows: -** -** #if defined(_WIN32_WINNT) -** # define SQLITE_OS_WINNT 1 -** #else -** # define SQLITE_OS_WINNT 0 -** #endif -** -** However, vs2005 does not set _WIN32_WINNT by default, as it ought to, -** so the above test does not work. We'll just assume that everything is -** winNT unless the programmer explicitly says otherwise by setting -** SQLITE_OS_WINNT to 0. -*/ -#if SQLITE_OS_WIN && !defined(SQLITE_OS_WINNT) -# define SQLITE_OS_WINNT 1 -#endif - -/* -** Determine if we are dealing with WindowsCE - which has a much -** reduced API. -*/ -#if defined(_WIN32_WCE) -# define SQLITE_OS_WINCE 1 -#else -# define SQLITE_OS_WINCE 0 -#endif - -/* -** Determine if we are dealing with WinRT, which provides only a subset of -** the full Win32 API. -*/ -#if !defined(SQLITE_OS_WINRT) -# define SQLITE_OS_WINRT 0 -#endif - -#endif /* _OS_WIN_H_ */ +/* +** 2013 November 25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code that is specific to Windows. +*/ +#ifndef _OS_WIN_H_ +#define _OS_WIN_H_ + +/* +** Include the Windows SDK header file. +*/ +#include "windows.h" + +#ifdef __CYGWIN__ +# include +# include /* amalgamator: keep */ +#endif + +/* +** Determine if we are dealing with Windows NT. +** +** We ought to be able to determine if we are compiling for win98 or winNT +** using the _WIN32_WINNT macro as follows: +** +** #if defined(_WIN32_WINNT) +** # define SQLITE_OS_WINNT 1 +** #else +** # define SQLITE_OS_WINNT 0 +** #endif +** +** However, vs2005 does not set _WIN32_WINNT by default, as it ought to, +** so the above test does not work. We'll just assume that everything is +** winNT unless the programmer explicitly says otherwise by setting +** SQLITE_OS_WINNT to 0. +*/ +#if SQLITE_OS_WIN && !defined(SQLITE_OS_WINNT) +# define SQLITE_OS_WINNT 1 +#endif + +/* +** Determine if we are dealing with WindowsCE - which has a much +** reduced API. +*/ +#if defined(_WIN32_WCE) +# define SQLITE_OS_WINCE 1 +#else +# define SQLITE_OS_WINCE 0 +#endif + +/* +** Determine if we are dealing with WinRT, which provides only a subset of +** the full Win32 API. +*/ +#if !defined(SQLITE_OS_WINRT) +# define SQLITE_OS_WINRT 0 +#endif + +#endif /* _OS_WIN_H_ */ From f74b9e0965bdf30ca8911ca28a5f7ad6e11d6b22 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 26 Nov 2013 01:00:31 +0000 Subject: [PATCH 03/71] Further work on Windows header file reform. FossilOrigin-Name: 540f5525160b516de282136ca77f909115a51630 --- Makefile.in | 2 ++ Makefile.msc | 2 ++ Makefile.vxworks | 2 ++ main.mk | 2 ++ manifest | 37 ++++++++++++------------ manifest.uuid | 2 +- src/os.h | 36 ++---------------------- src/os_setup.h | 56 +++++++++++++++++++++++++++++++++++++ src/test1.c | 4 +++ src/test_config.c | 4 +++ src/test_osinst.c | 7 ++++- src/test_quota.c | 40 ++------------------------ src/test_quota.h | 6 ---- tool/mksqlite3c-noext.tcl | 1 + tool/mksqlite3c.tcl | 1 + tool/mksqlite3internalh.tcl | 1 + 16 files changed, 106 insertions(+), 97 deletions(-) create mode 100644 src/os_setup.h diff --git a/Makefile.in b/Makefile.in index d96d30ae86..d554412348 100644 --- a/Makefile.in +++ b/Makefile.in @@ -239,6 +239,7 @@ SRC = \ $(TOP)/src/os.c \ $(TOP)/src/os.h \ $(TOP)/src/os_common.h \ + $(TOP)/src/os_setup.h \ $(TOP)/src/os_unix.c \ $(TOP)/src/os_win.c \ $(TOP)/src/os_win.h \ @@ -458,6 +459,7 @@ HDR = \ opcodes.h \ $(TOP)/src/os.h \ $(TOP)/src/os_common.h \ + $(TOP)/src/os_setup.h \ $(TOP)/src/os_win.h \ $(TOP)/src/pager.h \ $(TOP)/src/pcache.h \ diff --git a/Makefile.msc b/Makefile.msc index afe66baad8..18dfdf34b7 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -626,6 +626,7 @@ SRC = \ $(TOP)\src\os.c \ $(TOP)\src\os.h \ $(TOP)\src\os_common.h \ + $(TOP)\src\os_setup.h \ $(TOP)\src\os_unix.c \ $(TOP)\src\os_win.c \ $(TOP)\src\os_win.h \ @@ -848,6 +849,7 @@ HDR = \ opcodes.h \ $(TOP)\src\os.h \ $(TOP)\src\os_common.h \ + $(TOP)\src\os_setup.h \ $(TOP)\src\os_win.h \ $(TOP)\src\pager.h \ $(TOP)\src\pcache.h \ diff --git a/Makefile.vxworks b/Makefile.vxworks index 70b9bdde6b..0d9c27f635 100644 --- a/Makefile.vxworks +++ b/Makefile.vxworks @@ -262,6 +262,7 @@ SRC = \ $(TOP)/src/os.c \ $(TOP)/src/os.h \ $(TOP)/src/os_common.h \ + $(TOP)/src/os_setup.h \ $(TOP)/src/os_unix.c \ $(TOP)/src/os_win.c \ $(TOP)/src/os_win.h \ @@ -417,6 +418,7 @@ HDR = \ opcodes.h \ $(TOP)/src/os.h \ $(TOP)/src/os_common.h \ + $(TOP)/src/os_setup.h \ $(TOP)/src/os_win.h \ $(TOP)/src/pager.h \ $(TOP)/src/pcache.h \ diff --git a/main.mk b/main.mk index 4992de549b..904ddd708f 100644 --- a/main.mk +++ b/main.mk @@ -121,6 +121,7 @@ SRC = \ $(TOP)/src/os.c \ $(TOP)/src/os.h \ $(TOP)/src/os_common.h \ + $(TOP)/src/os_setup.h \ $(TOP)/src/os_unix.c \ $(TOP)/src/os_win.c \ $(TOP)/src/os_win.h \ @@ -340,6 +341,7 @@ HDR = \ opcodes.h \ $(TOP)/src/os.h \ $(TOP)/src/os_common.h \ + $(TOP)/src/os_setup.h \ $(TOP)/src/os_win.h \ $(TOP)/src/pager.h \ $(TOP)/src/pcache.h \ diff --git a/manifest b/manifest index 3f9a102e7b..2c2f3db4d5 100644 --- a/manifest +++ b/manifest @@ -1,10 +1,10 @@ -C Merge\supdates\sfrom\strunk. -D 2013-11-26T00:33:25.374 +C Further\swork\son\sWindows\sheader\sfile\sreform. +D 2013-11-26T01:00:31.440 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in 16254fb023b7e69788013f0da5ccdc2063ba2328 +F Makefile.in 5d0d35aef4d0601493f9b26e2f859b3e471b63ed F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc 7e91dccedae82c45dbf91a4e8240c642c2b90c83 -F Makefile.vxworks f921bd6a0600e36d67da080d820ae712ab393f30 +F Makefile.msc d66f5d9f338735e07aa4f6e3c99491421cca24f5 +F Makefile.vxworks 034289efa9d591b04b1a73598623119c306cbba0 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 F VERSION 52f7e22bfcec71a462e34194b4ae1671380fde59 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -142,7 +142,7 @@ F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt f439556c5ce01ced70987e5ee86549a45165d9ff -F main.mk c9b6d61c55f461543244d63992c63037c3f11dac +F main.mk 6e5cc23140ed3a85716c02376adc86aa11cfb673 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh d2a981497b404d6498f5ff3e3b1f3816bdfcb338 @@ -203,8 +203,9 @@ F src/mutex_unix.c c3a4e00f96ba068a8dbef34084465979aaf369cc F src/mutex_w32.c ab08c0fc54b71979370ca7c8f42fc64a9f211ebb F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c b4ad71336fd96f97776f75587cd9e8218288f5be -F src/os.h cd8fd6545d6a936bae05026136324ec8e620c7f8 +F src/os.h 60d419395e32a8029fa380a80a3da2e9030f635e F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 +F src/os_setup.h 55a27b7b02b8fb289e16749df17edcc376ba2dfb F src/os_unix.c 143624d9eabb3b997c59cf594e0d06c56edd43e9 F src/os_win.c 99a5668c207bee7fae238209a3043c03b6050056 F src/os_win.h ac253253d080dc3dd261efe5e98626c4b95e5a26 @@ -230,7 +231,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 3b5f3716e320480659239abe887164521c575d83 -F src/test1.c 5757066e503a8ed51313cb3a5d9bcdcced2991a9 +F src/test1.c 8d96df19f045c629311751ed89c02c374d108e51 F src/test2.c 7355101c085304b90024f2261e056cdff13c6c35 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -243,7 +244,7 @@ F src/test_async.c 21e11293a2f72080eda70e1124e9102044531cd8 F src/test_autoext.c dea8a01a7153b9adc97bd26161e4226329546e12 F src/test_backup.c 3875e899222b651e18b662f86e0e50daa946344e F src/test_btree.c 5b89601dcb42a33ba8b820a6b763cc9cb48bac16 -F src/test_config.c 10d0e00dd6315879a6d9fac20bd063c7bbbfb8f8 +F src/test_config.c 3186c5be34de7cd8c47487687e65ca2fb8bf6471 F src/test_demovfs.c 69b2085076654ebc18014cbc6386f04409c959a9 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c ced436e3d4b8e4681328409b8081051ce614e28f @@ -259,10 +260,10 @@ F src/test_multiplex.c 9f304bf04170c91c0318238d512df2da039eb1c8 F src/test_multiplex.h 110a8c4d356e0aa464ca8730375608a9a0b61ae1 F src/test_mutex.c 293042d623ebba969160f471a82aa1551626454f F src/test_onefile.c 0396f220561f3b4eedc450cef26d40c593c69a25 -F src/test_osinst.c 90a845c8183013d80eccb1f29e8805608516edba +F src/test_osinst.c 3d0340bc31a9f3d8a3547e0272373e80f78dde25 F src/test_pcache.c a5cd24730cb43c5b18629043314548c9169abb00 -F src/test_quota.c 30c64f0ef84734f2231a686df41ed882b0c59bc0 -F src/test_quota.h 8761e463b25e75ebc078bd67d70e39b9c817a0cb +F src/test_quota.c 65f6348fec0f2b3020c907247fb47556b214abb9 +F src/test_quota.h 2a8ad1952d1d2ca9af0ce0465e56e6c023b5e15d F src/test_rtree.c f3d1d12538dccb75fd916e3fa58f250edbdd3b47 F src/test_schema.c cd12a2223c3a394f4d07bb93bdf6d344c5c121b6 F src/test_server.c a2615049954cbb9cfb4a62e18e2f0616e4dc38fe @@ -1113,10 +1114,10 @@ F tool/mkkeywordhash.c 189d76644e373c7d0864c628deb8ce7b4f403591 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e F tool/mkpragmatab.tcl 78a77b2c554d534c6f2dc903130186ed15715460 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 -F tool/mksqlite3c-noext.tcl 1407684f6781496421be53841f1259303f0c8be9 -F tool/mksqlite3c.tcl c6b22da4d4ee272afda33f0aeb896d9c2e26dc95 +F tool/mksqlite3c-noext.tcl 1712d3d71256ca1f297046619c89e77a4d7c8f6d +F tool/mksqlite3c.tcl db30eeba8bb34561985938438abca200689e8bc0 F tool/mksqlite3h.tcl ba24038056f51fde07c0079c41885ab85e2cff12 -F tool/mksqlite3internalh.tcl ba0532989220eaef044cbb8c0108eca623d23bbc +F tool/mksqlite3internalh.tcl b6514145a7d5321b47e64e19b8116cc44f973eb1 F tool/mkvsix.tcl 6477fb9dab838b7eea1eed50658ff1cda04850b5 F tool/offsets.c fe4262fdfa378e8f5499a42136d17bf3b98f6091 F tool/omittest.tcl 4665982e95a6e5c1bd806cf7bc3dea95be422d77 @@ -1144,7 +1145,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 94219b9f1ff59ea408c7ded167ee28b00c0f38c1 9954327c0febc0ece46f62e05976330a1b82b48f -R 8ba15fb22b38fc61c6334c03ecc16c63 +P c5ab4378756965b0c2ab218e5ec59c40898b89ea +R 3f71be42bb92a89017dddc2d61b45fe7 U mistachkin -Z a3a638752bfaa9f69bc41b7e3e5dbf34 +Z beeb2b639ff26c8bb19c3f48e2444291 diff --git a/manifest.uuid b/manifest.uuid index b8c2a58092..5a187b121e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c5ab4378756965b0c2ab218e5ec59c40898b89ea \ No newline at end of file +540f5525160b516de282136ca77f909115a51630 \ No newline at end of file diff --git a/src/os.h b/src/os.h index 1d438d22cd..3920a62ee3 100644 --- a/src/os.h +++ b/src/os.h @@ -21,40 +21,10 @@ #define _SQLITE_OS_H_ /* -** Figure out if we are dealing with Unix, Windows, or some other -** operating system. After the following block of preprocess macros, -** all of SQLITE_OS_UNIX, SQLITE_OS_WIN, and SQLITE_OS_OTHER -** will defined to either 1 or 0. One of the four will be 1. The other -** three will be 0. +** Attempt to automatically detect the operating system and setup the +** necessary pre-processor macros for it. */ -#if defined(SQLITE_OS_OTHER) -# if SQLITE_OS_OTHER==1 -# undef SQLITE_OS_UNIX -# define SQLITE_OS_UNIX 0 -# undef SQLITE_OS_WIN -# define SQLITE_OS_WIN 0 -# else -# undef SQLITE_OS_OTHER -# endif -#endif -#if !defined(SQLITE_OS_UNIX) && !defined(SQLITE_OS_OTHER) -# define SQLITE_OS_OTHER 0 -# ifndef SQLITE_OS_WIN -# if defined(_WIN32) || defined(WIN32) || defined(__CYGWIN__) || defined(__MINGW32__) || defined(__BORLANDC__) -# define SQLITE_OS_WIN 1 -# define SQLITE_OS_UNIX 0 -# else -# define SQLITE_OS_WIN 0 -# define SQLITE_OS_UNIX 1 -# endif -# else -# define SQLITE_OS_UNIX 0 -# endif -#else -# ifndef SQLITE_OS_WIN -# define SQLITE_OS_WIN 0 -# endif -#endif +#include "os_setup.h" /* If the SET_FULLSYNC macro is not defined above, then make it ** a no-op diff --git a/src/os_setup.h b/src/os_setup.h new file mode 100644 index 0000000000..b04fbf7cf6 --- /dev/null +++ b/src/os_setup.h @@ -0,0 +1,56 @@ +/* +** 2013 November 25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code that is specific to Windows. +*/ +#ifndef _OS_SETUP_H_ +#define _OS_SETUP_H_ + +/* +** Figure out if we are dealing with Unix, Windows, or some other operating +** system. +** +** After the following block of preprocess macros, all of SQLITE_OS_UNIX, +** SQLITE_OS_WIN, and SQLITE_OS_OTHER will defined to either 1 or 0. One of +** the four will be 1. The other three will be 0. +*/ +#if defined(SQLITE_OS_OTHER) +# if SQLITE_OS_OTHER==1 +# undef SQLITE_OS_UNIX +# define SQLITE_OS_UNIX 0 +# undef SQLITE_OS_WIN +# define SQLITE_OS_WIN 0 +# else +# undef SQLITE_OS_OTHER +# endif +#endif +#if !defined(SQLITE_OS_UNIX) && !defined(SQLITE_OS_OTHER) +# define SQLITE_OS_OTHER 0 +# ifndef SQLITE_OS_WIN +# if defined(_WIN32) || defined(WIN32) || defined(__CYGWIN__) || \ + defined(__MINGW32__) || defined(__BORLANDC__) +# define SQLITE_OS_WIN 1 +# define SQLITE_OS_UNIX 0 +# else +# define SQLITE_OS_WIN 0 +# define SQLITE_OS_UNIX 1 +# endif +# else +# define SQLITE_OS_UNIX 0 +# endif +#else +# ifndef SQLITE_OS_WIN +# define SQLITE_OS_WIN 0 +# endif +#endif + +#endif /* _OS_SETUP_H_ */ diff --git a/src/test1.c b/src/test1.c index d8a9e52d21..88e42b2d76 100644 --- a/src/test1.c +++ b/src/test1.c @@ -14,6 +14,10 @@ ** testing of the SQLite library. */ #include "sqliteInt.h" +#if SQLITE_OS_WIN +# include "os_win.h" +#endif + #include "vdbeInt.h" #include "tcl.h" #include diff --git a/src/test_config.c b/src/test_config.c index f44be40508..e5d5d0739d 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -20,6 +20,10 @@ #include "sqliteLimit.h" #include "sqliteInt.h" +#if SQLITE_OS_WIN +# include "os_win.h" +#endif + #include "tcl.h" #include #include diff --git a/src/test_osinst.c b/src/test_osinst.c index 531433313e..1701def159 100644 --- a/src/test_osinst.c +++ b/src/test_osinst.c @@ -70,6 +70,12 @@ */ #include "sqlite3.h" + +#include "os_setup.h" +#if SQLITE_OS_WIN +# include "os_win.h" +#endif + #include #include @@ -221,7 +227,6 @@ static sqlite3_uint64 vfslog_time(){ return sTime.tv_usec + (sqlite3_uint64)sTime.tv_sec * 1000000; } #elif SQLITE_OS_WIN -#include #include static sqlite3_uint64 vfslog_time(){ FILETIME ft; diff --git a/src/test_quota.c b/src/test_quota.c index e590996ca4..80ebd0589e 100644 --- a/src/test_quota.c +++ b/src/test_quota.c @@ -44,49 +44,13 @@ #define sqlite3_mutex_notheld(X) ((void)(X),1) #endif /* SQLITE_THREADSAFE==0 */ - -/* -** Figure out if we are dealing with Unix, Windows, or some other -** operating system. After the following block of preprocess macros, -** all of SQLITE_OS_UNIX, SQLITE_OS_WIN, and SQLITE_OS_OTHER -** will defined to either 1 or 0. One of the four will be 1. The other -** three will be 0. -*/ -#if defined(SQLITE_OS_OTHER) -# if SQLITE_OS_OTHER==1 -# undef SQLITE_OS_UNIX -# define SQLITE_OS_UNIX 0 -# undef SQLITE_OS_WIN -# define SQLITE_OS_WIN 0 -# else -# undef SQLITE_OS_OTHER -# endif -#endif -#if !defined(SQLITE_OS_UNIX) && !defined(SQLITE_OS_OTHER) -# define SQLITE_OS_OTHER 0 -# ifndef SQLITE_OS_WIN -# if defined(_WIN32) || defined(WIN32) || defined(__CYGWIN__) \ - || defined(__MINGW32__) || defined(__BORLANDC__) -# define SQLITE_OS_WIN 1 -# define SQLITE_OS_UNIX 0 -# else -# define SQLITE_OS_WIN 0 -# define SQLITE_OS_UNIX 1 -# endif -# else -# define SQLITE_OS_UNIX 0 -# endif -#else -# ifndef SQLITE_OS_WIN -# define SQLITE_OS_WIN 0 -# endif -#endif +#include "os_setup.h" #if SQLITE_OS_UNIX # include #endif #if SQLITE_OS_WIN -# include +# include "os_win.h" # include #endif diff --git a/src/test_quota.h b/src/test_quota.h index 2d0767a19a..c17e15adca 100644 --- a/src/test_quota.h +++ b/src/test_quota.h @@ -31,12 +31,6 @@ #include #include #include -#if SQLITE_OS_UNIX -# include -#endif -#if SQLITE_OS_WIN -# include -#endif /* Make this callable from C++ */ #ifdef __cplusplus diff --git a/tool/mksqlite3c-noext.tcl b/tool/mksqlite3c-noext.tcl index 02d9ae5469..ecb9cb0439 100644 --- a/tool/mksqlite3c-noext.tcl +++ b/tool/mksqlite3c-noext.tcl @@ -99,6 +99,7 @@ foreach hdr { mutex.h opcodes.h os_common.h + os_setup.h os_win.h os.h pager.h diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl index e565bcbfe3..cdfba7add7 100644 --- a/tool/mksqlite3c.tcl +++ b/tool/mksqlite3c.tcl @@ -103,6 +103,7 @@ foreach hdr { mutex.h opcodes.h os_common.h + os_setup.h os_win.h os.h pager.h diff --git a/tool/mksqlite3internalh.tcl b/tool/mksqlite3internalh.tcl index 3938316eec..7e92b3ad7d 100644 --- a/tool/mksqlite3internalh.tcl +++ b/tool/mksqlite3internalh.tcl @@ -60,6 +60,7 @@ foreach hdr { keywordhash.h opcodes.h os_common.h + os_setup.h os_win.h os.h pager.h From 2e72791781e674f009d1cf534c740e9434d7b6e9 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 26 Nov 2013 01:13:18 +0000 Subject: [PATCH 04/71] Fix comment. FossilOrigin-Name: 8364af392da2946acd84633b45684e6fba5c39ed --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_setup.h | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 2c2f3db4d5..7076746d4d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\swork\son\sWindows\sheader\sfile\sreform. -D 2013-11-26T01:00:31.440 +C Fix\scomment. +D 2013-11-26T01:13:18.839 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5d0d35aef4d0601493f9b26e2f859b3e471b63ed F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -205,7 +205,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c b4ad71336fd96f97776f75587cd9e8218288f5be F src/os.h 60d419395e32a8029fa380a80a3da2e9030f635e F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_setup.h 55a27b7b02b8fb289e16749df17edcc376ba2dfb +F src/os_setup.h 3f9e87d981eef070c8a11e76b4704842a0e5870f F src/os_unix.c 143624d9eabb3b997c59cf594e0d06c56edd43e9 F src/os_win.c 99a5668c207bee7fae238209a3043c03b6050056 F src/os_win.h ac253253d080dc3dd261efe5e98626c4b95e5a26 @@ -1145,7 +1145,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P c5ab4378756965b0c2ab218e5ec59c40898b89ea -R 3f71be42bb92a89017dddc2d61b45fe7 +P 540f5525160b516de282136ca77f909115a51630 +R 4b6ba147833b38d11479654f460b02e2 U mistachkin -Z beeb2b639ff26c8bb19c3f48e2444291 +Z d79a165fcb75b9e4bfcba919773d7cea diff --git a/manifest.uuid b/manifest.uuid index 5a187b121e..aa49d27844 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -540f5525160b516de282136ca77f909115a51630 \ No newline at end of file +8364af392da2946acd84633b45684e6fba5c39ed \ No newline at end of file diff --git a/src/os_setup.h b/src/os_setup.h index b04fbf7cf6..5b010b894e 100644 --- a/src/os_setup.h +++ b/src/os_setup.h @@ -10,7 +10,8 @@ ** ****************************************************************************** ** -** This file contains code that is specific to Windows. +** This file contains pre-processor directives related to operating system +** detection and/or setup. */ #ifndef _OS_SETUP_H_ #define _OS_SETUP_H_ From 491451dead1b8848897cf80059f2255866514dce Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 18 Feb 2014 05:18:36 +0000 Subject: [PATCH 05/71] Enhance the Win32 VFS I/O retry logic. FossilOrigin-Name: adba783c702b05f83e0bee3eb1bc9e40cdec3785 --- manifest | 17 ++++++++++------- manifest.uuid | 2 +- src/os_win.c | 37 ++++++++++++++++++++++++++++++++++--- 3 files changed, 45 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 4606d87635..84d15c6531 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sVdbeCoverage()\sand\sVdbeCoverageIf()\smacros\sfor\simproved\sVDBE\scoverage\ntesting. -D 2014-02-18T03:07:12.342 +C Enhance\sthe\sWin32\sVFS\sI/O\sretry\slogic. +D 2014-02-18T05:18:36.863 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -203,7 +203,7 @@ F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 18f7f95dc6bcb9cf4d4a238d8e2de96611bc2ae5 -F src/os_win.c d4284f003445054a26689f1264b1b9bf7261bd1b +F src/os_win.c 5c0f315a7cfb513b7151c87a0699a8ad0bc1acb2 F src/pager.c 0ffa313a30ed6d061d9c6601b7b175cc50a1cab7 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y cce844ccb80b5f969b04c25100c8d94338488efb @@ -1151,7 +1151,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 915f6f1c7aab54583729e60bdc1565f25ecc6f74 -R d0043e1267f21a1ea521214d42169c3d -U drh -Z f5a439caf79133298419241dae122f4e +P b92d31a97d5fe4606d9ae1393c7f3e052f46bf5a +R 36c4c6416a0341396e2a781bd502abeb +T *branch * winIoRetry +T *sym-winIoRetry * +T -sym-trunk * +U mistachkin +Z b9d60eef2d6576ef4d346ab00acfffd9 diff --git a/manifest.uuid b/manifest.uuid index 2e856727c0..02a7c49e96 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b92d31a97d5fe4606d9ae1393c7f3e052f46bf5a \ No newline at end of file +adba783c702b05f83e0bee3eb1bc9e40cdec3785 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index d393c0d016..616616d0df 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -1836,6 +1836,32 @@ static int winLogErrorAtLine( static int winIoerrRetry = SQLITE_WIN32_IOERR_RETRY; static int winIoerrRetryDelay = SQLITE_WIN32_IOERR_RETRY_DELAY; +/* +** The "winIoerrCanRetry1" macro is used to determine if a particular I/O +** error code obtained via GetLastError() is eligible to be retried. It +** must accept the error code DWORD as its only argument and should return +** non-zero if the error code is transient in nature and the operation +** responsible for generating the original error might succeed upon being +** retried. The argument to this macro should be a variable. +** +** Additionally, a macro named "winIoerrCanRetry2" may be defined. If it +** is defined, it will be consulted only when the macro "winIoerrCanRetry1" +** returns zero. The "winIoerrCanRetry2" macro is completely optional and +** may be used to include additional error codes in the set that should +** result in the failing I/O operation being retried by the caller. If +** defined, the "winIoerrCanRetry2" macro must exhibit external semantics +** identical to those of the "winIoerrCanRetry1" macro. +*/ +#if !defined(winIoerrCanRetry1) +#define winIoerrCanRetry1(a) (((a)==ERROR_ACCESS_DENIED) || \ + ((a)==ERROR_SHARING_VIOLATION) || \ + ((a)==ERROR_LOCK_VIOLATION) || \ + ((a)==ERROR_DEV_NOT_EXIST) || \ + ((a)==ERROR_NETNAME_DELETED) || \ + ((a)==ERROR_SEM_TIMEOUT) || \ + ((a)==ERROR_NETWORK_UNREACHABLE)) +#endif + /* ** If a ReadFile() or WriteFile() error occurs, invoke this routine ** to see if it should be retried. Return TRUE to retry. Return FALSE @@ -1849,13 +1875,18 @@ static int winRetryIoerr(int *pnRetry, DWORD *pError){ } return 0; } - if( e==ERROR_ACCESS_DENIED || - e==ERROR_LOCK_VIOLATION || - e==ERROR_SHARING_VIOLATION ){ + if( winIoerrCanRetry1(e) ){ sqlite3_win32_sleep(winIoerrRetryDelay*(1+*pnRetry)); ++*pnRetry; return 1; } +#if defined(winIoerrCanRetry2) + else if( winIoerrCanRetry2(e) ){ + sqlite3_win32_sleep(winIoerrRetryDelay*(1+*pnRetry)); + ++*pnRetry; + return 1; + } +#endif if( pError ){ *pError = e; } From 00b638bb6f2e0fb2bdcff933111d46139ede35bd Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 18 Apr 2014 00:08:13 +0000 Subject: [PATCH 06/71] Preliminary changes to support building VSIX packages for Windows Phone 8.1. FossilOrigin-Name: a1b2f28ba85b043df68875f5d3446140db4f2cb0 --- manifest | 19 +++++++++++-------- manifest.uuid | 2 +- tool/mkvsix.tcl | 25 ++++++++++++++++++++----- tool/win/sqlite.vsix | Bin 32825 -> 32820 bytes 4 files changed, 32 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index a1452995b3..eea66b0d82 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sSQLITE_TESTCTRL_BYTEORDER\stest\scontrol\sto\sinterrogate\sSQLite's\snotion\nof\sthe\sprocessor\sbyte\sorder\sand\swhether\sthe\sbyte\sorder\sis\sknown\sat\scompile-time\nor\sdetermined\sat\srun-time. -D 2014-04-18T00:06:02.110 +C Preliminary\schanges\sto\ssupport\sbuilding\sVSIX\spackages\sfor\sWindows\sPhone\s8.1. +D 2014-04-18T00:08:13.436 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1132,7 +1132,7 @@ F tool/mksqlite3c-noext.tcl 8bce31074e4cbe631bb7676526a048335f4c9f02 F tool/mksqlite3c.tcl ed5b4e9ca8bf209ff401059a16362f37710fd8b8 F tool/mksqlite3h.tcl ba24038056f51fde07c0079c41885ab85e2cff12 F tool/mksqlite3internalh.tcl 3dca7bb5374cee003379b8cbac73714f610ef795 -F tool/mkvsix.tcl 6477fb9dab838b7eea1eed50658ff1cda04850b5 +F tool/mkvsix.tcl df937e1fadbd98995c701002b2a4b0187a04f6ff F tool/offsets.c fe4262fdfa378e8f5499a42136d17bf3b98f6091 F tool/omittest.tcl 34d7ac01fe4fd18e3637f64abe12c40eca0f6b97 F tool/opcodeDoc.awk b3a2a3d5d3075b8bd90b7afe24283efdd586659c @@ -1159,8 +1159,11 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 -F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P c6fa8d0d82805be230f672eabccdfa5680d4ddfd -R f2d9932522491a86ed313bf5e3a1693d -U drh -Z 4f6d25a564bc084e4d5fbc71ec75b6b1 +F tool/win/sqlite.vsix a94fb9b1b1ef06efc2898975cdfcfa9643731f5e +P 9c6961967ae00e563ebe2859eaf2639a79f2cb01 +R cd982f22bbf931205d39222b6f2ba26d +T *branch * vsixWinPhone81 +T *sym-vsixWinPhone81 * +T -sym-trunk * +U mistachkin +Z 0f515f990cd1ad75013b0794d9a0c168 diff --git a/manifest.uuid b/manifest.uuid index c03388d6de..f9d51641f8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9c6961967ae00e563ebe2859eaf2639a79f2cb01 \ No newline at end of file +a1b2f28ba85b043df68875f5d3446140db4f2cb0 \ No newline at end of file diff --git a/tool/mkvsix.tcl b/tool/mkvsix.tcl index 65fa731245..3268efdee7 100644 --- a/tool/mkvsix.tcl +++ b/tool/mkvsix.tcl @@ -256,7 +256,8 @@ if {[string equal -nocase $packageFlavor WinRT]} then { set displayName "SQLite for Windows Runtime" set targetPlatformIdentifier Windows set targetPlatformVersion v8.0 - set minVsVersion 11.0 + set minVsVersion [appendArgs \ + "\r\n " {MinVSVersion="11.0"}] set extraSdkPath "" set extraFileListAttributes [appendArgs \ "\r\n " {AppliesTo="WindowsAppContainer"} \ @@ -266,7 +267,8 @@ if {[string equal -nocase $packageFlavor WinRT]} then { set displayName "SQLite for Windows Runtime (Windows 8.1)" set targetPlatformIdentifier Windows set targetPlatformVersion v8.1 - set minVsVersion 12.0 + set minVsVersion [appendArgs \ + "\r\n " {MinVSVersion="12.0"}] set extraSdkPath "" set extraFileListAttributes [appendArgs \ "\r\n " {AppliesTo="WindowsAppContainer"} \ @@ -276,7 +278,17 @@ if {[string equal -nocase $packageFlavor WinRT]} then { set displayName "SQLite for Windows Phone" set targetPlatformIdentifier "Windows Phone" set targetPlatformVersion v8.0 - set minVsVersion 11.0 + set minVsVersion [appendArgs \ + "\r\n " {MinVSVersion="11.0"}] + set extraSdkPath "\\..\\$targetPlatformIdentifier" + set extraFileListAttributes "" +} elseif {[string equal -nocase $packageFlavor WP81]} then { + set shortName SQLite.WP81 + set displayName "SQLite for Windows Phone 8.1" + set targetPlatformIdentifier "WindowsPhoneApp" + set targetPlatformVersion v8.1 + set minVsVersion [appendArgs \ + "\r\n " {MinVSVersion="12.0"}] set extraSdkPath "\\..\\$targetPlatformIdentifier" set extraFileListAttributes "" } elseif {[string equal -nocase $packageFlavor Win32]} then { @@ -284,13 +296,16 @@ if {[string equal -nocase $packageFlavor WinRT]} then { set displayName "SQLite for Windows" set targetPlatformIdentifier Windows set targetPlatformVersion v8.0 - set minVsVersion 11.0 + set minVsVersion [appendArgs \ + "\r\n " {MinVSVersion="11.0"}] set extraSdkPath "" set extraFileListAttributes [appendArgs \ "\r\n " {AppliesTo="VisualC"} \ "\r\n " {DependsOn="Microsoft.VCLibs, version=11.0"}] } else { - fail "unsupported package flavor, must be one of: WinRT WinRT81 WP80 Win32" + fail [appendArgs \ + "unsupported package flavor, must be one of: " \ + "WinRT WinRT81 WP80 WP81 Win32"] } if {$argc >= 4} then { diff --git a/tool/win/sqlite.vsix b/tool/win/sqlite.vsix index ac4afb3f4bc2aa7c4419376732e5a7b3910d1070..de4d0a1f96ad3b53b899a7bcfbba53f23a1d4990 100644 GIT binary patch delta 313 zcmV-90mlBhfC99D0OuO<`_nW@U49E_iKh zRgh0h!$1(l?}dJcWy#e>>ZK#bm%`XWEq3S8|QbemE~Y0oaTn%_((vn3jMK$hu5rMX28f2R6=tSpo|+cp1k z&?E6gzKOLgD~sljMljq5(yIX^5M~`ES(~)C%Z4o z>h=0f?;lVcLX-FbvkrS(1PuUykwjgh(dOF#001|Wc7HGjqdpA6kXp}^t$#ZK36tr6 LN(K*n00000h#!t5 delta 327 zcmV-N0l5COfC9OI0OuO<`_nW@U49E_iKh zRgg_f!$1&*?}h${W$D#KZ=zjGK?{Nvq0ozZh{>cG*)L~jO-1tG-H^t%dwbxSecze+ zVsEtg1`6n`l2_TK6ktnNq1`DNxxToTVsST}%vY%42?I|jBE5SIe_j_q&xHXcXTQ}E zJ2@CHTmsxYYkz0f$~Q)}%A9&PQEY?%E`p0{nAg)fZ^nWJ4kYG}8c?a;FS$0QC1 z?h?bEsTdnBnF&84L+FO-P4qtHhIQ18Uc+H9DEUC-NS;|xEval$e#wy_Ij^4%t(c|# zxOQav_CdVezRUh93HcbB;15ts0|b-5e;BhRdp`sXQ(e?Tdi!qm-~a#sOp|ecFb5W4 ZChz5K&XcHrI{^-px001^@mD~UT From e756bfb3635d952a54420f9f473db41997c13428 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 21 Apr 2014 18:00:11 +0000 Subject: [PATCH 07/71] Preliminary changes to support building VSIX packages for Windows 8.0 and Windows Phone 8.0 using Visual Studio 2013. FossilOrigin-Name: 3bf97c811a25babf760f256245c756ed6710f8ba --- manifest | 15 ++++++--------- manifest.uuid | 2 +- tool/mkvsix.tcl | 22 +++++++++++++++++++++- 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index eea66b0d82..df82a686b4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Preliminary\schanges\sto\ssupport\sbuilding\sVSIX\spackages\sfor\sWindows\sPhone\s8.1. -D 2014-04-18T00:08:13.436 +C Preliminary\schanges\sto\ssupport\sbuilding\sVSIX\spackages\sfor\sWindows\s8.0\sand\sWindows\sPhone\s8.0\susing\sVisual\sStudio\s2013. +D 2014-04-21T18:00:11.146 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1132,7 +1132,7 @@ F tool/mksqlite3c-noext.tcl 8bce31074e4cbe631bb7676526a048335f4c9f02 F tool/mksqlite3c.tcl ed5b4e9ca8bf209ff401059a16362f37710fd8b8 F tool/mksqlite3h.tcl ba24038056f51fde07c0079c41885ab85e2cff12 F tool/mksqlite3internalh.tcl 3dca7bb5374cee003379b8cbac73714f610ef795 -F tool/mkvsix.tcl df937e1fadbd98995c701002b2a4b0187a04f6ff +F tool/mkvsix.tcl 0d439bb042d0f889210afa91674fe7bf07131d39 F tool/offsets.c fe4262fdfa378e8f5499a42136d17bf3b98f6091 F tool/omittest.tcl 34d7ac01fe4fd18e3637f64abe12c40eca0f6b97 F tool/opcodeDoc.awk b3a2a3d5d3075b8bd90b7afe24283efdd586659c @@ -1160,10 +1160,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix a94fb9b1b1ef06efc2898975cdfcfa9643731f5e -P 9c6961967ae00e563ebe2859eaf2639a79f2cb01 -R cd982f22bbf931205d39222b6f2ba26d -T *branch * vsixWinPhone81 -T *sym-vsixWinPhone81 * -T -sym-trunk * +P a1b2f28ba85b043df68875f5d3446140db4f2cb0 +R c68a023e71ddd46ca56f8a556c77ce10 U mistachkin -Z 0f515f990cd1ad75013b0794d9a0c168 +Z aa4cbe46e2e91580e28bf4f7ed1db0b0 diff --git a/manifest.uuid b/manifest.uuid index f9d51641f8..0f284be186 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a1b2f28ba85b043df68875f5d3446140db4f2cb0 \ No newline at end of file +3bf97c811a25babf760f256245c756ed6710f8ba \ No newline at end of file diff --git a/tool/mkvsix.tcl b/tool/mkvsix.tcl index 3268efdee7..787326f0f1 100644 --- a/tool/mkvsix.tcl +++ b/tool/mkvsix.tcl @@ -262,6 +262,17 @@ if {[string equal -nocase $packageFlavor WinRT]} then { set extraFileListAttributes [appendArgs \ "\r\n " {AppliesTo="WindowsAppContainer"} \ "\r\n " {DependsOn="Microsoft.VCLibs, version=11.0"}] +} elseif {[string equal -nocase $packageFlavor WinRT_2013]} then { + set shortName SQLite.WinRT.2013 + set displayName "SQLite for Windows Runtime (2013)" + set targetPlatformIdentifier Windows + set targetPlatformVersion v8.0 + set minVsVersion [appendArgs \ + "\r\n " {MinVSVersion="12.0"}] + set extraSdkPath "" + set extraFileListAttributes [appendArgs \ + "\r\n " {AppliesTo="WindowsAppContainer"} \ + "\r\n " {DependsOn="Microsoft.VCLibs, version=12.0"}] } elseif {[string equal -nocase $packageFlavor WinRT81]} then { set shortName SQLite.WinRT81 set displayName "SQLite for Windows Runtime (Windows 8.1)" @@ -282,6 +293,15 @@ if {[string equal -nocase $packageFlavor WinRT]} then { "\r\n " {MinVSVersion="11.0"}] set extraSdkPath "\\..\\$targetPlatformIdentifier" set extraFileListAttributes "" +} elseif {[string equal -nocase $packageFlavor WP80_2013]} then { + set shortName SQLite.WP80.2013 + set displayName "SQLite for Windows Phone (2013)" + set targetPlatformIdentifier "Windows Phone" + set targetPlatformVersion v8.0 + set minVsVersion [appendArgs \ + "\r\n " {MinVSVersion="12.0"}] + set extraSdkPath "\\..\\$targetPlatformIdentifier" + set extraFileListAttributes "" } elseif {[string equal -nocase $packageFlavor WP81]} then { set shortName SQLite.WP81 set displayName "SQLite for Windows Phone 8.1" @@ -305,7 +325,7 @@ if {[string equal -nocase $packageFlavor WinRT]} then { } else { fail [appendArgs \ "unsupported package flavor, must be one of: " \ - "WinRT WinRT81 WP80 WP81 Win32"] + "WinRT WinRT_2013 WinRT81 WP80 WP80_2013 WP81 Win32"] } if {$argc >= 4} then { From 0bf8277196ff168f7b581ff5d523c98d0a4ca0b5 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 22 Apr 2014 19:30:00 +0000 Subject: [PATCH 08/71] Add the win32-none VFS which omits rollback-journal locking. FossilOrigin-Name: 03e3c5a8b17a219a49bfbe79b7debd27ace8ad85 --- manifest | 15 +++++++++------ manifest.uuid | 2 +- src/os_win.c | 39 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 48 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 7f4e9189fd..9f24411d5f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Comment\stweaks\son\sthe\stest\scase\sfor\sthe\s[b75a9ca6b0]\sbug\sfix. -D 2014-04-21T13:36:54.639 +C Add\sthe\swin32-none\sVFS\swhich\somits\srollback-journal\slocking. +D 2014-04-22T19:30:00.797 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -204,7 +204,7 @@ F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c ae4b5240af4619d711301d7992396e182585269f -F src/os_win.c e71678ac927d0a0fb11d993db20a9748eabf808e +F src/os_win.c a49ee67a07d7aff98bd9793e7ac8c68ee12712b4 F src/pager.c ab62a24218d87dda1be641f6c5ad291bff78fd94 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 @@ -1161,7 +1161,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P de9a490f594183f337a2ec9e0f87792eac83548b -R ce888b84132e0cad3bcca115a32951d3 +P 65d2544af9adc1e2f1d193e57f8be0422fb0d5eb +R a9779ff2bb2c9843026c8b0117f1fe75 +T *branch * win32-none +T *sym-win32-none * +T -sym-trunk * U drh -Z cf9f241149456ab1fa24984e95a412d2 +Z d17256f50e94b0475de531ab714c2ce7 diff --git a/manifest.uuid b/manifest.uuid index a2761a597b..0a5654bde9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -65d2544af9adc1e2f1d193e57f8be0422fb0d5eb \ No newline at end of file +03e3c5a8b17a219a49bfbe79b7debd27ace8ad85 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 287dad3b57..6b22fab20a 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -275,6 +275,7 @@ struct winFile { #define WINFILE_RDONLY 0x02 /* Connection is read only */ #define WINFILE_PERSIST_WAL 0x04 /* Persistent WAL mode */ #define WINFILE_PSOW 0x10 /* SQLITE_IOCAP_POWERSAFE_OVERWRITE */ +#define WINFILE_NOLOCK 0x20 /* Never do any real locking */ /* * The size of the buffer used by sqlite3_win32_write_debug(). @@ -2859,6 +2860,9 @@ static int winLock(sqlite3_file *id, int locktype){ assert( id!=0 ); OSTRACE(("LOCK file=%p, oldLock=%d(%d), newLock=%d\n", pFile->h, pFile->locktype, pFile->sharedLockByte, locktype)); + if( pFile->ctrlFlags & WINFILE_NOLOCK ){ + return SQLITE_OK; + } /* If there is already a lock of this type or more restrictive on the ** OsFile, do nothing. Don't use the end_lock: exit path, as @@ -2986,7 +2990,9 @@ static int winCheckReservedLock(sqlite3_file *id, int *pResOut){ OSTRACE(("TEST-WR-LOCK file=%p, pResOut=%p\n", pFile->h, pResOut)); assert( id!=0 ); - if( pFile->locktype>=RESERVED_LOCK ){ + if( pFile->ctrlFlags & WINFILE_NOLOCK ){ + rc = 0; + }else if( pFile->locktype>=RESERVED_LOCK ){ rc = 1; OSTRACE(("TEST-WR-LOCK file=%p, rc=%d (local)\n", pFile->h, rc)); }else{ @@ -3022,6 +3028,9 @@ static int winUnlock(sqlite3_file *id, int locktype){ assert( locktype<=SHARED_LOCK ); OSTRACE(("UNLOCK file=%p, oldLock=%d(%d), newLock=%d\n", pFile->h, pFile->locktype, pFile->sharedLockByte, locktype)); + if( pFile->ctrlFlags & WINFILE_NOLOCK ){ + return SQLITE_OK; + } type = pFile->locktype; if( type>=EXCLUSIVE_LOCK ){ winUnlockFile(&pFile->h, SHARED_FIRST, 0, SHARED_SIZE, 0); @@ -4692,6 +4701,9 @@ static int winOpen( if( sqlite3_uri_boolean(zName, "psow", SQLITE_POWERSAFE_OVERWRITE) ){ pFile->ctrlFlags |= WINFILE_PSOW; } + if( strcmp(pVfs->zName,"win32-none")==0 ){ + pFile->ctrlFlags |= WINFILE_NOLOCK; + } pFile->lastErrno = NO_ERROR; pFile->zPath = zName; #if SQLITE_MAX_MMAP_SIZE>0 @@ -5416,6 +5428,30 @@ int sqlite3_os_init(void){ winNextSystemCall, /* xNextSystemCall */ }; #endif + static sqlite3_vfs winNoneVfs = { + 3, /* iVersion */ + sizeof(winFile), /* szOsFile */ + SQLITE_WIN32_MAX_PATH_BYTES, /* mxPathname */ + 0, /* pNext */ + "win32-none", /* zName */ + 0, /* pAppData */ + winOpen, /* xOpen */ + winDelete, /* xDelete */ + winAccess, /* xAccess */ + winFullPathname, /* xFullPathname */ + winDlOpen, /* xDlOpen */ + winDlError, /* xDlError */ + winDlSym, /* xDlSym */ + winDlClose, /* xDlClose */ + winRandomness, /* xRandomness */ + winSleep, /* xSleep */ + winCurrentTime, /* xCurrentTime */ + winGetLastError, /* xGetLastError */ + winCurrentTimeInt64, /* xCurrentTimeInt64 */ + winSetSystemCall, /* xSetSystemCall */ + winGetSystemCall, /* xGetSystemCall */ + winNextSystemCall, /* xNextSystemCall */ + }; /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ @@ -5432,6 +5468,7 @@ int sqlite3_os_init(void){ assert( winSysInfo.dwPageSize>0 ); sqlite3_vfs_register(&winVfs, 1); + sqlite3_vfs_register(&winNoneVfs, 0); #if defined(SQLITE_WIN32_HAS_WIDE) sqlite3_vfs_register(&winLongPathVfs, 0); From 9df245b07385378bb7d9054e5f722ec7c283946b Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 22 Apr 2014 19:34:16 +0000 Subject: [PATCH 09/71] Add some OSTRACE calls for consistency. FossilOrigin-Name: be292d559748a24ff012266a1f58ce6c027e0411 --- manifest | 17 +++++++---------- manifest.uuid | 2 +- src/os_win.c | 3 +++ 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 9f24411d5f..cea8c644a2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\swin32-none\sVFS\swhich\somits\srollback-journal\slocking. -D 2014-04-22T19:30:00.797 +C Add\ssome\sOSTRACE\scalls\sfor\sconsistency. +D 2014-04-22T19:34:16.263 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -204,7 +204,7 @@ F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c ae4b5240af4619d711301d7992396e182585269f -F src/os_win.c a49ee67a07d7aff98bd9793e7ac8c68ee12712b4 +F src/os_win.c 8c74fd4c2a7e2478e0b05530a51af4049acfcc43 F src/pager.c ab62a24218d87dda1be641f6c5ad291bff78fd94 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 @@ -1161,10 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 65d2544af9adc1e2f1d193e57f8be0422fb0d5eb -R a9779ff2bb2c9843026c8b0117f1fe75 -T *branch * win32-none -T *sym-win32-none * -T -sym-trunk * -U drh -Z d17256f50e94b0475de531ab714c2ce7 +P 03e3c5a8b17a219a49bfbe79b7debd27ace8ad85 +R 173fb04236ec33d12e5d272efbb7cda6 +U mistachkin +Z 7e8fdfc3e14c55c25e98572df7e0ac54 diff --git a/manifest.uuid b/manifest.uuid index 0a5654bde9..7c3c02fbb8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -03e3c5a8b17a219a49bfbe79b7debd27ace8ad85 \ No newline at end of file +be292d559748a24ff012266a1f58ce6c027e0411 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 6b22fab20a..7d1b1af92a 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -2861,6 +2861,7 @@ static int winLock(sqlite3_file *id, int locktype){ OSTRACE(("LOCK file=%p, oldLock=%d(%d), newLock=%d\n", pFile->h, pFile->locktype, pFile->sharedLockByte, locktype)); if( pFile->ctrlFlags & WINFILE_NOLOCK ){ + OSTRACE(("LOCK-NOP file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } @@ -2992,6 +2993,7 @@ static int winCheckReservedLock(sqlite3_file *id, int *pResOut){ assert( id!=0 ); if( pFile->ctrlFlags & WINFILE_NOLOCK ){ rc = 0; + OSTRACE(("TEST-WR-LOCK file=%p, rc=%d (nop)\n", pFile->h, rc)); }else if( pFile->locktype>=RESERVED_LOCK ){ rc = 1; OSTRACE(("TEST-WR-LOCK file=%p, rc=%d (local)\n", pFile->h, rc)); @@ -3029,6 +3031,7 @@ static int winUnlock(sqlite3_file *id, int locktype){ OSTRACE(("UNLOCK file=%p, oldLock=%d(%d), newLock=%d\n", pFile->h, pFile->locktype, pFile->sharedLockByte, locktype)); if( pFile->ctrlFlags & WINFILE_NOLOCK ){ + OSTRACE(("UNLOCK-NOP file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } type = pFile->locktype; From f34a9fef4c285a14d2b3213d033512a0607eb1e4 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 22 Apr 2014 19:42:22 +0000 Subject: [PATCH 10/71] Change from using the win32-none VFS to just a query parameter. FossilOrigin-Name: ffceacf61905dc3553978ba5f68e133947eca2fa --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_win.c | 27 +-------------------------- 3 files changed, 9 insertions(+), 34 deletions(-) diff --git a/manifest b/manifest index cea8c644a2..f71432393e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssome\sOSTRACE\scalls\sfor\sconsistency. -D 2014-04-22T19:34:16.263 +C Change\sfrom\susing\sthe\swin32-none\sVFS\sto\sjust\sa\squery\sparameter. +D 2014-04-22T19:42:22.086 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -204,7 +204,7 @@ F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c ae4b5240af4619d711301d7992396e182585269f -F src/os_win.c 8c74fd4c2a7e2478e0b05530a51af4049acfcc43 +F src/os_win.c 1662c40d22be162698a31725e836eede09e841ed F src/pager.c ab62a24218d87dda1be641f6c5ad291bff78fd94 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 @@ -1161,7 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 03e3c5a8b17a219a49bfbe79b7debd27ace8ad85 -R 173fb04236ec33d12e5d272efbb7cda6 -U mistachkin -Z 7e8fdfc3e14c55c25e98572df7e0ac54 +P be292d559748a24ff012266a1f58ce6c027e0411 +R f3c1c9d0a2061a78351193fe4c39a447 +U drh +Z 55529acaca6c8896715e4d74b5257ddb diff --git a/manifest.uuid b/manifest.uuid index 7c3c02fbb8..fb9c97c16a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -be292d559748a24ff012266a1f58ce6c027e0411 \ No newline at end of file +ffceacf61905dc3553978ba5f68e133947eca2fa \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 7d1b1af92a..bf5b9c72d1 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -4704,7 +4704,7 @@ static int winOpen( if( sqlite3_uri_boolean(zName, "psow", SQLITE_POWERSAFE_OVERWRITE) ){ pFile->ctrlFlags |= WINFILE_PSOW; } - if( strcmp(pVfs->zName,"win32-none")==0 ){ + if( sqlite3_uri_boolean(zName, "nolock", 0) ){ pFile->ctrlFlags |= WINFILE_NOLOCK; } pFile->lastErrno = NO_ERROR; @@ -5431,30 +5431,6 @@ int sqlite3_os_init(void){ winNextSystemCall, /* xNextSystemCall */ }; #endif - static sqlite3_vfs winNoneVfs = { - 3, /* iVersion */ - sizeof(winFile), /* szOsFile */ - SQLITE_WIN32_MAX_PATH_BYTES, /* mxPathname */ - 0, /* pNext */ - "win32-none", /* zName */ - 0, /* pAppData */ - winOpen, /* xOpen */ - winDelete, /* xDelete */ - winAccess, /* xAccess */ - winFullPathname, /* xFullPathname */ - winDlOpen, /* xDlOpen */ - winDlError, /* xDlError */ - winDlSym, /* xDlSym */ - winDlClose, /* xDlClose */ - winRandomness, /* xRandomness */ - winSleep, /* xSleep */ - winCurrentTime, /* xCurrentTime */ - winGetLastError, /* xGetLastError */ - winCurrentTimeInt64, /* xCurrentTimeInt64 */ - winSetSystemCall, /* xSetSystemCall */ - winGetSystemCall, /* xGetSystemCall */ - winNextSystemCall, /* xNextSystemCall */ - }; /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ @@ -5471,7 +5447,6 @@ int sqlite3_os_init(void){ assert( winSysInfo.dwPageSize>0 ); sqlite3_vfs_register(&winVfs, 1); - sqlite3_vfs_register(&winNoneVfs, 0); #if defined(SQLITE_WIN32_HAS_WIDE) sqlite3_vfs_register(&winLongPathVfs, 0); From aa9933c11529cabf659615d9c94277846dacbd15 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 24 Apr 2014 20:04:49 +0000 Subject: [PATCH 11/71] Changes to the way the planner calculates the costs of various table and index scans. Some test cases still failing. FossilOrigin-Name: c5a6ec0a880652dc8f4593d9f7acd58ddc3dc5f3 --- manifest | 25 +++++----- manifest.uuid | 2 +- src/where.c | 109 ++++++++++++++++++++++--------------------- test/analyze9.test | 8 ++-- test/autoindex1.test | 2 + test/where3.test | 3 +- test/whereG.test | 32 +++++++++++++ 7 files changed, 111 insertions(+), 70 deletions(-) diff --git a/manifest b/manifest index 7f4e9189fd..2b621bd738 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Comment\stweaks\son\sthe\stest\scase\sfor\sthe\s[b75a9ca6b0]\sbug\sfix. -D 2014-04-21T13:36:54.639 +C Changes\sto\sthe\sway\sthe\splanner\scalculates\sthe\scosts\sof\svarious\stable\sand\sindex\sscans.\sSome\stest\scases\sstill\sfailing. +D 2014-04-24T20:04:49.939 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 3b127bdc24b7aa84ffa69729170be11555cd7733 +F src/where.c c12bc20cd649bcae39de3e452bfc1a3f164454ee F src/whereInt.h 929c1349b5355fd44f22cee5c14d72b3329c58a6 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -311,7 +311,7 @@ F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4 F test/analyze6.test d31defa011a561b938b4608d3538c1b4e0b5e92c F test/analyze7.test bb1409afc9e8629e414387ef048b8e0e3e0bdc4f F test/analyze8.test 093d15c1c888eed5034304a98c992f7360130b88 -F test/analyze9.test e072a5172d55afcba98d6ca6a219ce8878c2f5c9 +F test/analyze9.test e219daa58fd8677c6a43d771798cf37d68f51d3e F test/analyzeA.test 1a5c40079894847976d983ca39c707aaa44b6944 F test/analyzeB.test 8bf35ee0a548aea831bf56762cb8e7fdb1db083d F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b @@ -329,7 +329,7 @@ F test/auth.test 5bdf154eb28c0e4bbc0473f335858c0d96171768 F test/auth2.test c3b415b76c033bedb81292118fb7c01f5f10cbcd F test/auth3.test a4755e6a2a2fea547ffe63c874eb569e60a28eb5 F test/autoinc.test c58912526998a39e11f66b533e23cfabea7f25b7 -F test/autoindex1.test d4dfe14001dfcb74cfbd7107f45a79fc1ab6183e +F test/autoindex1.test 762ff3f8e25d852aae55c6462ca166a80c0cde61 F test/autovacuum.test 941892505d2c0f410a0cb5970dfa1c7c4e5f6e74 F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4 F test/avtrans.test 0252654f4295ddda3b2cce0e894812259e655a85 @@ -1079,7 +1079,7 @@ F test/walslow.test e7be6d9888f83aa5d3d3c7c08aa9b5c28b93609a F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e F test/where.test 28b64e93428961b07b0d486778d63fd672948f6b F test/where2.test 455a2eb2666e66c1e84e2cb5815173a85e6237db -F test/where3.test d28c51f257e60be30f74308fa385ceeddfb54a6e +F test/where3.test 1ad55ba900bd7747f98b6082e65bd3e442c5004e F test/where4.test d8420ceeb8323a41ceff1f1841fc528e824e1ecf F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2 F test/where6.test 5da5a98cec820d488e82708301b96cb8c18a258b @@ -1093,7 +1093,7 @@ F test/whereC.test d6f4ecd4fa2d9429681a5b22a25d2bda8e86ab8a F test/whereD.test fd9120e262f9da3c45940f52aefeef4d15b904e5 F test/whereE.test b3a055eef928c992b0a33198a7b8dc10eea5ad2f F test/whereF.test 5b2ba0dbe8074aa13e416b37c753991f0a2492d7 -F test/whereG.test 2533b72ed4a31fd1687230a499b557b911525344 +F test/whereG.test 8189fedf3b98ab581bb70f830175e403a0ef1722 F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c @@ -1161,7 +1161,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P de9a490f594183f337a2ec9e0f87792eac83548b -R ce888b84132e0cad3bcca115a32951d3 -U drh -Z cf9f241149456ab1fa24984e95a412d2 +P 65d2544af9adc1e2f1d193e57f8be0422fb0d5eb +R 6cc54703275bf2ed6708c34ae52cf7ea +T *branch * experimental-costs +T *sym-experimental-costs * +T -sym-trunk * +U dan +Z 868aa60f36dda291ae018583501645e5 diff --git a/manifest.uuid b/manifest.uuid index a2761a597b..4c2b675cff 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -65d2544af9adc1e2f1d193e57f8be0422fb0d5eb \ No newline at end of file +c5a6ec0a880652dc8f4593d9f7acd58ddc3dc5f3 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 9bde27e52f..6a75840525 100644 --- a/src/where.c +++ b/src/where.c @@ -227,7 +227,7 @@ static int whereClauseInsert(WhereClause *pWC, Expr *p, u8 wtFlags){ if( p && ExprHasProperty(p, EP_Unlikely) ){ pTerm->truthProb = sqlite3LogEst(p->iTable) - 99; }else{ - pTerm->truthProb = -1; + pTerm->truthProb = 1; } pTerm->pExpr = sqlite3ExprSkipCollate(p); pTerm->wtFlags = wtFlags; @@ -1975,6 +1975,31 @@ static void whereKeyStats( } #endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ +/* +** If it is not NULL, pTerm is a term that provides an upper or lower +** bound on a range scan. Without considering pTerm, it is estimated +** that the scan will visit nNew rows. This function returns the number +** estimated to be visited after taking pTerm into account. +** +** If the user explicitly specified a likelihood() value for this term, +** then the return value is the likelihood multiplied by the number of +** input rows. Otherwise, this function assumes that an "IS NOT NULL" term +** has a likelihood of 0.50, and any other term a likelihood of 0.25. +*/ +static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){ + LogEst nRet = nNew; + if( pTerm ){ + if( pTerm->truthProb<=0 ){ + nRet += pTerm->truthProb; + }else if( pTerm->wtFlags & TERM_VNULL ){ + nRet -= 10; assert( 10==sqlite3LogEst(2) ); + }else{ + nRet -= 20; assert( 20==sqlite3LogEst(4) ); + } + } + return nRet; +} + /* ** This function is used to estimate the number of rows that will be visited ** by scanning an index for a range of values. The range may have an upper @@ -2127,17 +2152,9 @@ static int whereRangeScanEst( UNUSED_PARAMETER(pBuilder); #endif assert( pLower || pUpper ); - /* TUNING: Each inequality constraint reduces the search space 4-fold. - ** A BETWEEN operator, therefore, reduces the search space 16-fold */ - nNew = nOut; - if( pLower && (pLower->wtFlags & TERM_VNULL)==0 ){ - nNew -= 20; assert( 20==sqlite3LogEst(4) ); - nOut--; - } - if( pUpper ){ - nNew -= 20; assert( 20==sqlite3LogEst(4) ); - nOut--; - } + nNew = whereRangeAdjust(pLower, nOut); + nNew = whereRangeAdjust(pUpper, nNew); + nOut -= (pLower!=0) + (pUpper!=0); if( nNew<10 ) nNew = 10; if( nNewnOut = (LogEst)nOut; @@ -3987,7 +4004,9 @@ static void whereLoopOutputAdjust(WhereClause *pWC, WhereLoop *pLoop){ if( pX==pTerm ) break; if( pX->iParent>=0 && (&pWC->a[pX->iParent])==pTerm ) break; } - if( j<0 ) pLoop->nOut += pTerm->truthProb; + if( j<0 ){ + pLoop->nOut += (pTerm->truthProb<=0 ? pTerm->truthProb : -1); + } } } @@ -4081,6 +4100,7 @@ static int whereLoopAddBtreeIndex( pNew->nOut = saved_nOut; } for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ + LogEst rCostIdx; int nIn = 0; #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 int nRecValid = pBuilder->nRecValid; @@ -4154,7 +4174,8 @@ static int whereLoopAddBtreeIndex( pNew->aLTerm[pNew->nLTerm-2] : 0; } if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ - /* Adjust nOut and rRun for STAT3 range values */ + /* Adjust nOut using stat3/stat4 data. Or, if there is no stat3/stat4 + ** data, using some other estimate. */ assert( pNew->nOut==saved_nOut ); whereRangeScanEst(pParse, pBuilder, pBtm, pTop, pNew); } @@ -4181,13 +4202,16 @@ static int whereLoopAddBtreeIndex( } } #endif + /* Set rCostIdx to the cost of visiting selected rows in index. Add + ** it to pNew->rRun, which is currently set to the cost of the index + ** seek only. Then, if this is a non-covering index, add the cost of + ** visiting the rows in the main table. */ + rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow; + pNew->rRun = sqlite3LogEstAdd(pNew->rRun, rCostIdx); if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){ - /* Each row involves a step of the index, then a binary search of - ** the main table */ - pNew->rRun = sqlite3LogEstAdd(pNew->rRun,rLogSize>27 ? rLogSize-17 : 10); + pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16); } - /* Step cost for each output row */ - pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut); + whereLoopOutputAdjust(pBuilder->pWC, pNew); rc = whereLoopInsert(pBuilder, pNew); if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 @@ -4319,6 +4343,7 @@ static int whereLoopAddBtree( sPk.aiRowEst = aiRowEstPk; sPk.onError = OE_Replace; sPk.pTable = pTab; + sPk.szIdxRow = pTab->szTabRow; aiRowEstPk[0] = pTab->nRowEst; aiRowEstPk[1] = 1; pFirst = pSrc->pTab->pIndex; @@ -4396,10 +4421,8 @@ static int whereLoopAddBtree( /* Full table scan */ pNew->iSortIdx = b ? iSortIdx : 0; - /* TUNING: Cost of full table scan is 3*(N + log2(N)). - ** + The extra 3 factor is to encourage the use of indexed lookups - ** over full scans. FIXME */ - pNew->rRun = sqlite3LogEstAdd(rSize,rLogSize) + 16; + /* TUNING: Cost of full table scan is (N*3.0). */ + pNew->rRun = rSize + 16; whereLoopOutputAdjust(pWC, pNew); rc = whereLoopInsert(pBuilder, pNew); pNew->nOut = rSize; @@ -4426,35 +4449,16 @@ static int whereLoopAddBtree( ) ){ pNew->iSortIdx = b ? iSortIdx : 0; - /* TUNING: The base cost of an index scan is N + log2(N). - ** The log2(N) is for the initial seek to the beginning and the N - ** is for the scan itself. */ - pNew->rRun = sqlite3LogEstAdd(rSize, rLogSize); - if( m==0 ){ - /* TUNING: Cost of a covering index scan is K*(N + log2(N)). - ** + The extra factor K of between 1.1 and 3.0 that depends - ** on the relative sizes of the table and the index. K - ** is smaller for smaller indices, thus favoring them. - ** The upper bound on K (3.0) matches the penalty factor - ** on a full table scan that tries to encourage the use of - ** indexed lookups over full scans. - */ - pNew->rRun += 1 + (15*pProbe->szIdxRow)/pTab->szTabRow; - }else{ - /* TUNING: The cost of scanning a non-covering index is multiplied - ** by log2(N) to account for the binary search of the main table - ** that must happen for each row of the index. - ** TODO: Should there be a multiplier here, analogous to the 3x - ** multiplier for a fulltable scan or covering index scan, to - ** further discourage the use of an index scan? Or is the log2(N) - ** term sufficient discouragement? - ** TODO: What if some or all of the WHERE clause terms can be - ** computed without reference to the original table. Then the - ** penality should reduce to logK where K is the number of output - ** rows. - */ - pNew->rRun += rLogSize; + + /* The cost of visiting the index rows is N*K, where K is + ** between 1.1 and 3.0, depending on the relative sizes of the + ** index and table rows. If this is a non-covering index scan, + ** also add the cost of visiting table rows (N*3.0). */ + pNew->rRun = rSize + 1 + (15*pProbe->szIdxRow)/pTab->szTabRow; + if( m!=0 ){ + pNew->rRun = sqlite3LogEstAdd(pNew->rRun, rSize+16); } + whereLoopOutputAdjust(pWC, pNew); rc = whereLoopInsert(pBuilder, pNew); pNew->nOut = rSize; @@ -4732,8 +4736,7 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ pNew->iSortIdx = 0; memset(&pNew->u, 0, sizeof(pNew->u)); for(i=0; rc==SQLITE_OK && irRun = sSum.a[i].rRun + 18; + pNew->rRun = sSum.a[i].rRun; pNew->nOut = sSum.a[i].nOut; pNew->prereq = sSum.a[i].prereq; rc = whereLoopInsert(pBuilder, pNew); diff --git a/test/analyze9.test b/test/analyze9.test index 820bcdb0e7..125cecf182 100644 --- a/test/analyze9.test +++ b/test/analyze9.test @@ -577,16 +577,16 @@ do_test 13.1 { execsql ANALYZE } {} do_eqp_test 13.2.1 { - SELECT * FROM t1 WHERE a='abc' AND rowid<15 AND b<20 + SELECT * FROM t1 WHERE a='abc' AND rowid<15 AND b<12 } {/SEARCH TABLE t1 USING INDEX i1/} do_eqp_test 13.2.2 { - SELECT * FROM t1 WHERE a='abc' AND rowid<'15' AND b<20 + SELECT * FROM t1 WHERE a='abc' AND rowid<'15' AND b<12 } {/SEARCH TABLE t1 USING INDEX i1/} do_eqp_test 13.3.1 { - SELECT * FROM t1 WHERE a='abc' AND rowid<100 AND b<20 + SELECT * FROM t1 WHERE a='abc' AND rowid<100 AND b<12 } {/SEARCH TABLE t1 USING INDEX i2/} do_eqp_test 13.3.2 { - SELECT * FROM t1 WHERE a='abc' AND rowid<'100' AND b<20 + SELECT * FROM t1 WHERE a='abc' AND rowid<'100' AND b<12 } {/SEARCH TABLE t1 USING INDEX i2/} #------------------------------------------------------------------------- diff --git a/test/autoindex1.test b/test/autoindex1.test index 0e5032b9b9..6cb0ab146a 100644 --- a/test/autoindex1.test +++ b/test/autoindex1.test @@ -97,6 +97,8 @@ do_test autoindex1-210 { PRAGMA automatic_index=ON; ANALYZE; UPDATE sqlite_stat1 SET stat='10000' WHERE tbl='t1'; + -- Table t2 actually contains 8 rows. + UPDATE sqlite_stat1 SET stat='16' WHERE tbl='t2'; ANALYZE sqlite_master; SELECT b, (SELECT d FROM t2 WHERE c=a) FROM t1; } diff --git a/test/where3.test b/test/where3.test index 8fa9fa7840..c2804b5579 100644 --- a/test/where3.test +++ b/test/where3.test @@ -231,6 +231,7 @@ do_execsql_test where3-3.0 { CREATE TABLE t301(a INTEGER PRIMARY KEY,b,c); CREATE INDEX t301c ON t301(c); INSERT INTO t301 VALUES(1,2,3); + INSERT INTO t301 VALUES(2,2,3); CREATE TABLE t302(x, y); INSERT INTO t302 VALUES(4,5); ANALYZE; @@ -251,7 +252,7 @@ do_execsql_test where3-3.2 { } {} do_execsql_test where3-3.3 { SELECT * FROM t301 WHERE c=3 AND a IS NOT NULL; -} {1 2 3} +} {1 2 3 2 2 3} if 0 { # Query planner no longer does this # Verify that when there are multiple tables in a join which must be diff --git a/test/whereG.test b/test/whereG.test index 17d5653223..6274213491 100644 --- a/test/whereG.test +++ b/test/whereG.test @@ -14,6 +14,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +set testprefix whereG do_execsql_test whereG-1.0 { CREATE TABLE composer( @@ -179,5 +180,36 @@ do_execsql_test whereG-4.0 { ORDER BY x; } {right} +#------------------------------------------------------------------------- +# + +reset_db +do_execsql_test 5.1 { + CREATE TABLE t1(a, b, c); + CREATE INDEX i1 ON t1(a, b); +} +do_eqp_test 5.1.2 { + SELECT * FROM t1 WHERE a>? +} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a>?)}} +do_eqp_test 5.1.3 { + SELECT * FROM t1 WHERE likelihood(a>?, 0.9) +} {0 0 0 {SCAN TABLE t1}} + +do_test 5.2 { + for {set i 0} {$i < 100} {incr i} { + execsql { INSERT INTO t1 VALUES('abc', $i, $i); } + } + execsql { INSERT INTO t1 SELECT 'def', b, c FROM t1; } + execsql { ANALYZE } +} {} + +do_eqp_test 5.2.2 { + SELECT * FROM t1 WHERE likelihood(b>?, 0.01) +} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (ANY(a) AND b>?)}} + +do_eqp_test 5.2.3 { + SELECT * FROM t1 WHERE likelihood(b>?, 0.9) +} {0 0 0 {SCAN TABLE t1}} finish_test + From cfc9df76e11b05d275e77d067f359c12044ea943 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 25 Apr 2014 15:01:01 +0000 Subject: [PATCH 12/71] Store values loaded from the stat1 table as logarithmic values in memory. FossilOrigin-Name: 1bd74c49ddab6f53bb6eaa57907eff44c2580dd6 --- manifest | 25 +++++++++++-------------- manifest.uuid | 2 +- src/analyze.c | 19 ++++++++++++------- src/build.c | 26 +++++++++++++++++++------- src/pragma.c | 6 ++++-- src/select.c | 10 +++++----- src/sqliteInt.h | 5 ++++- src/where.c | 30 ++++++++++++++++-------------- 8 files changed, 72 insertions(+), 51 deletions(-) diff --git a/manifest b/manifest index 2b621bd738..0c855c53e6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sto\sthe\sway\sthe\splanner\scalculates\sthe\scosts\sof\svarious\stable\sand\sindex\sscans.\sSome\stest\scases\sstill\sfailing. -D 2014-04-24T20:04:49.939 +C Store\svalues\sloaded\sfrom\sthe\sstat1\stable\sas\slogarithmic\svalues\sin\smemory. +D 2014-04-25T15:01:01.691 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -158,7 +158,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c b00900877f766f116f9e16116f1ccacdc21d82f1 -F src/analyze.c 663e0b291d27eb03c9fd6b421e2d61ba348a2389 +F src/analyze.c 92f1495304dd33b4f9e0b0e5aa030b068ada504d F src/attach.c 3801129015ef59d76bf23c95ef9b0069d18a0c52 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 @@ -167,7 +167,7 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 6c9b51abd404ce5b78b173b6f2248e8cb824758c F src/btree.h d79306df4ed9181b48916737fe8871a4392c4594 F src/btreeInt.h cf180d86b2e9e418f638d65baa425c4c69c0e0e3 -F src/build.c 5bfeea8f302ec2926c9eea321a61daea92a29fa4 +F src/build.c 9f7b2ed2af66dd2d186c0835d1c2a672d1f768e0 F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a @@ -211,18 +211,18 @@ F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 F src/pcache.c d8eafac28290d4bb80332005435db44991d07fc2 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 102e6f5a2fbc646154463eb856d1fd716867b64c -F src/pragma.c 21ece94d4f3e76e8e150deecafb9c7abd398ec67 +F src/pragma.c 810ef31ccfaa233201dcf100637a9777cc24e897 F src/prepare.c 677521ab7132615a8a26107a1d1c3132f44ae337 F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be -F src/select.c bc7feff0fb4c4a1b9d655b717bef166846b48e33 +F src/select.c ed459f7f478a1e533d19c4b953693b3ffa2efd15 F src/shell.c 2afe7a7154e97be0c74c5feacf09626bda8493be F src/sqlite.h.in bde98816e1ba0c9ffef50afe7b32f4e5a8f54fe0 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 03e2f60ccb0745fa2d3a072cb4f75fa29251d2ee +F src/sqliteInt.h bad694fd6b91b10a7a5aafa16fd05b504bad6b6e F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c c12bc20cd649bcae39de3e452bfc1a3f164454ee +F src/where.c 15a5c94c8c93500e141c6cb25af600615dc196d8 F src/whereInt.h 929c1349b5355fd44f22cee5c14d72b3329c58a6 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1161,10 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 65d2544af9adc1e2f1d193e57f8be0422fb0d5eb -R 6cc54703275bf2ed6708c34ae52cf7ea -T *branch * experimental-costs -T *sym-experimental-costs * -T -sym-trunk * +P c5a6ec0a880652dc8f4593d9f7acd58ddc3dc5f3 +R 63fad85eb66cf540e6aa11923b167cbf U dan -Z 868aa60f36dda291ae018583501645e5 +Z 04f4e3645ebd9e33526df3bd26c04a76 diff --git a/manifest.uuid b/manifest.uuid index 4c2b675cff..5436721392 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c5a6ec0a880652dc8f4593d9f7acd58ddc3dc5f3 \ No newline at end of file +1bd74c49ddab6f53bb6eaa57907eff44c2580dd6 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 2a03e292c8..4fbaaa5173 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1371,6 +1371,7 @@ static void decodeIntArray( char *zIntArray, /* String containing int array to decode */ int nOut, /* Number of slots in aOut[] */ tRowcnt *aOut, /* Store integers here */ + LogEst *aLog, /* Or, if aOut==0, here */ Index *pIndex /* Handle extra flags for this index, if not NULL */ ){ char *z = zIntArray; @@ -1389,7 +1390,11 @@ static void decodeIntArray( v = v*10 + c - '0'; z++; } - aOut[i] = v; + if( aOut ){ + aOut[i] = v; + }else{ + aLog[i] = sqlite3LogEst(v); + } if( *z==' ' ) z++; } #ifndef SQLITE_ENABLE_STAT3_OR_STAT4 @@ -1445,12 +1450,12 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ z = argv[2]; if( pIndex ){ - decodeIntArray((char*)z, pIndex->nKeyCol+1, pIndex->aiRowEst, pIndex); - if( pIndex->pPartIdxWhere==0 ) pTable->nRowEst = pIndex->aiRowEst[0]; + decodeIntArray((char*)z, pIndex->nKeyCol+1, 0, pIndex->aiRowLogEst, pIndex); + if( pIndex->pPartIdxWhere==0 ) pTable->nRowLogEst = pIndex->aiRowLogEst[0]; }else{ Index fakeIdx; fakeIdx.szIdxRow = pTable->szTabRow; - decodeIntArray((char*)z, 1, &pTable->nRowEst, &fakeIdx); + decodeIntArray((char*)z, 1, 0, &pTable->nRowLogEst, &fakeIdx); pTable->szTabRow = fakeIdx.szIdxRow; } @@ -1642,9 +1647,9 @@ static int loadStatTbl( pPrevIdx = pIdx; } pSample = &pIdx->aSample[pIdx->nSample]; - decodeIntArray((char*)sqlite3_column_text(pStmt,1), nCol, pSample->anEq, 0); - decodeIntArray((char*)sqlite3_column_text(pStmt,2), nCol, pSample->anLt, 0); - decodeIntArray((char*)sqlite3_column_text(pStmt,3), nCol, pSample->anDLt,0); + decodeIntArray((char*)sqlite3_column_text(pStmt,1),nCol,pSample->anEq,0,0); + decodeIntArray((char*)sqlite3_column_text(pStmt,2),nCol,pSample->anLt,0,0); + decodeIntArray((char*)sqlite3_column_text(pStmt,3),nCol,pSample->anDLt,0,0); /* Take a copy of the sample. Add two 0x00 bytes the end of the buffer. ** This is in case the sample record is corrupted. In that case, the diff --git a/src/build.c b/src/build.c index 4d4155ba90..fb025495dc 100644 --- a/src/build.c +++ b/src/build.c @@ -905,7 +905,7 @@ void sqlite3StartTable( pTable->iPKey = -1; pTable->pSchema = db->aDb[iDb].pSchema; pTable->nRef = 1; - pTable->nRowEst = 1048576; + pTable->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); assert( pParse->pNewTable==0 ); pParse->pNewTable = pTable; @@ -2730,15 +2730,15 @@ Index *sqlite3AllocateIndexObject( nByte = ROUND8(sizeof(Index)) + /* Index structure */ ROUND8(sizeof(char*)*nCol) + /* Index.azColl */ - ROUND8(sizeof(tRowcnt)*(nCol+1) + /* Index.aiRowEst */ + ROUND8(sizeof(LogEst)*(nCol+1) + /* Index.aiRowLogEst */ sizeof(i16)*nCol + /* Index.aiColumn */ sizeof(u8)*nCol); /* Index.aSortOrder */ p = sqlite3DbMallocZero(db, nByte + nExtra); if( p ){ char *pExtra = ((char*)p)+ROUND8(sizeof(Index)); - p->azColl = (char**)pExtra; pExtra += ROUND8(sizeof(char*)*nCol); - p->aiRowEst = (tRowcnt*)pExtra; pExtra += sizeof(tRowcnt)*(nCol+1); - p->aiColumn = (i16*)pExtra; pExtra += sizeof(i16)*nCol; + p->azColl = (char**)pExtra; pExtra += ROUND8(sizeof(char*)*nCol); + p->aiRowLogEst = (LogEst*)pExtra; pExtra += sizeof(LogEst)*(nCol+1); + p->aiColumn = (i16*)pExtra; pExtra += sizeof(i16)*nCol; p->aSortOrder = (u8*)pExtra; p->nColumn = nCol; p->nKeyCol = nCol - 1; @@ -2968,7 +2968,7 @@ Index *sqlite3CreateIndex( if( db->mallocFailed ){ goto exit_create_index; } - assert( EIGHT_BYTE_ALIGNMENT(pIndex->aiRowEst) ); + assert( EIGHT_BYTE_ALIGNMENT(pIndex->aiRowLogEst) ); assert( EIGHT_BYTE_ALIGNMENT(pIndex->azColl) ); pIndex->zName = zExtra; zExtra += nName + 1; @@ -3249,7 +3249,7 @@ exit_create_index: ** Since we do not know, guess 1 million. aiRowEst[1] is an estimate of the ** number of rows in the table that match any particular value of the ** first column of the index. aiRowEst[2] is an estimate of the number -** of rows that match any particular combiniation of the first 2 columns +** of rows that match any particular combination of the first 2 columns ** of the index. And so forth. It must always be the case that * ** aiRowEst[N]<=aiRowEst[N-1] @@ -3260,6 +3260,7 @@ exit_create_index: ** are based on typical values found in actual indices. */ void sqlite3DefaultRowEst(Index *pIdx){ +#if 0 tRowcnt *a = pIdx->aiRowEst; int i; tRowcnt n; @@ -3274,6 +3275,17 @@ void sqlite3DefaultRowEst(Index *pIdx){ if( pIdx->onError!=OE_None ){ a[pIdx->nKeyCol] = 1; } +#endif + /* 1000000, 10, 9, 8, 7, 6, 5, 4, 3, 2 */ + LogEst aVal[] = { 33, 32, 30, 28, 26, 23, 20, 16, 10 }; + LogEst *a = pIdx->aiRowLogEst; + int nCopy = MIN(ArraySize(aVal), pIdx->nKeyCol); + + a[0] = pIdx->pTable->nRowLogEst; + memcpy(&a[1], aVal, nCopy*sizeof(LogEst)); + if( pIdx->onError!=OE_None ){ + a[pIdx->nKeyCol] = 0; + } } /* diff --git a/src/pragma.c b/src/pragma.c index 66d0c3328e..4c69ceb4fd 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1488,13 +1488,15 @@ void sqlite3Pragma( sqlite3VdbeAddOp2(v, OP_Null, 0, 2); sqlite3VdbeAddOp2(v, OP_Integer, (int)sqlite3LogEstToInt(pTab->szTabRow), 3); - sqlite3VdbeAddOp2(v, OP_Integer, (int)pTab->nRowEst, 4); + sqlite3VdbeAddOp2(v, OP_Integer, + (int)sqlite3LogEstToInt(pTab->nRowLogEst), 4); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pIdx->zName, 0); sqlite3VdbeAddOp2(v, OP_Integer, (int)sqlite3LogEstToInt(pIdx->szIdxRow), 3); - sqlite3VdbeAddOp2(v, OP_Integer, (int)pIdx->aiRowEst[0], 4); + sqlite3VdbeAddOp2(v, OP_Integer, + (int)sqlite3LogEstToInt(pIdx->aiRowLogEst[0]), 4); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); } } diff --git a/src/select.c b/src/select.c index 6efdde4b72..dfca6d3f83 100644 --- a/src/select.c +++ b/src/select.c @@ -1690,7 +1690,7 @@ Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){ assert( db->lookaside.bEnabled==0 ); pTab->nRef = 1; pTab->zName = 0; - pTab->nRowEst = 1048576; + pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); selectColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol); selectAddColumnTypeAndCollation(pParse, pTab, pSelect); pTab->iPKey = -1; @@ -3829,7 +3829,7 @@ static int withExpand( pTab->nRef = 1; pTab->zName = sqlite3DbStrDup(db, pCte->zName); pTab->iPKey = -1; - pTab->nRowEst = 1048576; + pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); pTab->tabFlags |= TF_Ephemeral; pFrom->pSelect = sqlite3SelectDup(db, pCte->pSelect, 0); if( db->mallocFailed ) return SQLITE_NOMEM; @@ -4005,7 +4005,7 @@ static int selectExpander(Walker *pWalker, Select *p){ while( pSel->pPrior ){ pSel = pSel->pPrior; } selectColumnsFromExprList(pParse, pSel->pEList, &pTab->nCol, &pTab->aCol); pTab->iPKey = -1; - pTab->nRowEst = 1048576; + pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); pTab->tabFlags |= TF_Ephemeral; #endif }else{ @@ -4655,7 +4655,7 @@ int sqlite3Select( sqlite3SelectDestInit(&dest, SRT_Coroutine, pItem->regReturn); explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); sqlite3Select(pParse, pSub, &dest); - pItem->pTab->nRowEst = (unsigned)pSub->nSelectRow; + pItem->pTab->nRowLogEst = sqlite3LogEst(pSub->nSelectRow); pItem->viaCoroutine = 1; pItem->regResult = dest.iSdst; sqlite3VdbeAddOp1(v, OP_EndCoroutine, pItem->regReturn); @@ -4686,7 +4686,7 @@ int sqlite3Select( sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor); explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); sqlite3Select(pParse, pSub, &dest); - pItem->pTab->nRowEst = (unsigned)pSub->nSelectRow; + pItem->pTab->nRowLogEst = sqlite3LogEst(pSub->nSelectRow); if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr); retAddr = sqlite3VdbeAddOp1(v, OP_Return, pItem->regReturn); VdbeComment((v, "end %s", pItem->pTab->zName)); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 8e81b7a970..3ddcc8610f 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1471,7 +1471,7 @@ struct Table { #ifndef SQLITE_OMIT_CHECK ExprList *pCheck; /* All CHECK constraints */ #endif - tRowcnt nRowEst; /* Estimated rows in table - from sqlite_stat1 table */ + LogEst nRowLogEst; /* Estimated rows in table - from sqlite_stat1 table */ int tnum; /* Root BTree node for this table (see note above) */ i16 iPKey; /* If not negative, use aCol[iPKey] as the primary key */ i16 nCol; /* Number of columns in this table */ @@ -1680,7 +1680,10 @@ struct UnpackedRecord { struct Index { char *zName; /* Name of this index */ i16 *aiColumn; /* Which columns are used by this index. 1st is 0 */ +#if 0 tRowcnt *aiRowEst; /* From ANALYZE: Est. rows selected by each column */ +#endif + LogEst *aiRowLogEst; /* From ANALYZE: Est. rows selected by each column */ Table *pTable; /* The SQL table being indexed */ char *zColAff; /* String defining the affinity of each column */ Index *pNext; /* The next index associated with the same table */ diff --git a/src/where.c b/src/where.c index 6a75840525..909129f666 100644 --- a/src/where.c +++ b/src/where.c @@ -1956,7 +1956,8 @@ static void whereKeyStats( iLower = 0; iUpper = aSample[0].anLt[iCol]; }else{ - iUpper = i>=pIdx->nSample ? pIdx->aiRowEst[0] : aSample[i].anLt[iCol]; + i64 nRow0 = sqlite3LogEstToInt(pIdx->aiRowLogEst[0]); + iUpper = i>=pIdx->nSample ? nRow0 : aSample[i].anLt[iCol]; iLower = aSample[i-1].anEq[iCol] + aSample[i-1].anLt[iCol]; } aStat[1] = (pIdx->nKeyCol>iCol ? pIdx->aAvgEq[iCol] : 1); @@ -2092,7 +2093,7 @@ static int whereRangeScanEst( /* Determine iLower and iUpper using ($P) only. */ if( nEq==0 ){ iLower = 0; - iUpper = p->aiRowEst[0]; + iUpper = sqlite3LogEstToInt(p->aiRowLogEst[0]); }else{ /* Note: this call could be optimized away - since the same values must ** have been requested when testing key $P in whereEqualScanEst(). */ @@ -2251,6 +2252,7 @@ static int whereInScanEst( tRowcnt *pnRow /* Write the revised row estimate here */ ){ Index *p = pBuilder->pNew->u.btree.pIndex; + i64 nRow0 = sqlite3LogEstToInt(p->aiRowLogEst[0]); int nRecValid = pBuilder->nRecValid; int rc = SQLITE_OK; /* Subfunction return code */ tRowcnt nEst; /* Number of rows for a single term */ @@ -2259,14 +2261,14 @@ static int whereInScanEst( assert( p->aSample!=0 ); for(i=0; rc==SQLITE_OK && inExpr; i++){ - nEst = p->aiRowEst[0]; + nEst = nRow0; rc = whereEqualScanEst(pParse, pBuilder, pList->a[i].pExpr, &nEst); nRowEst += nEst; pBuilder->nRecValid = nRecValid; } if( rc==SQLITE_OK ){ - if( nRowEst > p->aiRowEst[0] ) nRowEst = p->aiRowEst[0]; + if( nRowEst > nRow0 ) nRowEst = nRow0; *pnRow = nRowEst; WHERETRACE(0x10,("IN row estimate: est=%g\n", nRowEst)); } @@ -4059,8 +4061,7 @@ static int whereLoopAddBtreeIndex( assert( pNew->u.btree.nEq<=pProbe->nKeyCol ); if( pNew->u.btree.nEq < pProbe->nKeyCol ){ iCol = pProbe->aiColumn[pNew->u.btree.nEq]; - nRowEst = sqlite3LogEst(pProbe->aiRowEst[pNew->u.btree.nEq+1]); - if( nRowEst==0 && pProbe->onError==OE_None ) nRowEst = 1; + nRowEst = pProbe->aiRowLogEst[pNew->u.btree.nEq+1]; }else{ iCol = -1; nRowEst = 0; @@ -4074,7 +4075,7 @@ static int whereLoopAddBtreeIndex( saved_prereq = pNew->prereq; saved_nOut = pNew->nOut; pNew->rSetup = 0; - rLogSize = estLog(sqlite3LogEst(pProbe->aiRowEst[0])); + rLogSize = estLog(pProbe->aiRowLogEst[0]); /* Consider using a skip-scan if there are no WHERE clause constraints ** available for the left-most terms of the index, and if the average @@ -4082,10 +4083,11 @@ static int whereLoopAddBtreeIndex( ** number 18 was found by experimentation to be the payoff point where ** skip-scan become faster than a full-scan. */ + assert( 42==sqlite3LogEst(18) ); if( pTerm==0 && saved_nEq==saved_nSkip && saved_nEq+1nKeyCol - && pProbe->aiRowEst[saved_nEq+1]>=18 /* TUNING: Minimum for skip-scan */ + && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */ && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK ){ LogEst nIter; @@ -4093,7 +4095,7 @@ static int whereLoopAddBtreeIndex( pNew->u.btree.nSkip++; pNew->aLTerm[pNew->nLTerm++] = 0; pNew->wsFlags |= WHERE_SKIPSCAN; - nIter = sqlite3LogEst(pProbe->aiRowEst[0]/pProbe->aiRowEst[saved_nEq+1]); + nIter = pProbe->aiRowLogEst[0] - pProbe->aiRowLogEst[saved_nEq+1]; pNew->rRun = rLogSize + nIter; pNew->nOut += nIter; whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter); @@ -4305,7 +4307,7 @@ static int whereLoopAddBtree( WhereInfo *pWInfo; /* WHERE analysis context */ Index *pProbe; /* An index we are evaluating */ Index sPk; /* A fake index object for the primary key */ - tRowcnt aiRowEstPk[2]; /* The aiRowEst[] value for the sPk index */ + LogEst aiRowEstPk[2]; /* The aiRowLogEst[] value for the sPk index */ i16 aiColumnPk = -1; /* The aColumn[] value for the sPk index */ SrcList *pTabList; /* The FROM clause */ struct SrcList_item *pSrc; /* The FROM clause btree term to add */ @@ -4340,12 +4342,12 @@ static int whereLoopAddBtree( memset(&sPk, 0, sizeof(Index)); sPk.nKeyCol = 1; sPk.aiColumn = &aiColumnPk; - sPk.aiRowEst = aiRowEstPk; + sPk.aiRowLogEst = aiRowEstPk; sPk.onError = OE_Replace; sPk.pTable = pTab; sPk.szIdxRow = pTab->szTabRow; - aiRowEstPk[0] = pTab->nRowEst; - aiRowEstPk[1] = 1; + aiRowEstPk[0] = pTab->nRowLogEst; + aiRowEstPk[1] = 0; pFirst = pSrc->pTab->pIndex; if( pSrc->notIndexed==0 ){ /* The real indices of the table are only considered if the @@ -4354,7 +4356,7 @@ static int whereLoopAddBtree( } pProbe = &sPk; } - rSize = sqlite3LogEst(pTab->nRowEst); + rSize = pTab->nRowLogEst; rLogSize = estLog(rSize); #ifndef SQLITE_OMIT_AUTOMATIC_INDEX From 8ad1d8ba38ef4bf965ab1a657892b0e57be00a97 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 25 Apr 2014 20:22:45 +0000 Subject: [PATCH 13/71] If the user provides likelihood() data for a WHERE clause term used as part of an index key, have the planner use it when calculating the expected number of rows visited by the loop. FossilOrigin-Name: c51efaa5d29ee0a91b9e6a83a8dd82530670811a --- manifest | 18 +++--- manifest.uuid | 2 +- src/sqliteInt.h | 3 - src/where.c | 146 ++++++++++++++++++++++++++++------------------- src/whereInt.h | 1 + test/whereG.test | 18 ++++-- 6 files changed, 111 insertions(+), 77 deletions(-) diff --git a/manifest b/manifest index 0c855c53e6..afdc8889fe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Store\svalues\sloaded\sfrom\sthe\sstat1\stable\sas\slogarithmic\svalues\sin\smemory. -D 2014-04-25T15:01:01.691 +C If\sthe\suser\sprovides\slikelihood()\sdata\sfor\sa\sWHERE\sclause\sterm\sused\sas\spart\sof\san\sindex\skey,\shave\sthe\splanner\suse\sit\swhen\scalculating\sthe\sexpected\snumber\sof\srows\svisited\sby\sthe\sloop. +D 2014-04-25T20:22:45.291 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -222,7 +222,7 @@ F src/shell.c 2afe7a7154e97be0c74c5feacf09626bda8493be F src/sqlite.h.in bde98816e1ba0c9ffef50afe7b32f4e5a8f54fe0 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h bad694fd6b91b10a7a5aafa16fd05b504bad6b6e +F src/sqliteInt.h 63656cfa5a8221c3eb1a182e97d61b1fe2dfd7da F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -291,8 +291,8 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 15a5c94c8c93500e141c6cb25af600615dc196d8 -F src/whereInt.h 929c1349b5355fd44f22cee5c14d72b3329c58a6 +F src/where.c 2b3f47801939c2853c03bba3d5aa83abdd51211e +F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1093,7 +1093,7 @@ F test/whereC.test d6f4ecd4fa2d9429681a5b22a25d2bda8e86ab8a F test/whereD.test fd9120e262f9da3c45940f52aefeef4d15b904e5 F test/whereE.test b3a055eef928c992b0a33198a7b8dc10eea5ad2f F test/whereF.test 5b2ba0dbe8074aa13e416b37c753991f0a2492d7 -F test/whereG.test 8189fedf3b98ab581bb70f830175e403a0ef1722 +F test/whereG.test 0ac23e5e8311b69d87245f4a85112de321031658 F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c @@ -1161,7 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P c5a6ec0a880652dc8f4593d9f7acd58ddc3dc5f3 -R 63fad85eb66cf540e6aa11923b167cbf +P 1bd74c49ddab6f53bb6eaa57907eff44c2580dd6 +R 804d3aff8175381c892eb4eebbb86307 U dan -Z 04f4e3645ebd9e33526df3bd26c04a76 +Z 6dd1c2a2c429274ab108441094bd0963 diff --git a/manifest.uuid b/manifest.uuid index 5436721392..71ddf4b2d3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1bd74c49ddab6f53bb6eaa57907eff44c2580dd6 \ No newline at end of file +c51efaa5d29ee0a91b9e6a83a8dd82530670811a \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 3ddcc8610f..b584ff401d 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1680,9 +1680,6 @@ struct UnpackedRecord { struct Index { char *zName; /* Name of this index */ i16 *aiColumn; /* Which columns are used by this index. 1st is 0 */ -#if 0 - tRowcnt *aiRowEst; /* From ANALYZE: Est. rows selected by each column */ -#endif LogEst *aiRowLogEst; /* From ANALYZE: Est. rows selected by each column */ Table *pTable; /* The SQL table being indexed */ char *zColAff; /* String defining the affinity of each column */ diff --git a/src/where.c b/src/where.c index 909129f666..73d3a22b7c 100644 --- a/src/where.c +++ b/src/where.c @@ -4040,7 +4040,6 @@ static int whereLoopAddBtreeIndex( LogEst saved_nOut; /* Original value of pNew->nOut */ int iCol; /* Index of the column in the table */ int rc = SQLITE_OK; /* Return code */ - LogEst nRowEst; /* Estimated index selectivity */ LogEst rLogSize; /* Logarithm of table size */ WhereTerm *pTop = 0, *pBtm = 0; /* Top and bottom range constraints */ @@ -4061,10 +4060,8 @@ static int whereLoopAddBtreeIndex( assert( pNew->u.btree.nEq<=pProbe->nKeyCol ); if( pNew->u.btree.nEq < pProbe->nKeyCol ){ iCol = pProbe->aiColumn[pNew->u.btree.nEq]; - nRowEst = pProbe->aiRowLogEst[pNew->u.btree.nEq+1]; }else{ iCol = -1; - nRowEst = 0; } pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, iCol, opMask, pProbe); @@ -4095,35 +4092,40 @@ static int whereLoopAddBtreeIndex( pNew->u.btree.nSkip++; pNew->aLTerm[pNew->nLTerm++] = 0; pNew->wsFlags |= WHERE_SKIPSCAN; - nIter = pProbe->aiRowLogEst[0] - pProbe->aiRowLogEst[saved_nEq+1]; - pNew->rRun = rLogSize + nIter; - pNew->nOut += nIter; - whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter); + nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1]; + pNew->nOut -= nIter; + whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul); pNew->nOut = saved_nOut; } for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ + u16 eOp = pTerm->eOperator; /* Shorthand for pTerm->eOperator */ LogEst rCostIdx; + LogEst nOutUnadjusted; /* nOut before IN() and WHERE adjustments */ int nIn = 0; #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 int nRecValid = pBuilder->nRecValid; #endif - if( (pTerm->eOperator==WO_ISNULL || (pTerm->wtFlags&TERM_VNULL)!=0) + if( (eOp==WO_ISNULL || (pTerm->wtFlags&TERM_VNULL)!=0) && (iCol<0 || pSrc->pTab->aCol[iCol].notNull) ){ continue; /* ignore IS [NOT] NULL constraints on NOT NULL columns */ } if( pTerm->prereqRight & pNew->maskSelf ) continue; - assert( pNew->nOut==saved_nOut ); - pNew->wsFlags = saved_wsFlags; pNew->u.btree.nEq = saved_nEq; pNew->nLTerm = saved_nLTerm; if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */ pNew->aLTerm[pNew->nLTerm++] = pTerm; pNew->prereq = (saved_prereq | pTerm->prereqRight) & ~pNew->maskSelf; - pNew->rRun = rLogSize; /* Baseline cost is log2(N). Adjustments below */ - if( pTerm->eOperator & WO_IN ){ + + assert( nInMul==0 + || (pNew->wsFlags & WHERE_COLUMN_NULL)!=0 + || (pNew->wsFlags & WHERE_COLUMN_IN)!=0 + || (pNew->wsFlags & WHERE_SKIPSCAN)!=0 + ); + + if( eOp & WO_IN ){ Expr *pExpr = pTerm->pExpr; pNew->wsFlags |= WHERE_COLUMN_IN; if( ExprHasProperty(pExpr, EP_xIsSelect) ){ @@ -4135,87 +4137,111 @@ static int whereLoopAddBtreeIndex( } assert( nIn>0 ); /* RHS always has 2 or more terms... The parser ** changes "x IN (?)" into "x=?". */ - pNew->rRun += nIn; - pNew->u.btree.nEq++; - pNew->nOut = nRowEst + nInMul + nIn; - }else if( pTerm->eOperator & (WO_EQ) ){ - assert( - (pNew->wsFlags & (WHERE_COLUMN_NULL|WHERE_COLUMN_IN|WHERE_SKIPSCAN))!=0 - || nInMul==0 - ); + + }else if( eOp & (WO_EQ) ){ pNew->wsFlags |= WHERE_COLUMN_EQ; - if( iCol<0 || (nInMul==0 && pNew->u.btree.nEq==pProbe->nKeyCol-1)){ - assert( (pNew->wsFlags & WHERE_COLUMN_IN)==0 || iCol<0 ); + if( iCol<0 || (nInMul==0 && pNew->u.btree.nEq==pProbe->nKeyCol-1) ){ if( iCol>=0 && pProbe->onError==OE_None ){ pNew->wsFlags |= WHERE_UNQ_WANTED; }else{ pNew->wsFlags |= WHERE_ONEROW; } } - pNew->u.btree.nEq++; - pNew->nOut = nRowEst + nInMul; - }else if( pTerm->eOperator & (WO_ISNULL) ){ - pNew->wsFlags |= WHERE_COLUMN_NULL; - pNew->u.btree.nEq++; - /* TUNING: IS NULL selects 2 rows */ - nIn = 10; assert( 10==sqlite3LogEst(2) ); - pNew->nOut = nRowEst + nInMul + nIn; - }else if( pTerm->eOperator & (WO_GT|WO_GE) ){ - testcase( pTerm->eOperator & WO_GT ); - testcase( pTerm->eOperator & WO_GE ); + + }else if( eOp & (WO_GT|WO_GE) ){ + testcase( eOp & WO_GT ); + testcase( eOp & WO_GE ); pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; pBtm = pTerm; pTop = 0; - }else{ - assert( pTerm->eOperator & (WO_LT|WO_LE) ); - testcase( pTerm->eOperator & WO_LT ); - testcase( pTerm->eOperator & WO_LE ); + }else if( (eOp & WO_ISNULL)==0 ){ + assert( eOp & (WO_LT|WO_LE) ); + testcase( eOp & WO_LT ); + testcase( eOp & WO_LE ); pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT; pTop = pTerm; pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ? pNew->aLTerm[pNew->nLTerm-2] : 0; } + + /* At this point pNew->nOut is set to the number of rows expected to + ** be visited by the index scan before considering term pTerm, or the + ** values of nIn and nInMul. In other words, assuming that all + ** "x IN(...)" terms are replaced with "x = ?". This block updates + ** the value of pNew->nOut to account for pTerm (but not nIn/nInMul). */ + assert( pNew->nOut==saved_nOut ); if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ /* Adjust nOut using stat3/stat4 data. Or, if there is no stat3/stat4 ** data, using some other estimate. */ - assert( pNew->nOut==saved_nOut ); whereRangeScanEst(pParse, pBuilder, pBtm, pTop, pNew); - } + }else{ + int nEq = ++pNew->u.btree.nEq; + assert( eOp & (WO_ISNULL|WO_EQ|WO_IN) ); + + assert( pNew->nOut==saved_nOut ); + if( pTerm->truthProb<=0 ){ + assert( (eOp & WO_IN) || nIn==0 ); + pNew->nOut += pTerm->truthProb; + pNew->nOut -= nIn; + pNew->wsFlags |= WHERE_LIKELIHOOD; + }else{ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - if( nInMul==0 - && pProbe->nSample - && pNew->u.btree.nEq<=pProbe->nSampleCol - && OptimizationEnabled(db, SQLITE_Stat3) - ){ - Expr *pExpr = pTerm->pExpr; - tRowcnt nOut = 0; - if( (pTerm->eOperator & (WO_EQ|WO_ISNULL))!=0 ){ - testcase( pTerm->eOperator & WO_EQ ); - testcase( pTerm->eOperator & WO_ISNULL ); - rc = whereEqualScanEst(pParse, pBuilder, pExpr->pRight, &nOut); - }else if( (pTerm->eOperator & WO_IN) - && !ExprHasProperty(pExpr, EP_xIsSelect) ){ - rc = whereInScanEst(pParse, pBuilder, pExpr->x.pList, &nOut); - } - assert( nOut==0 || rc==SQLITE_OK ); - if( nOut ){ - pNew->nOut = sqlite3LogEst(nOut); - if( pNew->nOut>saved_nOut ) pNew->nOut = saved_nOut; + tRowcnt nOut = 0; + if( nInMul==0 + && pProbe->nSample + && pNew->u.btree.nEq<=pProbe->nSampleCol + && OptimizationEnabled(db, SQLITE_Stat3) + && ((eOp & WO_IN)==0 || !ExprHasProperty(pTerm->pExpr, EP_xIsSelect)) + && (pNew->wsFlags & WHERE_LIKELIHOOD)==0 + ){ + Expr *pExpr = pTerm->pExpr; + if( (eOp & (WO_EQ|WO_ISNULL))!=0 ){ + testcase( eOp & WO_EQ ); + testcase( eOp & WO_ISNULL ); + rc = whereEqualScanEst(pParse, pBuilder, pExpr->pRight, &nOut); + }else{ + rc = whereInScanEst(pParse, pBuilder, pExpr->x.pList, &nOut); + } + assert( rc!=SQLITE_OK || nOut>0 ); + if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK; + if( rc!=SQLITE_OK ) break; /* Jump out of the pTerm loop */ + if( nOut ){ + pNew->nOut = sqlite3LogEst(nOut); + if( pNew->nOut>saved_nOut ) pNew->nOut = saved_nOut; + pNew->nOut -= nIn; + } + } + if( nOut==0 ) +#endif + { + pNew->nOut += (pProbe->aiRowLogEst[nEq] - pProbe->aiRowLogEst[nEq-1]); + if( eOp & WO_ISNULL ){ + /* TUNING: If there is no likelihood() value, assume that a + ** "col IS NULL" expression matches twice as many rows + ** as (col=?). */ + pNew->nOut += 10; + } + } } } -#endif + /* Set rCostIdx to the cost of visiting selected rows in index. Add ** it to pNew->rRun, which is currently set to the cost of the index ** seek only. Then, if this is a non-covering index, add the cost of ** visiting the rows in the main table. */ rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow; - pNew->rRun = sqlite3LogEstAdd(pNew->rRun, rCostIdx); + pNew->rRun = sqlite3LogEstAdd(rLogSize, rCostIdx); if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){ pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16); } + nOutUnadjusted = pNew->nOut; + pNew->rRun += nInMul + nIn; + pNew->nOut += nInMul + nIn; whereLoopOutputAdjust(pBuilder->pWC, pNew); rc = whereLoopInsert(pBuilder, pNew); + pNew->nOut = nOutUnadjusted; + if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 && pNew->u.btree.nEq<(pProbe->nKeyCol + (pProbe->zName!=0)) ){ diff --git a/src/whereInt.h b/src/whereInt.h index 72e7530db9..010cd6e8ae 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -458,3 +458,4 @@ struct WhereInfo { #define WHERE_AUTO_INDEX 0x00004000 /* Uses an ephemeral index */ #define WHERE_SKIPSCAN 0x00008000 /* Uses the skip-scan algorithm */ #define WHERE_UNQ_WANTED 0x00010000 /* WHERE_ONEROW would have been helpful*/ +#define WHERE_LIKELIHOOD 0x00020000 /* A likelihood() is affecting nOut */ diff --git a/test/whereG.test b/test/whereG.test index 6274213491..66918a35fa 100644 --- a/test/whereG.test +++ b/test/whereG.test @@ -181,9 +181,14 @@ do_execsql_test whereG-4.0 { } {right} #------------------------------------------------------------------------- -# - +# Test that likelihood() specifications on indexed terms are taken into +# account by various forms of loops. +# +# 5.1.*: open ended range scans +# 5.2.*: skip-scans +# reset_db + do_execsql_test 5.1 { CREATE TABLE t1(a, b, c); CREATE INDEX i1 ON t1(a, b); @@ -202,14 +207,19 @@ do_test 5.2 { execsql { INSERT INTO t1 SELECT 'def', b, c FROM t1; } execsql { ANALYZE } } {} - do_eqp_test 5.2.2 { SELECT * FROM t1 WHERE likelihood(b>?, 0.01) } {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (ANY(a) AND b>?)}} - do_eqp_test 5.2.3 { SELECT * FROM t1 WHERE likelihood(b>?, 0.9) } {0 0 0 {SCAN TABLE t1}} +do_eqp_test 5.3.1 { + SELECT * FROM t1 WHERE a=? +} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?)}} +do_eqp_test 5.3.2 { + SELECT * FROM t1 WHERE likelihood(a=?, 0.9) +} {0 0 0 {SCAN TABLE t1}} + finish_test From 2dd3cdcfab89b47ee8b47260d0cdf73801094018 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 26 Apr 2014 20:21:14 +0000 Subject: [PATCH 14/71] Fix an sqlite3_stmt_status() problem caused by recent changs on this branch. FossilOrigin-Name: dee204092421a239f9f60ab83c3a5b3e24d1baea --- manifest | 15 +++++----- manifest.uuid | 2 +- src/where.c | 10 ++++--- test/cost.test | 68 ++++++++++++++++++++++++++++++++++++++++++++++ test/orderby5.test | 4 +-- 5 files changed, 85 insertions(+), 14 deletions(-) create mode 100644 test/cost.test diff --git a/manifest b/manifest index afdc8889fe..c32bc38c65 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sthe\suser\sprovides\slikelihood()\sdata\sfor\sa\sWHERE\sclause\sterm\sused\sas\spart\sof\san\sindex\skey,\shave\sthe\splanner\suse\sit\swhen\scalculating\sthe\sexpected\snumber\sof\srows\svisited\sby\sthe\sloop. -D 2014-04-25T20:22:45.291 +C Fix\san\ssqlite3_stmt_status()\sproblem\scaused\sby\srecent\schangs\son\sthis\sbranch. +D 2014-04-26T20:21:14.140 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 2b3f47801939c2853c03bba3d5aa83abdd51211e +F src/where.c 488cc018a39d33d97555b5a1e079dfafc32d348a F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -406,6 +406,7 @@ F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804 F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb F test/corruptI.test b3e4203d420490fc3d3062711597bc1dea06a789 +F test/cost.test 8a2b846e896c744b940c29e8bab46416a1324717 F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -722,7 +723,7 @@ F test/orderby1.test 9b524aff9147288da43a6d7ddfdcff47fa2303c6 F test/orderby2.test bc11009f7cd99d96b1b11e57b199b00633eb5b04 F test/orderby3.test 8619d06a3debdcd80a27c0fdea5c40b468854b99 F test/orderby4.test 4d39bfbaaa3ae64d026ca2ff166353d2edca4ba4 -F test/orderby5.test 2490183fef54417209d1df253633a605d46bd350 +F test/orderby5.test 8f08a54836d21fb7c70245360751aedd1c2286fb F test/orderby6.test 8b38138ab0972588240b3fca0985d2e400432859 F test/oserror.test 50417780d0e0d7cd23cf12a8277bb44024765df3 F test/pager1.test 1acbdb14c5952a72dd43129cabdbf69aaa3ed1fa @@ -1161,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 1bd74c49ddab6f53bb6eaa57907eff44c2580dd6 -R 804d3aff8175381c892eb4eebbb86307 +P c51efaa5d29ee0a91b9e6a83a8dd82530670811a +R 95df5cff60bc1a7189139d237d77a9d7 U dan -Z 6dd1c2a2c429274ab108441094bd0963 +Z c6c47e040169e1387f5404e0d25c212f diff --git a/manifest.uuid b/manifest.uuid index 71ddf4b2d3..22512678d7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c51efaa5d29ee0a91b9e6a83a8dd82530670811a \ No newline at end of file +dee204092421a239f9f60ab83c3a5b3e24d1baea \ No newline at end of file diff --git a/src/where.c b/src/where.c index 73d3a22b7c..6d924e0782 100644 --- a/src/where.c +++ b/src/where.c @@ -4147,14 +4147,15 @@ static int whereLoopAddBtreeIndex( pNew->wsFlags |= WHERE_ONEROW; } } - + }else if( eOp & WO_ISNULL ){ + pNew->wsFlags |= WHERE_COLUMN_NULL; }else if( eOp & (WO_GT|WO_GE) ){ testcase( eOp & WO_GT ); testcase( eOp & WO_GE ); pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; pBtm = pTerm; pTop = 0; - }else if( (eOp & WO_ISNULL)==0 ){ + }else{ assert( eOp & (WO_LT|WO_LE) ); testcase( eOp & WO_LT ); testcase( eOp & WO_LE ); @@ -5225,9 +5226,10 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ ** scans instead? */ LogEst rScale, rSortCost; - assert( nOrderBy>0 ); + assert( nOrderBy>0 && 66==sqlite3LogEst(100) ); rScale = sqlite3LogEst((nOrderBy-isOrdered)*100/nOrderBy) - 66; - rSortCost = nRowEst + estLog(nRowEst) + rScale; + rSortCost = nRowEst + estLog(nRowEst) + rScale + 16; + /* TUNING: The cost of implementing DISTINCT using a B-TREE is ** also N*log(N) but it has a larger constant of proportionality. ** Multiply by 3.0. */ diff --git a/test/cost.test b/test/cost.test new file mode 100644 index 0000000000..8df32d8ed1 --- /dev/null +++ b/test/cost.test @@ -0,0 +1,68 @@ +# 2014-04-26 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix cost + + +do_execsql_test 1.1 { + CREATE TABLE t3(id INTEGER PRIMARY KEY, b NOT NULL); + CREATE TABLE t4(c, d, e); + CREATE UNIQUE INDEX i3 ON t3(b); + CREATE UNIQUE INDEX i4 ON t4(c, d); +} +do_eqp_test 1.2 { + SELECT e FROM t3, t4 WHERE b=c ORDER BY b, d; +} { + 0 0 0 {SCAN TABLE t3 USING COVERING INDEX i3} + 0 1 1 {SEARCH TABLE t4 USING INDEX i4 (c=?)} +} + + +do_execsql_test 2.1 { + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a); +} + +# It is better to use an index for ORDER BY than sort externally, even +# if the index is a non-covering index. +do_eqp_test 2.2 { + SELECT * FROM t1 ORDER BY a; +} { + 0 0 0 {SCAN TABLE t1 USING INDEX i1} +} + +do_execsql_test 3.1 { + CREATE TABLE t5(a INTEGER PRIMARY KEY,b,c,d,e,f,g); + CREATE INDEX t5b ON t5(b); + CREATE INDEX t5c ON t5(c); + CREATE INDEX t5d ON t5(d); + CREATE INDEX t5e ON t5(e); + CREATE INDEX t5f ON t5(f); + CREATE INDEX t5g ON t5(g); +} + +do_eqp_test 3.2 { + SELECT a FROM t5 + WHERE b IS NULL OR c IS NULL OR d IS NULL + ORDER BY a; +} { + 0 0 0 {SEARCH TABLE t5 USING INDEX t5b (b=?)} + 0 0 0 {SEARCH TABLE t5 USING INDEX t5c (c=?)} + 0 0 0 {SEARCH TABLE t5 USING INDEX t5d (d=?)} + 0 0 0 {USE TEMP B-TREE FOR ORDER BY} +} + + + +finish_test diff --git a/test/orderby5.test b/test/orderby5.test index bccd469f25..c9cce703ba 100644 --- a/test/orderby5.test +++ b/test/orderby5.test @@ -80,12 +80,12 @@ do_execsql_test 2.1a { EXPLAIN QUERY PLAN SELECT * FROM t2 WHERE a=0 ORDER BY a, b, c; } {~/B-TREE/} + do_execsql_test 2.1b { EXPLAIN QUERY PLAN - SELECT * FROM t1 WHERE a=0 ORDER BY a, b, c; + SELECT * FROM t1 WHERE likelihood(a=0, 0.05) ORDER BY a, b, c; } {/B-TREE/} - do_execsql_test 2.2 { EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE +a=0 ORDER BY a, b, c; From 440e6ff3b023b66ecefa545990d5eeb85bda03ea Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 28 Apr 2014 08:49:54 +0000 Subject: [PATCH 15/71] Fix an error in estimating of the number of rows visited by a range scan. FossilOrigin-Name: d491de62fce69d93e89f65f7713972f7c2c451f7 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 7 ++++++- test/cost.test | 27 +++++++++++++++++++++++++++ 4 files changed, 41 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index c32bc38c65..2a1d6b7303 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\ssqlite3_stmt_status()\sproblem\scaused\sby\srecent\schangs\son\sthis\sbranch. -D 2014-04-26T20:21:14.140 +C Fix\san\serror\sin\sestimating\sof\sthe\snumber\sof\srows\svisited\sby\sa\srange\sscan. +D 2014-04-28T08:49:54.584 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 488cc018a39d33d97555b5a1e079dfafc32d348a +F src/where.c d4efd09b7b2c3604e48b015199f5393578ed417b F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -406,7 +406,7 @@ F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804 F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb F test/corruptI.test b3e4203d420490fc3d3062711597bc1dea06a789 -F test/cost.test 8a2b846e896c744b940c29e8bab46416a1324717 +F test/cost.test 39d014a90b67169f8482ede8713bff694761d879 F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -1162,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P c51efaa5d29ee0a91b9e6a83a8dd82530670811a -R 95df5cff60bc1a7189139d237d77a9d7 +P dee204092421a239f9f60ab83c3a5b3e24d1baea +R 3b87cec4307de74fea71def903d4f518 U dan -Z c6c47e040169e1387f5404e0d25c212f +Z dce0ef4becf30f1f5b6dab0637076369 diff --git a/manifest.uuid b/manifest.uuid index 22512678d7..94e48b42ff 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dee204092421a239f9f60ab83c3a5b3e24d1baea \ No newline at end of file +d491de62fce69d93e89f65f7713972f7c2c451f7 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 6d924e0782..dd80ab4ccb 100644 --- a/src/where.c +++ b/src/where.c @@ -4241,7 +4241,12 @@ static int whereLoopAddBtreeIndex( pNew->nOut += nInMul + nIn; whereLoopOutputAdjust(pBuilder->pWC, pNew); rc = whereLoopInsert(pBuilder, pNew); - pNew->nOut = nOutUnadjusted; + + if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ + pNew->nOut = saved_nOut; + }else{ + pNew->nOut = nOutUnadjusted; + } if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 && pNew->u.btree.nEq<(pProbe->nKeyCol + (pProbe->zName!=0)) diff --git a/test/cost.test b/test/cost.test index 8df32d8ed1..faaa7bdf16 100644 --- a/test/cost.test +++ b/test/cost.test @@ -63,6 +63,33 @@ do_eqp_test 3.2 { 0 0 0 {USE TEMP B-TREE FOR ORDER BY} } +#------------------------------------------------------------------------- +# If there is no likelihood() or stat3 data, SQLite assumes that a closed +# range scan (e.g. one constrained by "col BETWEEN ? AND ?" constraint) +# visits 1/16 of the rows in a table. +# +# Note: 1/17 =~ 0.058 +# Note: 1/15 =~ 0.067 +# +reset_db +do_execsql_test 4.1 { + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a); + CREATE INDEX i2 ON t1(b); +} +do_eqp_test 4.2 { + SELECT * FROM t1 WHERE likelihood(a=?, 0.058) AND b BETWEEN ? AND ?; +} { + 0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?)} +} +do_eqp_test 4.3 { + SELECT * FROM t1 WHERE likelihood(a=?, 0.067) AND b BETWEEN ? AND ?; +} { + 0 0 0 {SEARCH TABLE t1 USING INDEX i2 (b>? AND b Date: Mon, 28 Apr 2014 09:35:31 +0000 Subject: [PATCH 16/71] Modify internal function whereLoopAdjustCost() so that it does not prefer a skip-scan over a regular index scan even if the regular scan uses a subset of the WHERE terms used by the skip-scan. FossilOrigin-Name: 88a5758dcce891eb7be15432ebdc9f80071d413b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 2a1d6b7303..b301a439d1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\serror\sin\sestimating\sof\sthe\snumber\sof\srows\svisited\sby\sa\srange\sscan. -D 2014-04-28T08:49:54.584 +C Modify\sinternal\sfunction\swhereLoopAdjustCost()\sso\sthat\sit\sdoes\snot\sprefer\sa\sskip-scan\sover\sa\sregular\sindex\sscan\seven\sif\sthe\sregular\sscan\suses\sa\ssubset\sof\sthe\sWHERE\sterms\sused\sby\sthe\sskip-scan. +D 2014-04-28T09:35:31.541 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c d4efd09b7b2c3604e48b015199f5393578ed417b +F src/where.c 263f8940aa21adabbc397590046f040c54aca5f4 F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1162,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P dee204092421a239f9f60ab83c3a5b3e24d1baea -R 3b87cec4307de74fea71def903d4f518 +P d491de62fce69d93e89f65f7713972f7c2c451f7 +R a36e9b880c1d76e2d8cd1072dd5871be U dan -Z dce0ef4becf30f1f5b6dab0637076369 +Z 07995bb3734998fef6b8bca04739721a diff --git a/manifest.uuid b/manifest.uuid index 94e48b42ff..8be55d6794 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d491de62fce69d93e89f65f7713972f7c2c451f7 \ No newline at end of file +88a5758dcce891eb7be15432ebdc9f80071d413b \ No newline at end of file diff --git a/src/where.c b/src/where.c index dd80ab4ccb..4c1d81e183 100644 --- a/src/where.c +++ b/src/where.c @@ -3779,9 +3779,11 @@ static int whereLoopCheaperProperSubset( */ static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ if( (pTemplate->wsFlags & WHERE_INDEXED)==0 ) return; + if( (pTemplate->wsFlags & WHERE_SKIPSCAN)!=0 ) return; for(; p; p=p->pNextLoop){ if( p->iTab!=pTemplate->iTab ) continue; if( (p->wsFlags & WHERE_INDEXED)==0 ) continue; + if( (p->wsFlags & WHERE_SKIPSCAN)!=0 ) continue; if( whereLoopCheaperProperSubset(p, pTemplate) ){ /* Adjust pTemplate cost downward so that it is cheaper than its ** subset p */ From b51926e67c4921137cac5e3b66257a445f23ad5f Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 28 Apr 2014 10:00:59 +0000 Subject: [PATCH 17/71] Update test script analyze3.test to account for the fact that SQLite now prefers a full-table scan over a non-covering index scan that visits a large percentage of the table rows. FossilOrigin-Name: 35f46a55d866b9a87c1321aab8e0cfe86ccadb93 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/analyze3.test | 42 ++++++++++++++++++++++++++++++------------ 3 files changed, 37 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index b301a439d1..06eee5c7d2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modify\sinternal\sfunction\swhereLoopAdjustCost()\sso\sthat\sit\sdoes\snot\sprefer\sa\sskip-scan\sover\sa\sregular\sindex\sscan\seven\sif\sthe\sregular\sscan\suses\sa\ssubset\sof\sthe\sWHERE\sterms\sused\sby\sthe\sskip-scan. -D 2014-04-28T09:35:31.541 +C Update\stest\sscript\sanalyze3.test\sto\saccount\sfor\sthe\sfact\sthat\sSQLite\snow\sprefers\sa\sfull-table\sscan\sover\sa\snon-covering\sindex\sscan\sthat\svisits\sa\slarge\spercentage\sof\sthe\stable\srows. +D 2014-04-28T10:00:59.075 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -305,7 +305,7 @@ F test/alter4.test d6c011fa0d6227abba762498cafbb607c9609e93 F test/altermalloc.test e81ac9657ed25c6c5bb09bebfa5a047cd8e4acfc F test/amatch1.test b5ae7065f042b7f4c1c922933f4700add50cdb9f F test/analyze.test 1772936d66471c65221e437b6d1999c3a03166c4 -F test/analyze3.test 412f690dfe95b337475e3e78a84a85d25f6f125d +F test/analyze3.test bf41f0f680dd1e0d44eed5e769531e93a5320275 F test/analyze4.test eff2df19b8dd84529966420f29ea52edc6b56213 F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4 F test/analyze6.test d31defa011a561b938b4608d3538c1b4e0b5e92c @@ -1162,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P d491de62fce69d93e89f65f7713972f7c2c451f7 -R a36e9b880c1d76e2d8cd1072dd5871be +P 88a5758dcce891eb7be15432ebdc9f80071d413b +R a1e1c4a3f11d62d0bb62f198394034d8 U dan -Z 07995bb3734998fef6b8bca04739721a +Z fd306a356911acfe444509c2efb0dade diff --git a/manifest.uuid b/manifest.uuid index 8be55d6794..5b247aefe3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -88a5758dcce891eb7be15432ebdc9f80071d413b \ No newline at end of file +35f46a55d866b9a87c1321aab8e0cfe86ccadb93 \ No newline at end of file diff --git a/test/analyze3.test b/test/analyze3.test index fb26303ee3..e7416d5730 100644 --- a/test/analyze3.test +++ b/test/analyze3.test @@ -103,12 +103,21 @@ do_test analyze3-1.1.1 { } } {1} +do_execsql_test analyze3-1.1.x { + SELECT count(*) FROM t1 WHERE x>200 AND x<300; + SELECT count(*) FROM t1 WHERE x>0 AND x<1100; +} {99 1000} + +# The first of the following two SELECT statements visits 99 rows. So +# it is better to use the index. But the second visits every row in +# the table (1000 in total) so it is better to do a full-table scan. +# do_eqp_test analyze3-1.1.2 { SELECT sum(y) FROM t1 WHERE x>200 AND x<300 } {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (x>? AND x0 AND x<1100 -} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (x>? AND x200 AND x<300 } @@ -125,17 +134,17 @@ do_test analyze3-1.1.6 { } {199 0 14850} do_test analyze3-1.1.7 { sf_execsql { SELECT sum(y) FROM t1 WHERE x>0 AND x<1100 } -} {2000 0 499500} +} {999 999 499500} do_test analyze3-1.1.8 { set l [string range "0" 0 end] set u [string range "1100" 0 end] sf_execsql { SELECT sum(y) FROM t1 WHERE x>$l AND x<$u } -} {2000 0 499500} +} {999 999 499500} do_test analyze3-1.1.9 { set l [expr int(0)] set u [expr int(1100)] sf_execsql { SELECT sum(y) FROM t1 WHERE x>$l AND x<$u } -} {2000 0 499500} +} {999 999 499500} # The following tests are similar to the block above. The difference is @@ -152,12 +161,17 @@ do_test analyze3-1.2.1 { ANALYZE; } } {} +do_execsql_test analyze3-2.1.x { + SELECT count(*) FROM t2 WHERE x>1 AND x<2; + SELECT count(*) FROM t2 WHERE x>0 AND x<99; +} {200 990} do_eqp_test analyze3-1.2.2 { SELECT sum(y) FROM t2 WHERE x>1 AND x<2 } {0 0 0 {SEARCH TABLE t2 USING INDEX i2 (x>? AND x0 AND x<99 -} {0 0 0 {SEARCH TABLE t2 USING INDEX i2 (x>? AND x12 AND x<20 } } {161 0 4760} @@ -173,17 +187,17 @@ do_test analyze3-1.2.6 { } {161 0 integer integer 4760} do_test analyze3-1.2.7 { sf_execsql { SELECT sum(y) FROM t2 WHERE x>0 AND x<99 } -} {1981 0 490555} +} {999 999 490555} do_test analyze3-1.2.8 { set l [string range "0" 0 end] set u [string range "99" 0 end] sf_execsql {SELECT typeof($l), typeof($u), sum(y) FROM t2 WHERE x>$l AND x<$u} -} {1981 0 text text 490555} +} {999 999 text text 490555} do_test analyze3-1.2.9 { set l [expr int(0)] set u [expr int(99)] sf_execsql {SELECT typeof($l), typeof($u), sum(y) FROM t2 WHERE x>$l AND x<$u} -} {1981 0 integer integer 490555} +} {999 999 integer integer 490555} # Same tests a third time. This time, column x has INTEGER affinity and # is not the leftmost column of the table. This triggered a bug causing @@ -199,12 +213,16 @@ do_test analyze3-1.3.1 { ANALYZE; } } {} +do_execsql_test analyze3-1.3.x { + SELECT count(*) FROM t3 WHERE x>200 AND x<300; + SELECT count(*) FROM t3 WHERE x>0 AND x<1100 +} {99 1000} do_eqp_test analyze3-1.3.2 { SELECT sum(y) FROM t3 WHERE x>200 AND x<300 } {0 0 0 {SEARCH TABLE t3 USING INDEX i3 (x>? AND x0 AND x<1100 -} {0 0 0 {SEARCH TABLE t3 USING INDEX i3 (x>? AND x200 AND x<300 } @@ -221,17 +239,17 @@ do_test analyze3-1.3.6 { } {199 0 14850} do_test analyze3-1.3.7 { sf_execsql { SELECT sum(y) FROM t3 WHERE x>0 AND x<1100 } -} {2000 0 499500} +} {999 999 499500} do_test analyze3-1.3.8 { set l [string range "0" 0 end] set u [string range "1100" 0 end] sf_execsql { SELECT sum(y) FROM t3 WHERE x>$l AND x<$u } -} {2000 0 499500} +} {999 999 499500} do_test analyze3-1.3.9 { set l [expr int(0)] set u [expr int(1100)] sf_execsql { SELECT sum(y) FROM t3 WHERE x>$l AND x<$u } -} {2000 0 499500} +} {999 999 499500} #------------------------------------------------------------------------- # Test that the values of bound SQL variables may be used for the LIKE From 9881d60d17451055282b2bde12454fa42aeaabc8 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 28 Apr 2014 12:08:23 +0000 Subject: [PATCH 18/71] Add an extra column to a table in analyze9.test to give the planner a little more reason to select an index. FossilOrigin-Name: 1b95544f84bf83c28cc15f6d0690fdf8a6bb3941 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/analyze9.test | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 06eee5c7d2..04a8c1374d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\stest\sscript\sanalyze3.test\sto\saccount\sfor\sthe\sfact\sthat\sSQLite\snow\sprefers\sa\sfull-table\sscan\sover\sa\snon-covering\sindex\sscan\sthat\svisits\sa\slarge\spercentage\sof\sthe\stable\srows. -D 2014-04-28T10:00:59.075 +C Add\san\sextra\scolumn\sto\sa\stable\sin\sanalyze9.test\sto\sgive\sthe\splanner\sa\slittle\smore\sreason\sto\sselect\san\sindex. +D 2014-04-28T12:08:23.821 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -311,7 +311,7 @@ F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4 F test/analyze6.test d31defa011a561b938b4608d3538c1b4e0b5e92c F test/analyze7.test bb1409afc9e8629e414387ef048b8e0e3e0bdc4f F test/analyze8.test 093d15c1c888eed5034304a98c992f7360130b88 -F test/analyze9.test e219daa58fd8677c6a43d771798cf37d68f51d3e +F test/analyze9.test 623e02a99a78fa12fe5def2fd559032d5d887e0f F test/analyzeA.test 1a5c40079894847976d983ca39c707aaa44b6944 F test/analyzeB.test 8bf35ee0a548aea831bf56762cb8e7fdb1db083d F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b @@ -1162,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 88a5758dcce891eb7be15432ebdc9f80071d413b -R a1e1c4a3f11d62d0bb62f198394034d8 +P 35f46a55d866b9a87c1321aab8e0cfe86ccadb93 +R 8aefc1341eaf7a6ece9efb278712f5a6 U dan -Z fd306a356911acfe444509c2efb0dade +Z ba7221d22e50ff31dad54b0ec475658f diff --git a/manifest.uuid b/manifest.uuid index 5b247aefe3..7dba22fdc4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -35f46a55d866b9a87c1321aab8e0cfe86ccadb93 \ No newline at end of file +1b95544f84bf83c28cc15f6d0690fdf8a6bb3941 \ No newline at end of file diff --git a/test/analyze9.test b/test/analyze9.test index 125cecf182..f25e5924e6 100644 --- a/test/analyze9.test +++ b/test/analyze9.test @@ -566,7 +566,7 @@ foreach {tn schema} { drop_all_tables do_test 13.1 { execsql { - CREATE TABLE t1(a, b, c); + CREATE TABLE t1(a, b, c, d); CREATE INDEX i1 ON t1(a); CREATE INDEX i2 ON t1(b, c); } From 6b6828625b2843a719fb123dc46feb5666b2cda3 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 28 Apr 2014 15:11:25 +0000 Subject: [PATCH 19/71] Update unordered.test to take into account for the fact that SQLite now prefers a full-table scan over a non-covering index scan that visits a large percentage of the table rows. FossilOrigin-Name: 20f468dfbcb247e51446fad411a6e6cc0d130411 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/unordered.test | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 04a8c1374d..05f3d798c0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sextra\scolumn\sto\sa\stable\sin\sanalyze9.test\sto\sgive\sthe\splanner\sa\slittle\smore\sreason\sto\sselect\san\sindex. -D 2014-04-28T12:08:23.821 +C Update\sunordered.test\sto\stake\sinto\saccount\sfor\sthe\sfact\sthat\sSQLite\snow\sprefers\sa\sfull-table\sscan\sover\sa\snon-covering\sindex\sscan\sthat\svisits\sa\slarge\spercentage\sof\sthe\stable\srows. +D 2014-04-28T15:11:25.118 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1023,7 +1023,7 @@ F test/types2.test 3555aacf8ed8dc883356e59efc314707e6247a84 F test/types3.test 99e009491a54f4dc02c06bdbc0c5eea56ae3e25a F test/unique.test 93f8b2ef5ea51b9495f8d6493429b1fd0f465264 F test/unixexcl.test cd6c765f75e50e8e2c2ba763149e5d340ea19825 -F test/unordered.test ef85ac8f2f3c93ed2b9e811b684de73175fc464c +F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8 F test/update.test 1b6c488a8f993d090b7ee9ad0e234faa161b3aeb F test/uri.test 23662b7b61958b0f0e47082de7d06341ccf85d5b F test/utf16align.test 54cd35a27c005a9b6e7815d887718780b6a462ae @@ -1162,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 35f46a55d866b9a87c1321aab8e0cfe86ccadb93 -R 8aefc1341eaf7a6ece9efb278712f5a6 +P 1b95544f84bf83c28cc15f6d0690fdf8a6bb3941 +R 307957877e44688fb3b83b922c0183e6 U dan -Z ba7221d22e50ff31dad54b0ec475658f +Z 085ae09cb0f2460f340245b5be0529cf diff --git a/manifest.uuid b/manifest.uuid index 7dba22fdc4..8fd0bd92ad 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1b95544f84bf83c28cc15f6d0690fdf8a6bb3941 \ No newline at end of file +20f468dfbcb247e51446fad411a6e6cc0d130411 \ No newline at end of file diff --git a/test/unordered.test b/test/unordered.test index d8f7aa6add..147e91f0d9 100644 --- a/test/unordered.test +++ b/test/unordered.test @@ -42,7 +42,7 @@ foreach idxmode {ordered unordered} { 1 "SELECT * FROM t1 ORDER BY a" {0 0 0 {SCAN TABLE t1 USING INDEX i1}} {0 0 0 {SCAN TABLE t1} 0 0 0 {USE TEMP B-TREE FOR ORDER BY}} - 2 "SELECT * FROM t1 WHERE a >?" + 2 "SELECT * FROM t1 WHERE a > 100" {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a>?)}} {0 0 0 {SCAN TABLE t1}} 3 "SELECT * FROM t1 WHERE a = ? ORDER BY rowid" From 42685f211e0ba5610a2702f39a80f950c9d4ae4f Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 28 Apr 2014 19:34:06 +0000 Subject: [PATCH 20/71] The trunk assumes that an open range constraint on an indexed term (col>?) term matches 1/4 of the indexed rows, and that a closed constraint (col BETWEEN ? AND ?) matches 1/64. Change this branch to do the same. FossilOrigin-Name: 4047ac75e2a8f0b330255501c42e4f04e5ab500d --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 6 ++++++ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 05f3d798c0..b037731f09 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sunordered.test\sto\stake\sinto\saccount\sfor\sthe\sfact\sthat\sSQLite\snow\sprefers\sa\sfull-table\sscan\sover\sa\snon-covering\sindex\sscan\sthat\svisits\sa\slarge\spercentage\sof\sthe\stable\srows. -D 2014-04-28T15:11:25.118 +C The\strunk\sassumes\sthat\san\sopen\srange\sconstraint\son\san\sindexed\sterm\s(col>?)\sterm\smatches\s1/4\sof\sthe\sindexed\srows,\sand\sthat\sa\sclosed\sconstraint\s(col\sBETWEEN\s?\sAND\s?)\smatches\s1/64.\sChange\sthis\sbranch\sto\sdo\sthe\ssame. +D 2014-04-28T19:34:06.281 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 263f8940aa21adabbc397590046f040c54aca5f4 +F src/where.c 0a518940065c10ad8dedf8f2d6e73f1e14eb8472 F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1162,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 1b95544f84bf83c28cc15f6d0690fdf8a6bb3941 -R 307957877e44688fb3b83b922c0183e6 +P 20f468dfbcb247e51446fad411a6e6cc0d130411 +R eabc843f8d46cb0d3b0b7dd8faf4ef00 U dan -Z 085ae09cb0f2460f340245b5be0529cf +Z 37bdf577e72e4cd110b1edb4853d30e0 diff --git a/manifest.uuid b/manifest.uuid index 8fd0bd92ad..405d43fc27 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -20f468dfbcb247e51446fad411a6e6cc0d130411 \ No newline at end of file +4047ac75e2a8f0b330255501c42e4f04e5ab500d \ No newline at end of file diff --git a/src/where.c b/src/where.c index 4c1d81e183..4b2eef9c52 100644 --- a/src/where.c +++ b/src/where.c @@ -2155,6 +2155,12 @@ static int whereRangeScanEst( assert( pLower || pUpper ); nNew = whereRangeAdjust(pLower, nOut); nNew = whereRangeAdjust(pUpper, nNew); + /* TUNING: If there is both an upper and lower limit, assume the range is + ** reduced by an additional 75%. This means that, by default, an open-ended + ** range query (e.g. col > ?) is assumed to match 1/4 of the rows in the + ** index. While a closed range (e.g. col BETWEEN ? AND ?) is estimated to + ** match 1/64 of the index. */ + if( pLower && pUpper ) nNew -= 20; nOut -= (pLower!=0) + (pUpper!=0); if( nNew<10 ) nNew = 10; if( nNew Date: Mon, 28 Apr 2014 20:11:20 +0000 Subject: [PATCH 21/71] Do not reduce the number of rows scanned at all for "IS NOT NULL" constraints. Fix a bug in calculating the number of rows visited by scans of partial indicies. FossilOrigin-Name: a8ae93f0cffa116df0ba34d46a53f49d42dace41 --- manifest | 16 ++++---- manifest.uuid | 2 +- src/where.c | 8 ++-- test/cost.test | 95 ++++++++++++++++++++++++++++++++++++++++++++++++ test/index6.test | 5 ++- 5 files changed, 112 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index b037731f09..823653ac7d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\strunk\sassumes\sthat\san\sopen\srange\sconstraint\son\san\sindexed\sterm\s(col>?)\sterm\smatches\s1/4\sof\sthe\sindexed\srows,\sand\sthat\sa\sclosed\sconstraint\s(col\sBETWEEN\s?\sAND\s?)\smatches\s1/64.\sChange\sthis\sbranch\sto\sdo\sthe\ssame. -D 2014-04-28T19:34:06.281 +C Do\snot\sreduce\sthe\snumber\sof\srows\sscanned\sat\sall\sfor\s"IS\sNOT\sNULL"\sconstraints.\sFix\sa\sbug\sin\scalculating\sthe\snumber\sof\srows\svisited\sby\sscans\sof\spartial\sindicies. +D 2014-04-28T20:11:20.674 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 0a518940065c10ad8dedf8f2d6e73f1e14eb8472 +F src/where.c da9c2c1d0c2ecf51ea3b7a27c826999972f24086 F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -406,7 +406,7 @@ F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804 F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb F test/corruptI.test b3e4203d420490fc3d3062711597bc1dea06a789 -F test/cost.test 39d014a90b67169f8482ede8713bff694761d879 +F test/cost.test 84473f27749e0f3b6837a8e1403967010c347ca5 F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -611,7 +611,7 @@ F test/index2.test ee83c6b5e3173a3d7137140d945d9a5d4fdfb9d6 F test/index3.test 55a90cff99834305e8141df7afaef39674b57062 F test/index4.test ab92e736d5946840236cd61ac3191f91a7856bf6 F test/index5.test fc07c14193c0430814e7a08b5da46888ee795c33 -F test/index6.test a0a2d286ffa6d35813f5003fdb7be124825b4422 +F test/index6.test fb370966ac3cd0989053dd5385757b5c3e24ab6a F test/index7.test a3baf9a625bda7fd49471e99aeae04095fbfeecf F test/indexedby.test b2f22f3e693a53813aa3f50b812eb609ba6df1ec F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d @@ -1162,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 20f468dfbcb247e51446fad411a6e6cc0d130411 -R eabc843f8d46cb0d3b0b7dd8faf4ef00 +P 4047ac75e2a8f0b330255501c42e4f04e5ab500d +R fc72ae401d0e8a316c9dd5e0ec98069f U dan -Z 37bdf577e72e4cd110b1edb4853d30e0 +Z f6f096240da2e8d5cf26af9c9310dce5 diff --git a/manifest.uuid b/manifest.uuid index 405d43fc27..99d11f9b9b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4047ac75e2a8f0b330255501c42e4f04e5ab500d \ No newline at end of file +a8ae93f0cffa116df0ba34d46a53f49d42dace41 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 4b2eef9c52..8abe6bd1a1 100644 --- a/src/where.c +++ b/src/where.c @@ -1992,9 +1992,7 @@ static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){ if( pTerm ){ if( pTerm->truthProb<=0 ){ nRet += pTerm->truthProb; - }else if( pTerm->wtFlags & TERM_VNULL ){ - nRet -= 10; assert( 10==sqlite3LogEst(2) ); - }else{ + }else if( (pTerm->wtFlags & TERM_VNULL)==0 ){ nRet -= 20; assert( 20==sqlite3LogEst(4) ); } } @@ -2153,14 +2151,17 @@ static int whereRangeScanEst( UNUSED_PARAMETER(pBuilder); #endif assert( pLower || pUpper ); + assert( pUpper==0 || (pUpper->wtFlags & TERM_VNULL)==0 ); nNew = whereRangeAdjust(pLower, nOut); nNew = whereRangeAdjust(pUpper, nNew); + /* TUNING: If there is both an upper and lower limit, assume the range is ** reduced by an additional 75%. This means that, by default, an open-ended ** range query (e.g. col > ?) is assumed to match 1/4 of the rows in the ** index. While a closed range (e.g. col BETWEEN ? AND ?) is estimated to ** match 1/64 of the index. */ if( pLower && pUpper ) nNew -= 20; + nOut -= (pLower!=0) + (pUpper!=0); if( nNew<10 ) nNew = 10; if( nNewiTab, pWC, pProbe->pPartIdxWhere) ){ continue; /* Partial index inappropriate for this query */ } + rSize = pProbe->aiRowLogEst[0]; pNew->u.btree.nEq = 0; pNew->u.btree.nSkip = 0; pNew->nLTerm = 0; diff --git a/test/cost.test b/test/cost.test index faaa7bdf16..0f9314fc8e 100644 --- a/test/cost.test +++ b/test/cost.test @@ -89,6 +89,101 @@ do_eqp_test 4.3 { } +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 5.1 { + CREATE TABLE t2(x, y); + CREATE INDEX t2i1 ON t2(x); +} + +do_eqp_test 5.2 { + SELECT * FROM t2 ORDER BY x, y; +} {} +#exit + +# TODO: Check this one out!! +# set sqlite_where_trace 0xfff +do_eqp_test 5.3 { + SELECT * FROM t2 WHERE x BETWEEN ? AND ? ORDER BY rowid; +} {} +#exit + +# where7.test, where8.test: +# +do_execsql_test 6.1 { + CREATE TABLE t3(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX t3i1 ON t3(b); + CREATE INDEX t3i2 ON t3(c); +} + +#set sqlite_where_trace 0xfff +# eqp.test +do_eqp_test 6.2 { + SELECT a FROM t3 WHERE (b BETWEEN 2 AND 4) OR c=100 ORDER BY a +} { +} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 7.1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY,b,c,d,e,f,g); + CREATE INDEX t1b ON t1(b); + CREATE INDEX t1c ON t1(c); + CREATE INDEX t1d ON t1(d); + CREATE INDEX t1e ON t1(e); + CREATE INDEX t1f ON t1(f); + CREATE INDEX t1g ON t1(g); +} + +do_eqp_test 7.2 { + SELECT a FROM t1 + WHERE (b>=950 AND b<=1010) OR (b IS NULL AND c NOT NULL) + ORDER BY a +} { +} + +#set sqlite_where_trace 0xfff +do_eqp_test 7.3 { + SELECT rowid FROM t1 + WHERE (+b IS NULL AND c NOT NULL AND d NOT NULL) + OR (b NOT NULL AND c IS NULL AND d NOT NULL) + OR (b NOT NULL AND c NOT NULL AND d IS NULL) +} {} +#exit + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 8.1 { + CREATE TABLE composer( + cid INTEGER PRIMARY KEY, + cname TEXT + ); + CREATE TABLE album( + aid INTEGER PRIMARY KEY, + aname TEXT + ); + CREATE TABLE track( + tid INTEGER PRIMARY KEY, + cid INTEGER REFERENCES composer, + aid INTEGER REFERENCES album, + title TEXT + ); + CREATE INDEX track_i1 ON track(cid); + CREATE INDEX track_i2 ON track(aid); +} + +do_eqp_test 8.2 { + SELECT DISTINCT aname + FROM album, composer, track + WHERE cname LIKE '%bach%' + AND unlikely(composer.cid=track.cid) + AND unlikely(album.aid=track.aid); +} { +} + finish_test diff --git a/test/index6.test b/test/index6.test index 3451e5c1df..68bdd06c14 100644 --- a/test/index6.test +++ b/test/index6.test @@ -145,11 +145,11 @@ do_test index6-2.1 { execsql { CREATE TABLE t2(a,b); INSERT INTO t2(a,b) SELECT value, value FROM nums WHERE value<1000; - UPDATE t2 SET a=NULL WHERE b%5==0; + UPDATE t2 SET a=NULL WHERE b%2==0; CREATE INDEX t2a1 ON t2(a) WHERE a IS NOT NULL; SELECT count(*) FROM t2 WHERE a IS NOT NULL; } -} {800} +} {500} do_test index6-2.2 { execsql { EXPLAIN QUERY PLAN @@ -157,6 +157,7 @@ do_test index6-2.2 { } } {/.* TABLE t2 USING INDEX t2a1 .*/} ifcapable stat4||stat3 { + execsql ANALYZE do_test index6-2.3stat4 { execsql { EXPLAIN QUERY PLAN From 67a03cfd6e5213f1b7997356d5db620bd21e5329 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 29 Apr 2014 12:01:35 +0000 Subject: [PATCH 22/71] Fix a test case so that it updates sqlite_stat1 consistently. FossilOrigin-Name: 2dc5a0b55567f13f0528ed17242e680cde2f2a29 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/skipscan2.test | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 823653ac7d..a7e8fbdb50 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sreduce\sthe\snumber\sof\srows\sscanned\sat\sall\sfor\s"IS\sNOT\sNULL"\sconstraints.\sFix\sa\sbug\sin\scalculating\sthe\snumber\sof\srows\svisited\sby\sscans\sof\spartial\sindicies. -D 2014-04-28T20:11:20.674 +C Fix\sa\stest\scase\sso\sthat\sit\supdates\ssqlite_stat1\sconsistently. +D 2014-04-29T12:01:35.119 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -814,7 +814,7 @@ F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3 F test/shrink.test 8c70f62b6e8eb4d54533de6d65bd06b1b9a17868 F test/sidedelete.test f0ad71abe6233e3b153100f3b8d679b19a488329 F test/skipscan1.test bed8cbe9d554c8c27afb6c88500f704c86a9196f -F test/skipscan2.test 5a4db0799c338ddbacb154aaa5589c0254b36a8d +F test/skipscan2.test d77f79cdbba25f0f6f35298136cff21a7d7a553a F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test 0e4456e729e5a92a625907c63dcdedfbe72c5dc5 @@ -1162,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 4047ac75e2a8f0b330255501c42e4f04e5ab500d -R fc72ae401d0e8a316c9dd5e0ec98069f +P a8ae93f0cffa116df0ba34d46a53f49d42dace41 +R 176e01464114f39a9f52a0420836d3f7 U dan -Z f6f096240da2e8d5cf26af9c9310dce5 +Z 9c22069b17d97b138b62636ee980813e diff --git a/manifest.uuid b/manifest.uuid index 99d11f9b9b..5debc1c5f8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a8ae93f0cffa116df0ba34d46a53f49d42dace41 \ No newline at end of file +2dc5a0b55567f13f0528ed17242e680cde2f2a29 \ No newline at end of file diff --git a/test/skipscan2.test b/test/skipscan2.test index e39b16ed27..27d193e94d 100644 --- a/test/skipscan2.test +++ b/test/skipscan2.test @@ -74,6 +74,7 @@ do_execsql_test skipscan2-1.4 { -- of a skip-scan. So make a manual adjustment to the stat1 table -- to make it seem like there are many more. UPDATE sqlite_stat1 SET stat='10000 5000 20' WHERE idx='people_idx1'; + UPDATE sqlite_stat1 SET stat='10000 1' WHERE idx='sqlite_autoindex_people_1'; ANALYZE sqlite_master; } db cache flush From 09e1df6c009bc90558ec0250f005bb59fe8f808e Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 29 Apr 2014 16:10:22 +0000 Subject: [PATCH 23/71] Ignore likelihood() values on indexed IPK lookups. FossilOrigin-Name: 5bb7757a7b32a74482d3e93e9c9eea02273fe981 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 2 +- test/cost.test | 36 ++++++++++++++++++++++-------------- 4 files changed, 31 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index a7e8fbdb50..cf16f91a6e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stest\scase\sso\sthat\sit\supdates\ssqlite_stat1\sconsistently. -D 2014-04-29T12:01:35.119 +C Ignore\slikelihood()\svalues\son\sindexed\sIPK\slookups. +D 2014-04-29T16:10:22.104 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c da9c2c1d0c2ecf51ea3b7a27c826999972f24086 +F src/where.c 397bd5d4f402238c396d73b3c98b9fa2c312eea4 F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -406,7 +406,7 @@ F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804 F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb F test/corruptI.test b3e4203d420490fc3d3062711597bc1dea06a789 -F test/cost.test 84473f27749e0f3b6837a8e1403967010c347ca5 +F test/cost.test 62386ccac862ea9a808a0954037f430690974b0f F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -1162,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P a8ae93f0cffa116df0ba34d46a53f49d42dace41 -R 176e01464114f39a9f52a0420836d3f7 +P 2dc5a0b55567f13f0528ed17242e680cde2f2a29 +R b5feca3cdcf90dbcdc4d81625fcb13e8 U dan -Z 9c22069b17d97b138b62636ee980813e +Z d7294cbb283905147264033ecb5a94d9 diff --git a/manifest.uuid b/manifest.uuid index 5debc1c5f8..0097b17163 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2dc5a0b55567f13f0528ed17242e680cde2f2a29 \ No newline at end of file +5bb7757a7b32a74482d3e93e9c9eea02273fe981 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 8abe6bd1a1..566c907e55 100644 --- a/src/where.c +++ b/src/where.c @@ -4189,7 +4189,7 @@ static int whereLoopAddBtreeIndex( assert( eOp & (WO_ISNULL|WO_EQ|WO_IN) ); assert( pNew->nOut==saved_nOut ); - if( pTerm->truthProb<=0 ){ + if( pTerm->truthProb<=0 && iCol>=0 ){ assert( (eOp & WO_IN) || nIn==0 ); pNew->nOut += pTerm->truthProb; pNew->nOut -= nIn; diff --git a/test/cost.test b/test/cost.test index 0f9314fc8e..979815f9f7 100644 --- a/test/cost.test +++ b/test/cost.test @@ -66,10 +66,10 @@ do_eqp_test 3.2 { #------------------------------------------------------------------------- # If there is no likelihood() or stat3 data, SQLite assumes that a closed # range scan (e.g. one constrained by "col BETWEEN ? AND ?" constraint) -# visits 1/16 of the rows in a table. +# visits 1/64 of the rows in a table. # -# Note: 1/17 =~ 0.058 -# Note: 1/15 =~ 0.067 +# Note: 1/63 =~ 0.016 +# Note: 1/65 =~ 0.015 # reset_db do_execsql_test 4.1 { @@ -78,12 +78,12 @@ do_execsql_test 4.1 { CREATE INDEX i2 ON t1(b); } do_eqp_test 4.2 { - SELECT * FROM t1 WHERE likelihood(a=?, 0.058) AND b BETWEEN ? AND ?; + SELECT * FROM t1 WHERE likelihood(a=?, 0.014) AND b BETWEEN ? AND ?; } { 0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?)} } do_eqp_test 4.3 { - SELECT * FROM t1 WHERE likelihood(a=?, 0.067) AND b BETWEEN ? AND ?; + SELECT * FROM t1 WHERE likelihood(a=?, 0.016) AND b BETWEEN ? AND ?; } { 0 0 0 {SEARCH TABLE t1 USING INDEX i2 (b>? AND b? AND x? AND b=950 AND b<=1010) OR (b IS NULL AND c NOT NULL) ORDER BY a } { + 0 0 0 {SEARCH TABLE t1 USING INDEX t1b (b>? AND b Date: Tue, 29 Apr 2014 19:01:57 +0000 Subject: [PATCH 24/71] Test that the default values used when sqlite_stat1 data is not available are calculated correctly. Fixes for the same. FossilOrigin-Name: e2d42f909de85a0586389f2dc0e654f7af2e351a --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/build.c | 35 +++++++++++++++-------------------- test/cost.test | 45 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 28 deletions(-) diff --git a/manifest b/manifest index 5dc1bf6a38..f4b5efc137 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\schanges\sinto\sthis\sbranch. -D 2014-04-29T16:46:24.176 +C Test\sthat\sthe\sdefault\svalues\sused\swhen\ssqlite_stat1\sdata\sis\snot\savailable\sare\scalculated\scorrectly.\sFixes\sfor\sthe\ssame. +D 2014-04-29T19:01:57.481 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -168,7 +168,7 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 6c9b51abd404ce5b78b173b6f2248e8cb824758c F src/btree.h d79306df4ed9181b48916737fe8871a4392c4594 F src/btreeInt.h cf180d86b2e9e418f638d65baa425c4c69c0e0e3 -F src/build.c 9f7b2ed2af66dd2d186c0835d1c2a672d1f768e0 +F src/build.c 02665ca158431a0926b10cbd7d8178a4c9fc4a22 F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a @@ -407,7 +407,7 @@ F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804 F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb F test/corruptI.test b3e4203d420490fc3d3062711597bc1dea06a789 -F test/cost.test 62386ccac862ea9a808a0954037f430690974b0f +F test/cost.test 41b350fcc811a4fcccffbab06a3a79f33b8ea0de F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 5bb7757a7b32a74482d3e93e9c9eea02273fe981 af2cbe64adab5f9e3b0f3da00d06428088589d7f -R 9c74729ef8e10b97c890b0be6d308bd3 +P d74299f037f3a6a4b3bce8b4d1c76c407c1f3b3e +R e73cccb2b7e246d7f5476a97d396afde U dan -Z e5760c6e9001de9044df542c0c7c165c +Z 19f8efc43d85a395f7ff59d4035b6f55 diff --git a/manifest.uuid b/manifest.uuid index e9d3392789..92aa1b2d7c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d74299f037f3a6a4b3bce8b4d1c76c407c1f3b3e \ No newline at end of file +e2d42f909de85a0586389f2dc0e654f7af2e351a \ No newline at end of file diff --git a/src/build.c b/src/build.c index fb025495dc..a79abed3b6 100644 --- a/src/build.c +++ b/src/build.c @@ -3260,32 +3260,27 @@ exit_create_index: ** are based on typical values found in actual indices. */ void sqlite3DefaultRowEst(Index *pIdx){ -#if 0 - tRowcnt *a = pIdx->aiRowEst; - int i; - tRowcnt n; - assert( a!=0 ); - a[0] = pIdx->pTable->nRowEst; - if( a[0]<10 ) a[0] = 10; - n = 10; - for(i=1; i<=pIdx->nKeyCol; i++){ - a[i] = n; - if( n>5 ) n--; - } - if( pIdx->onError!=OE_None ){ - a[pIdx->nKeyCol] = 1; - } -#endif - /* 1000000, 10, 9, 8, 7, 6, 5, 4, 3, 2 */ - LogEst aVal[] = { 33, 32, 30, 28, 26, 23, 20, 16, 10 }; + /* 10, 9, 8, 7, 6 */ + LogEst aVal[] = { 33, 32, 30, 28, 26 }; LogEst *a = pIdx->aiRowLogEst; int nCopy = MIN(ArraySize(aVal), pIdx->nKeyCol); + int i; + /* Set the first entry (number of rows in the index) to the estimated + ** number of rows in the table. Or 10, if the estimated number of rows + ** in the table is less than that. */ a[0] = pIdx->pTable->nRowLogEst; + if( a[0]<33 ) a[0] = 33; assert( 33==sqlite3LogEst(10) ); + + /* Estimate that a[1] is 10, a[2] is 9, a[3] is 8, a[4] is 7, a[5] is + ** 6 and each subsequent value (if any) is 5. */ memcpy(&a[1], aVal, nCopy*sizeof(LogEst)); - if( pIdx->onError!=OE_None ){ - a[pIdx->nKeyCol] = 0; + for(i=nCopy+1; i<=pIdx->nKeyCol; i++){ + a[i] = 23; assert( 23==sqlite3LogEst(5) ); } + + assert( 0==sqlite3LogEst(1) ); + if( pIdx->onError!=OE_None ) a[pIdx->nKeyCol] = 0; } /* diff --git a/test/cost.test b/test/cost.test index 979815f9f7..e04dc9392a 100644 --- a/test/cost.test +++ b/test/cost.test @@ -192,6 +192,51 @@ do_eqp_test 8.2 { 0 0 0 {USE TEMP B-TREE FOR DISTINCT} } +#------------------------------------------------------------------------- +# +do_execsql_test 9.1 { + CREATE TABLE t1( + a,b,c,d,e, f,g,h,i,j, + k,l,m,n,o, p,q,r,s,t + ); + CREATE INDEX i1 ON t1(k,l,m,n,o,p,q,r,s,t); +} +do_test 9.2 { + for {set i 0} {$i < 100} {incr i} { + execsql { INSERT INTO t1 DEFAULT VALUES } + } + execsql { + ANALYZE; + CREATE INDEX i2 ON t1(a,b,c,d,e,f,g,h,i,j); + } +} {} + +set L [list a=? b=? c=? d=? e=? f=? g=? h=? i=? j=?] +foreach {tn nTerm nRow} { + 1 1 10 + 2 2 9 + 3 3 8 + 4 4 7 + 5 5 6 + 6 6 5 + 7 7 5 + 8 8 5 + 9 9 5 + 10 10 5 +} { + set w [join [lrange $L 0 [expr $nTerm-1]] " AND "] + set p1 [expr ($nRow-1) / 100.0] + set p2 [expr ($nRow+1) / 100.0] + + set sql1 "SELECT * FROM t1 WHERE likelihood(k=?, $p1) AND $w" + set sql2 "SELECT * FROM t1 WHERE likelihood(k=?, $p2) AND $w" + + do_eqp_test 9.3.$tn.1 $sql1 {/INDEX i1/} + do_eqp_test 9.3.$tn.2 $sql2 {/INDEX i2/} +} + + + finish_test From 224155dd1c165a133279ebaa52f80f771844cedc Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 30 Apr 2014 13:19:09 +0000 Subject: [PATCH 25/71] Fix long-standing typos in comments. FossilOrigin-Name: b9f91317c34d07769a95dc2f905a6ccabceb64a3 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/sqliteInt.h | 4 ++-- src/util.c | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index f4b5efc137..44e14c4938 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Test\sthat\sthe\sdefault\svalues\sused\swhen\ssqlite_stat1\sdata\sis\snot\savailable\sare\scalculated\scorrectly.\sFixes\sfor\sthe\ssame. -D 2014-04-29T19:01:57.481 +C Fix\slong-standing\stypos\sin\scomments. +D 2014-04-30T13:19:09.070 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -223,7 +223,7 @@ F src/shell.c 2afe7a7154e97be0c74c5feacf09626bda8493be F src/sqlite.h.in bde98816e1ba0c9ffef50afe7b32f4e5a8f54fe0 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 63656cfa5a8221c3eb1a182e97d61b1fe2dfd7da +F src/sqliteInt.h b2947801eccefd7ba3e5f14e1353289351a83cf3 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -277,7 +277,7 @@ F src/tokenize.c 6da2de6e12218ccb0aea5184b56727d011f4bee7 F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c -F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf +F src/util.c 2b5fb283a190aacdb286f7835a447c45b345b83c F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 F src/vdbe.c 699693bea6710ed436392c928b02cb4e91944137 F src/vdbe.h 394464909ed682334aa3d5831aae0c2fe2abef94 @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P d74299f037f3a6a4b3bce8b4d1c76c407c1f3b3e -R e73cccb2b7e246d7f5476a97d396afde -U dan -Z 19f8efc43d85a395f7ff59d4035b6f55 +P e2d42f909de85a0586389f2dc0e654f7af2e351a +R cdeda4c4d8ae1ea7276fce5a8779e48d +U drh +Z 6915f97ffb226384426800091a7da39d diff --git a/manifest.uuid b/manifest.uuid index 92aa1b2d7c..42cd607e10 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e2d42f909de85a0586389f2dc0e654f7af2e351a \ No newline at end of file +b9f91317c34d07769a95dc2f905a6ccabceb64a3 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index b584ff401d..4bd65a690f 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -525,10 +525,10 @@ typedef INT8_TYPE i8; /* 1-byte signed integer */ ** gives a possible range of values of approximately 1.0e986 to 1e-986. ** But the allowed values are "grainy". Not every value is representable. ** For example, quantities 16 and 17 are both represented by a LogEst -** of 40. However, since LogEst quantatites are suppose to be estimates, +** of 40. However, since LogEst quantaties are suppose to be estimates, ** not exact values, this imprecision is not a problem. ** -** "LogEst" is short for "Logarithimic Estimate". +** "LogEst" is short for "Logarithmic Estimate". ** ** Examples: ** 1 -> 0 20 -> 43 10000 -> 132 diff --git a/src/util.c b/src/util.c index d88c17b759..577d552ac3 100644 --- a/src/util.c +++ b/src/util.c @@ -1246,8 +1246,8 @@ LogEst sqlite3LogEstAdd(LogEst a, LogEst b){ } /* -** Convert an integer into a LogEst. In other words, compute a -** good approximatation for 10*log2(x). +** Convert an integer into a LogEst. In other words, compute an +** approximation for 10*log2(x). */ LogEst sqlite3LogEst(u64 x){ static LogEst a[] = { 0, 2, 3, 5, 6, 7, 8, 9 }; From 253666e5203974b28d990e321b243153f3a432dd Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 30 Apr 2014 14:22:38 +0000 Subject: [PATCH 26/71] Improved rendering of LogEst values corresponding to real values near 0.0 in the tool/logest.c utility program. FossilOrigin-Name: 32910c8c595858245bb7ecfe3aa0f90eeae641af --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/logest.c | 7 +++++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 44e14c4938..29bd3a7832 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\slong-standing\stypos\sin\scomments. -D 2014-04-30T13:19:09.070 +C Improved\srendering\sof\sLogEst\svalues\scorresponding\sto\sreal\svalues\snear\s0.0\nin\sthe\stool/logest.c\sutility\sprogram. +D 2014-04-30T14:22:38.579 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1128,7 +1128,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/lemon.c 07aba6270d5a5016ba8107b09e431eea4ecdc123 F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc -F tool/logest.c 388c318c7ac8b52b7c08ca1e2de0f4ca9a8f7e81 +F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 F tool/mkautoconfamal.sh f8d8dbf7d62f409ebed5134998bf5b51d7266383 F tool/mkkeywordhash.c c9e05e4a7bcab8fab9f583d5b321fb72f565ad97 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e2d42f909de85a0586389f2dc0e654f7af2e351a -R cdeda4c4d8ae1ea7276fce5a8779e48d +P b9f91317c34d07769a95dc2f905a6ccabceb64a3 +R 0e5a317f465f102c0002288cd477a07d U drh -Z 6915f97ffb226384426800091a7da39d +Z 88c640e8cc6f42fa637c9ebb251cd8ac diff --git a/manifest.uuid b/manifest.uuid index 42cd607e10..7421cca676 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b9f91317c34d07769a95dc2f905a6ccabceb64a3 \ No newline at end of file +32910c8c595858245bb7ecfe3aa0f90eeae641af \ No newline at end of file diff --git a/tool/logest.c b/tool/logest.c index 1ac337d36c..347fa68a4f 100644 --- a/tool/logest.c +++ b/tool/logest.c @@ -83,7 +83,8 @@ static LogEst logEstFromDouble(double x){ LogEst e; assert( sizeof(x)==8 && sizeof(a)==8 ); if( x<=0.0 ) return -32768; - if( x<1.0 ) return -logEstFromDouble(1/x); + if( x<0.01 ) return -logEstFromDouble(1.0/x); + if( x<1.0 ) return logEstFromDouble(100.0*x) - 66; if( x<1024.0 ) return logEstFromInteger((sqlite3_uint64)(1024.0*x)) - 100; if( x<=2000000000.0 ) return logEstFromInteger((sqlite3_uint64)x); memcpy(&a, &x, 8); @@ -156,8 +157,10 @@ int main(int argc, char **argv){ } } for(i=n-1; i>=0; i--){ - if( a[i]<0 ){ + if( a[i]<-40 ){ printf("%5d (%f)\n", a[i], 1.0/(double)logEstToInt(-a[i])); + }else if( a[i]<10 ){ + printf("%5d (%f)\n", a[i], logEstToInt(a[i]+100)/1024.0); }else{ sqlite3_uint64 x = logEstToInt(a[i]+100)*100/1024; printf("%5d (%lld.%02lld)\n", a[i], x/100, x%100); From 4a6b8a05cd203606f227263f9146aa7dff36c888 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 30 Apr 2014 14:47:01 +0000 Subject: [PATCH 27/71] Fix a couple of out-of-date comments in where.c. FossilOrigin-Name: eefeda32d54efbbdf7d20b719299eda48b891fae --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 47 ++++++++++++++++++++++++++++++----------------- 3 files changed, 38 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index 29bd3a7832..8167b75420 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\srendering\sof\sLogEst\svalues\scorresponding\sto\sreal\svalues\snear\s0.0\nin\sthe\stool/logest.c\sutility\sprogram. -D 2014-04-30T14:22:38.579 +C Fix\sa\scouple\sof\sout-of-date\scomments\sin\swhere.c. +D 2014-04-30T14:47:01.628 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -292,7 +292,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 9651faf05f900d9235bf3166fa54a9e5f12a40f2 +F src/where.c e91b92908c973b2b6b03145dcd131a8ca8b1bf64 F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P b9f91317c34d07769a95dc2f905a6ccabceb64a3 -R 0e5a317f465f102c0002288cd477a07d -U drh -Z 88c640e8cc6f42fa637c9ebb251cd8ac +P 32910c8c595858245bb7ecfe3aa0f90eeae641af +R 26fff8d56fc56eff56bc1e57cba958ab +U dan +Z b9aa63fbb519903c6b0c5b0a6a8e051f diff --git a/manifest.uuid b/manifest.uuid index 7421cca676..6c9c8ac5b7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -32910c8c595858245bb7ecfe3aa0f90eeae641af \ No newline at end of file +eefeda32d54efbbdf7d20b719299eda48b891fae \ No newline at end of file diff --git a/src/where.c b/src/where.c index aea0be3042..ec7a908b1a 100644 --- a/src/where.c +++ b/src/where.c @@ -4022,8 +4022,13 @@ static void whereLoopOutputAdjust(WhereClause *pWC, WhereLoop *pLoop){ } /* -** We have so far matched pBuilder->pNew->u.btree.nEq terms of the index pIndex. -** Try to match one more. +** We have so far matched pBuilder->pNew->u.btree.nEq terms of the +** index pIndex. Try to match one more. +** +** When this function is called, pBuilder->pNew->nOut contains the +** number of rows expected to be visited by filtering using the nEq +** terms only. If it is modified, this value is restored before this +** function returns. ** ** If pProbe->tnum==0, that means pIndex is a fake index used for the ** INTEGER PRIMARY KEY. @@ -4085,10 +4090,14 @@ static int whereLoopAddBtreeIndex( /* Consider using a skip-scan if there are no WHERE clause constraints ** available for the left-most terms of the index, and if the average - ** number of repeats in the left-most terms is at least 18. The magic - ** number 18 was found by experimentation to be the payoff point where - ** skip-scan become faster than a full-scan. - */ + ** number of repeats in the left-most terms is at least 18. + ** + ** The magic number 18 is selected on the basis that scanning 17 rows + ** is almost always quicker than an index seek (even though if the index + ** contains fewer than 2^17 rows we assume otherwise in other parts of + ** the code). And, even if it is not, it should not be too much slower. + ** On the other hand, the extra seeks could end up being significantly + ** more expensive. */ assert( 42==sqlite3LogEst(18) ); if( pTerm==0 && saved_nEq==saved_nSkip @@ -5226,23 +5235,27 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags, iLoop, pWLoop, &revMask); if( isOrdered>=0 && isOrdered0 && 66==sqlite3LogEst(100) ); rScale = sqlite3LogEst((nOrderBy-isOrdered)*100/nOrderBy) - 66; rSortCost = nRowEst + estLog(nRowEst) + rScale + 16; /* TUNING: The cost of implementing DISTINCT using a B-TREE is - ** also N*log(N) but it has a larger constant of proportionality. - ** Multiply by 3.0. */ + ** similar but with a larger constant of proportionality. + ** Multiply by an additional factor of 3.0. */ if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){ rSortCost += 16; } From 75525cbe2423852e551e1843e5fad91acf803024 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 30 Apr 2014 14:53:21 +0000 Subject: [PATCH 28/71] Update a couple of test cases to account for the fact that this branch prefers an index scan and partial sort over a full-table scan and full external sort. FossilOrigin-Name: 9b975bf33cd8fc28c64183a9642bf9fb436a4746 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/cost.test | 5 ++++- test/eqp.test | 12 ++++++------ 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 8167b75420..9f27ec6f9b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scouple\sof\sout-of-date\scomments\sin\swhere.c. -D 2014-04-30T14:47:01.628 +C Update\sa\scouple\sof\stest\scases\sto\saccount\sfor\sthe\sfact\sthat\sthis\sbranch\sprefers\san\sindex\sscan\sand\spartial\ssort\sover\sa\sfull-table\sscan\sand\sfull\sexternal\ssort. +D 2014-04-30T14:53:21.345 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -407,7 +407,7 @@ F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804 F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb F test/corruptI.test b3e4203d420490fc3d3062711597bc1dea06a789 -F test/cost.test 41b350fcc811a4fcccffbab06a3a79f33b8ea0de +F test/cost.test 3f7904d623ef8dc6e55f2206db5ce0549077b438 F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -454,7 +454,7 @@ F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea F test/enc2.test 83437a79ba1545a55fb549309175c683fb334473 F test/enc3.test 90683ad0e6ea587b9d5542ca93568af9a9858c40 F test/enc4.test c8f1ce3618508fd0909945beb8b8831feef2c020 -F test/eqp.test 57c6c604c2807fb5531731c5323133453c24afac +F test/eqp.test 90b56d03a93a2e7bb90f88be6083a8ea53f11a0e F test/errmsg.test f31592a594b44ee121371d25ddd5d63497bb3401 F test/eval.test bc269c365ba877554948441e91ad5373f9f91be3 F test/exclusive.test c7ebbc756eacf544c108b15eed64d7d4e5f86b75 @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 32910c8c595858245bb7ecfe3aa0f90eeae641af -R 26fff8d56fc56eff56bc1e57cba958ab +P eefeda32d54efbbdf7d20b719299eda48b891fae +R 50e654a1c0b26308b8a927bcd26c6421 U dan -Z b9aa63fbb519903c6b0c5b0a6a8e051f +Z f4d62e136a5d6e05753f5050c6fb0c4a diff --git a/manifest.uuid b/manifest.uuid index 6c9c8ac5b7..00e612df5f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eefeda32d54efbbdf7d20b719299eda48b891fae \ No newline at end of file +9b975bf33cd8fc28c64183a9642bf9fb436a4746 \ No newline at end of file diff --git a/test/cost.test b/test/cost.test index e04dc9392a..045a4c08d6 100644 --- a/test/cost.test +++ b/test/cost.test @@ -99,7 +99,10 @@ do_execsql_test 5.1 { do_eqp_test 5.2 { SELECT * FROM t2 ORDER BY x, y; -} {} +} { + 0 0 0 {SCAN TABLE t2 USING INDEX t2i1} + 0 0 0 {USE TEMP B-TREE FOR RIGHT PART OF ORDER BY} +} do_eqp_test 5.3 { SELECT * FROM t2 WHERE x BETWEEN ? AND ? ORDER BY rowid; diff --git a/test/eqp.test b/test/eqp.test index 610ebdc5ca..9e2e9505cc 100644 --- a/test/eqp.test +++ b/test/eqp.test @@ -312,8 +312,8 @@ do_eqp_test 4.2.3 { } { 1 0 0 {SCAN TABLE t1} 1 0 0 {USE TEMP B-TREE FOR ORDER BY} - 2 0 0 {SCAN TABLE t2} - 2 0 0 {USE TEMP B-TREE FOR ORDER BY} + 2 0 0 {SCAN TABLE t2 USING INDEX t2i1} + 2 0 0 {USE TEMP B-TREE FOR RIGHT PART OF ORDER BY} 0 0 0 {COMPOUND SUBQUERIES 1 AND 2 (UNION)} } do_eqp_test 4.2.4 { @@ -321,8 +321,8 @@ do_eqp_test 4.2.4 { } { 1 0 0 {SCAN TABLE t1} 1 0 0 {USE TEMP B-TREE FOR ORDER BY} - 2 0 0 {SCAN TABLE t2} - 2 0 0 {USE TEMP B-TREE FOR ORDER BY} + 2 0 0 {SCAN TABLE t2 USING INDEX t2i1} + 2 0 0 {USE TEMP B-TREE FOR RIGHT PART OF ORDER BY} 0 0 0 {COMPOUND SUBQUERIES 1 AND 2 (INTERSECT)} } do_eqp_test 4.2.5 { @@ -330,8 +330,8 @@ do_eqp_test 4.2.5 { } { 1 0 0 {SCAN TABLE t1} 1 0 0 {USE TEMP B-TREE FOR ORDER BY} - 2 0 0 {SCAN TABLE t2} - 2 0 0 {USE TEMP B-TREE FOR ORDER BY} + 2 0 0 {SCAN TABLE t2 USING INDEX t2i1} + 2 0 0 {USE TEMP B-TREE FOR RIGHT PART OF ORDER BY} 0 0 0 {COMPOUND SUBQUERIES 1 AND 2 (EXCEPT)} } From 8164722c5806410eb5443a2498f619475fee3fff Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 30 Apr 2014 15:00:16 +0000 Subject: [PATCH 29/71] Add text to the header comment of whereLoopAddBtree() describing how the costs of various b-tree loops are estimated. FossilOrigin-Name: 05e6e16cb28c9ffb4596bd2ef81f687c5403ecbb --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 23 +++++++++++++++++++++++ 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 9f27ec6f9b..35f78b46c2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sa\scouple\sof\stest\scases\sto\saccount\sfor\sthe\sfact\sthat\sthis\sbranch\sprefers\san\sindex\sscan\sand\spartial\ssort\sover\sa\sfull-table\sscan\sand\sfull\sexternal\ssort. -D 2014-04-30T14:53:21.345 +C Add\stext\sto\sthe\sheader\scomment\sof\swhereLoopAddBtree()\sdescribing\show\sthe\scosts\sof\svarious\sb-tree\sloops\sare\sestimated. +D 2014-04-30T15:00:16.197 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -292,7 +292,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c e91b92908c973b2b6b03145dcd131a8ca8b1bf64 +F src/where.c 4aeb1caa0a16c76e0c0566af4c64ba003836a0aa F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P eefeda32d54efbbdf7d20b719299eda48b891fae -R 50e654a1c0b26308b8a927bcd26c6421 +P 9b975bf33cd8fc28c64183a9642bf9fb436a4746 +R 20e49e26218874ddfe5fb9438df8f580 U dan -Z f4d62e136a5d6e05753f5050c6fb0c4a +Z c1e32968c738a6da213a45755f91539c diff --git a/manifest.uuid b/manifest.uuid index 00e612df5f..6ee1deefef 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9b975bf33cd8fc28c64183a9642bf9fb436a4746 \ No newline at end of file +05e6e16cb28c9ffb4596bd2ef81f687c5403ecbb \ No newline at end of file diff --git a/src/where.c b/src/where.c index ec7a908b1a..7d01db61ee 100644 --- a/src/where.c +++ b/src/where.c @@ -4349,6 +4349,29 @@ static int whereUsablePartialIndex(int iTab, WhereClause *pWC, Expr *pWhere){ ** Add all WhereLoop objects for a single table of the join where the table ** is idenfied by pBuilder->pNew->iTab. That table is guaranteed to be ** a b-tree table, not a virtual table. +** +** The costs (WhereLoop.rRun) of the b-tree loops added by this function +** are calculated as follows: +** +** For a full scan, assuming the table (or index) contains nRow rows: +** +** cost = nRow * 3.0 // full-table scan +** cost = nRow * K // scan of covering index +** cost = nRow * (K+3.0) // scan of non-covering index +** +** where K is a value between 1.1 and 3.0 set based on the relative +** estimated average size of the index and table records. +** +** For an index scan, where nVisit is the number of index rows visited +** by the scan, and nSeek is the number of seek operations required on +** the index b-tree: +** +** cost = nSeek * (log(nRow) + K * nVisit) // covering index +** cost = nSeek * (log(nRow) + (K+3.0) * nVisit) // non-covering index +** +** Normally, nSeek is 1. nSeek values greater than 1 come about if the +** WHERE clause includes "x IN (....)" terms used in place of "x=?". Or when +** implicit "x IN (SELECT x FROM tbl)" terms are added for skip-scans. */ static int whereLoopAddBtree( WhereLoopBuilder *pBuilder, /* WHERE clause information */ From 5da73e1a09b89f3605af1e438ac5e269969015f5 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 30 Apr 2014 18:11:55 +0000 Subject: [PATCH 30/71] Fix a problem in calculating the costs of "OR" scans. FossilOrigin-Name: 9bbca48b42e4fe16f2188e18dc736da30a96435c --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 17 +++++++++++++++-- test/cost.test | 7 ++++++- 4 files changed, 29 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index c2ff19fbfe..88f6a39dac 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modify\sthe\sway\sthe\scosts\sof\svarious\squery\splans\sare\sestimated.\sIf\sthe\suser\ssupplies\sa\slikelihood()\svalue\s(or\sequivalent)\son\san\sindexed\sWHERE\sconstraint,\suse\sit\sto\sestimate\sthe\snumber\sof\sindex\srows\svisited. -D 2014-04-30T15:22:25.359 +C Fix\sa\sproblem\sin\scalculating\sthe\scosts\sof\s"OR"\sscans. +D 2014-04-30T18:11:55.566 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -292,7 +292,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 4aeb1caa0a16c76e0c0566af4c64ba003836a0aa +F src/where.c 3eaf3d241d86452c0f21aa3fe2b5df25d8c99a24 F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -407,7 +407,7 @@ F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804 F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb F test/corruptI.test b3e4203d420490fc3d3062711597bc1dea06a789 -F test/cost.test 3f7904d623ef8dc6e55f2206db5ce0549077b438 +F test/cost.test 04842adec34311d70c3f3c5fd698b22be54138fd F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P af2cbe64adab5f9e3b0f3da00d06428088589d7f 05e6e16cb28c9ffb4596bd2ef81f687c5403ecbb -R 20e49e26218874ddfe5fb9438df8f580 +P 90e36676476e8db00658772e6c938242f766d306 +R cf0f5a4a25fa17ee63b35d342904cb95 U dan -Z 75da520dd73c7e9f7e5ee96ec52c5c59 +Z 15897e765882c5d27e5f655ffd8d3c94 diff --git a/manifest.uuid b/manifest.uuid index 437f1a2753..274547b53f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -90e36676476e8db00658772e6c938242f766d306 \ No newline at end of file +9bbca48b42e4fe16f2188e18dc736da30a96435c \ No newline at end of file diff --git a/src/where.c b/src/where.c index 7d01db61ee..3b96e5fbea 100644 --- a/src/where.c +++ b/src/where.c @@ -4738,7 +4738,7 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ int iCur; WhereClause tempWC; WhereLoopBuilder sSubBuild; - WhereOrSet sSum, sCur, sPrev; + WhereOrSet sSum, sCur; struct SrcList_item *pItem; pWC = pBuilder->pWC; @@ -4794,6 +4794,7 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ whereOrMove(&sSum, &sCur); once = 0; }else{ + WhereOrSet sPrev; whereOrMove(&sPrev, &sSum); sSum.n = 0; for(i=0; iiSortIdx = 0; memset(&pNew->u, 0, sizeof(pNew->u)); for(i=0; rc==SQLITE_OK && irRun = sSum.a[i].rRun; + /* TUNING: Currently sSum.a[i].rRun is set to the sum of the costs + ** of all sub-scans required by the OR-scan. However, due to rounding + ** errors, it may be that the cost of the OR-scan is equal to its + ** most expensive sub-scan. Add the smallest possible penalty + ** (equivalent to multiplying the cost by 1.07) to ensure that + ** this does not happen. Otherwise, for WHERE clauses such as the + ** following where there is an index on "y": + ** + ** WHERE likelihood(x=?, 0.99) OR y=? + ** + ** the planner may elect to "OR" together a full-table scan and an + ** index lookup. And other similarly odd results. */ + pNew->rRun = sSum.a[i].rRun + 1; pNew->nOut = sSum.a[i].nOut; pNew->prereq = sSum.a[i].prereq; rc = whereLoopInsert(pBuilder, pNew); diff --git a/test/cost.test b/test/cost.test index 045a4c08d6..c413c3cdfa 100644 --- a/test/cost.test +++ b/test/cost.test @@ -150,7 +150,6 @@ do_eqp_test 7.2 { 0 0 0 {USE TEMP B-TREE FOR ORDER BY} } -#set sqlite_where_trace 0xfff do_eqp_test 7.3 { SELECT rowid FROM t1 WHERE (+b IS NULL AND c NOT NULL AND d NOT NULL) @@ -160,6 +159,12 @@ do_eqp_test 7.3 { 0 0 0 {SCAN TABLE t1} } +do_eqp_test 7.4 { + SELECT rowid FROM t1 WHERE (+b IS NULL AND c NOT NULL) OR c IS NULL +} { + 0 0 0 {SCAN TABLE t1} +} + #------------------------------------------------------------------------- # reset_db From d1ae96d39e7fe67db42aebb3613a22d88809daf4 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 1 May 2014 01:13:08 +0000 Subject: [PATCH 31/71] Add the immutable=TRUE query parameter and SQLITE_IOCAP_IMMUTABLE, either of which prevents locking of the database and omits tests for existance of journal files. FossilOrigin-Name: 71f152c85d79ebd9ca48bc9c4c71fc4debaf2437 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/pager.c | 51 +++++++++++++++++++++++++++---------------------- src/sqlite.h.in | 6 +++++- 4 files changed, 41 insertions(+), 32 deletions(-) diff --git a/manifest b/manifest index 25f2d0b735..9473264928 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sin\sall\srecent\schanges\sand\senhancements\sfrom\strunk. -D 2014-04-30T20:32:41.139 +C Add\sthe\simmutable=TRUE\squery\sparameter\sand\sSQLITE_IOCAP_IMMUTABLE,\seither\nof\swhich\sprevents\slocking\sof\sthe\sdatabase\sand\somits\stests\sfor\sexistance\nof\sjournal\sfiles. +D 2014-05-01T01:13:08.624 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -206,7 +206,7 @@ F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c ae4b5240af4619d711301d7992396e182585269f F src/os_win.c 1662c40d22be162698a31725e836eede09e841ed -F src/pager.c ab62a24218d87dda1be641f6c5ad291bff78fd94 +F src/pager.c 5c296b584df0d675e2377196472ff8a993ad079d F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 F src/pcache.c d8eafac28290d4bb80332005435db44991d07fc2 @@ -220,7 +220,7 @@ F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be F src/select.c ed459f7f478a1e533d19c4b953693b3ffa2efd15 F src/shell.c 2afe7a7154e97be0c74c5feacf09626bda8493be -F src/sqlite.h.in bde98816e1ba0c9ffef50afe7b32f4e5a8f54fe0 +F src/sqlite.h.in ad2bbeb6a41c228f6a9dbf24df62e4d3eff79ee5 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc F src/sqliteInt.h b2947801eccefd7ba3e5f14e1353289351a83cf3 @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P ffceacf61905dc3553978ba5f68e133947eca2fa 9bbca48b42e4fe16f2188e18dc736da30a96435c -R de599c68866896adcd34d6a6ff4634e0 +P 84243f844417f888ab731841f7dbf95749f6e034 +R 5c0369593ee1e8996babaee6957b771b U drh -Z adbf9541c1c60ea64244c304cb7bb6d4 +Z 54ea9e774afb5e969c7607c1ec92b0eb diff --git a/manifest.uuid b/manifest.uuid index 2b647b0726..717b11ca0b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -84243f844417f888ab731841f7dbf95749f6e034 \ No newline at end of file +71f152c85d79ebd9ca48bc9c4c71fc4debaf2437 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index b09d6cb930..78f2cae020 100644 --- a/src/pager.c +++ b/src/pager.c @@ -4674,30 +4674,37 @@ int sqlite3PagerOpen( ** + The value returned by sqlite3OsSectorSize() ** + The largest page size that can be written atomically. */ - if( rc==SQLITE_OK && !readOnly ){ - setSectorSize(pPager); - assert(SQLITE_DEFAULT_PAGE_SIZE<=SQLITE_MAX_DEFAULT_PAGE_SIZE); - if( szPageDfltsectorSize ){ - if( pPager->sectorSize>SQLITE_MAX_DEFAULT_PAGE_SIZE ){ - szPageDflt = SQLITE_MAX_DEFAULT_PAGE_SIZE; - }else{ - szPageDflt = (u32)pPager->sectorSize; - } - } -#ifdef SQLITE_ENABLE_ATOMIC_WRITE - { + if( rc==SQLITE_OK ){ + if( !readOnly ){ int iDc = sqlite3OsDeviceCharacteristics(pPager->fd); - int ii; - assert(SQLITE_IOCAP_ATOMIC512==(512>>8)); - assert(SQLITE_IOCAP_ATOMIC64K==(65536>>8)); - assert(SQLITE_MAX_DEFAULT_PAGE_SIZE<=65536); - for(ii=szPageDflt; ii<=SQLITE_MAX_DEFAULT_PAGE_SIZE; ii=ii*2){ - if( iDc&(SQLITE_IOCAP_ATOMIC|(ii>>8)) ){ - szPageDflt = ii; + setSectorSize(pPager); + assert(SQLITE_DEFAULT_PAGE_SIZE<=SQLITE_MAX_DEFAULT_PAGE_SIZE); + if( szPageDfltsectorSize ){ + if( pPager->sectorSize>SQLITE_MAX_DEFAULT_PAGE_SIZE ){ + szPageDflt = SQLITE_MAX_DEFAULT_PAGE_SIZE; + }else{ + szPageDflt = (u32)pPager->sectorSize; + } + } +#ifdef SQLITE_ENABLE_ATOMIC_WRITE + { + int ii; + assert(SQLITE_IOCAP_ATOMIC512==(512>>8)); + assert(SQLITE_IOCAP_ATOMIC64K==(65536>>8)); + assert(SQLITE_MAX_DEFAULT_PAGE_SIZE<=65536); + for(ii=szPageDflt; ii<=SQLITE_MAX_DEFAULT_PAGE_SIZE; ii=ii*2){ + if( iDc&(SQLITE_IOCAP_ATOMIC|(ii>>8)) ){ + szPageDflt = ii; + } } } - } #endif + if( (iDc & SQLITE_IOCAP_IMMUTABLE)!=0 + || sqlite3_uri_boolean(zFilename, "immutable", 0) ){ + vfsFlags |= SQLITE_OPEN_READONLY; + goto act_like_temp_file; + } + } } }else{ /* If a temporary file is requested, it is not opened immediately. @@ -4708,6 +4715,7 @@ int sqlite3PagerOpen( ** database is the same as a temp-file that is never written out to ** disk and uses an in-memory rollback journal. */ +act_like_temp_file: tempFile = 1; pPager->eState = PAGER_READER; pPager->eLock = EXCLUSIVE_LOCK; @@ -4751,9 +4759,6 @@ int sqlite3PagerOpen( /* pPager->nPage = 0; */ pPager->mxPgno = SQLITE_MAX_PAGE_COUNT; /* pPager->state = PAGER_UNLOCK; */ -#if 0 - assert( pPager->state == (tempFile ? PAGER_EXCLUSIVE : PAGER_UNLOCK) ); -#endif /* pPager->errMask = 0; */ pPager->tempFile = (u8)tempFile; assert( tempFile==PAGER_LOCKINGMODE_NORMAL diff --git a/src/sqlite.h.in b/src/sqlite.h.in index e8dddc468c..81220a8822 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -555,7 +555,10 @@ int sqlite3_exec( ** file that were written at the application level might have changed ** and that adjacent bytes, even bytes within the same sector are ** guaranteed to be unchanged. The SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN -** flag indicate that a file cannot be deleted when open. +** flag indicate that a file cannot be deleted when open. The +** SQLITE_IOCAP_IMMUTABLE flag indicates that the file is on +** read-only media and cannot be changed even by processes with +** elevated privileges. */ #define SQLITE_IOCAP_ATOMIC 0x00000001 #define SQLITE_IOCAP_ATOMIC512 0x00000002 @@ -570,6 +573,7 @@ int sqlite3_exec( #define SQLITE_IOCAP_SEQUENTIAL 0x00000400 #define SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN 0x00000800 #define SQLITE_IOCAP_POWERSAFE_OVERWRITE 0x00001000 +#define SQLITE_IOCAP_IMMUTABLE 0x00002000 /* ** CAPI3REF: File Locking Levels From 48cdabcb1af2762fdfb8ede588328def41d1b74a Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 1 May 2014 01:20:05 +0000 Subject: [PATCH 32/71] Avoid unnecessary xUnlock operations on temp file. FossilOrigin-Name: 1829c38c3233c8cb194fae6d560d35a8916b1348 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 9473264928..2f57028bf7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\simmutable=TRUE\squery\sparameter\sand\sSQLITE_IOCAP_IMMUTABLE,\seither\nof\swhich\sprevents\slocking\sof\sthe\sdatabase\sand\somits\stests\sfor\sexistance\nof\sjournal\sfiles. -D 2014-05-01T01:13:08.624 +C Avoid\sunnecessary\sxUnlock\soperations\son\stemp\sfile. +D 2014-05-01T01:20:05.808 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -206,7 +206,7 @@ F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c ae4b5240af4619d711301d7992396e182585269f F src/os_win.c 1662c40d22be162698a31725e836eede09e841ed -F src/pager.c 5c296b584df0d675e2377196472ff8a993ad079d +F src/pager.c 52b7f435cb06063ccb52ff7089dfbc7be613b9b2 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 F src/pcache.c d8eafac28290d4bb80332005435db44991d07fc2 @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 84243f844417f888ab731841f7dbf95749f6e034 -R 5c0369593ee1e8996babaee6957b771b +P 71f152c85d79ebd9ca48bc9c4c71fc4debaf2437 +R 936969710ee66727ffa03c716e1344c2 U drh -Z 54ea9e774afb5e969c7607c1ec92b0eb +Z b6af320858436be94ae3e128e461e236 diff --git a/manifest.uuid b/manifest.uuid index 717b11ca0b..8ca149ad7f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -71f152c85d79ebd9ca48bc9c4c71fc4debaf2437 \ No newline at end of file +1829c38c3233c8cb194fae6d560d35a8916b1348 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 78f2cae020..15d21cd9b1 100644 --- a/src/pager.c +++ b/src/pager.c @@ -1089,7 +1089,7 @@ static int pagerUnlockDb(Pager *pPager, int eLock){ assert( !pPager->exclusiveMode || pPager->eLock==eLock ); assert( eLock==NO_LOCK || eLock==SHARED_LOCK ); assert( eLock!=NO_LOCK || pagerUseWal(pPager)==0 ); - if( isOpen(pPager->fd) ){ + if( isOpen(pPager->fd) && !pPager->tempFile ){ assert( pPager->eLock>=eLock ); rc = sqlite3OsUnlock(pPager->fd, eLock); if( pPager->eLock!=UNKNOWN_LOCK ){ From 57fe136b23aca972e867a3c7925980c1a5e80842 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 1 May 2014 01:49:22 +0000 Subject: [PATCH 33/71] Take out the special handling of nolock=true in os_win.c and add it to pager.c, so that it works for all VFSes. Add the pPager->noLock boolean for clarity. FossilOrigin-Name: 725c1c14be331bce9bf074960dbfa7b6d4c1ba87 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_win.c | 17 +---------------- src/pager.c | 17 +++++++++++------ 4 files changed, 20 insertions(+), 30 deletions(-) diff --git a/manifest b/manifest index 2f57028bf7..6ac75f7987 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sunnecessary\sxUnlock\soperations\son\stemp\sfile. -D 2014-05-01T01:20:05.808 +C Take\sout\sthe\sspecial\shandling\sof\snolock=true\sin\sos_win.c\sand\sadd\sit\sto\npager.c,\sso\sthat\sit\sworks\sfor\sall\sVFSes.\s\sAdd\sthe\spPager->noLock\sboolean\nfor\sclarity. +D 2014-05-01T01:49:22.279 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -205,8 +205,8 @@ F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c ae4b5240af4619d711301d7992396e182585269f -F src/os_win.c 1662c40d22be162698a31725e836eede09e841ed -F src/pager.c 52b7f435cb06063ccb52ff7089dfbc7be613b9b2 +F src/os_win.c e71678ac927d0a0fb11d993db20a9748eabf808e +F src/pager.c 6ed54e169933200b8d113674d5da46a5cbd32c6c F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 F src/pcache.c d8eafac28290d4bb80332005435db44991d07fc2 @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 71f152c85d79ebd9ca48bc9c4c71fc4debaf2437 -R 936969710ee66727ffa03c716e1344c2 +P 1829c38c3233c8cb194fae6d560d35a8916b1348 +R 9585bcff14fbe775451a3a647d896079 U drh -Z b6af320858436be94ae3e128e461e236 +Z 3dae204f6c4a7be4483961b87ae0c17a diff --git a/manifest.uuid b/manifest.uuid index 8ca149ad7f..d015fa2fa6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1829c38c3233c8cb194fae6d560d35a8916b1348 \ No newline at end of file +725c1c14be331bce9bf074960dbfa7b6d4c1ba87 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index bf5b9c72d1..287dad3b57 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -275,7 +275,6 @@ struct winFile { #define WINFILE_RDONLY 0x02 /* Connection is read only */ #define WINFILE_PERSIST_WAL 0x04 /* Persistent WAL mode */ #define WINFILE_PSOW 0x10 /* SQLITE_IOCAP_POWERSAFE_OVERWRITE */ -#define WINFILE_NOLOCK 0x20 /* Never do any real locking */ /* * The size of the buffer used by sqlite3_win32_write_debug(). @@ -2860,10 +2859,6 @@ static int winLock(sqlite3_file *id, int locktype){ assert( id!=0 ); OSTRACE(("LOCK file=%p, oldLock=%d(%d), newLock=%d\n", pFile->h, pFile->locktype, pFile->sharedLockByte, locktype)); - if( pFile->ctrlFlags & WINFILE_NOLOCK ){ - OSTRACE(("LOCK-NOP file=%p, rc=SQLITE_OK\n", pFile->h)); - return SQLITE_OK; - } /* If there is already a lock of this type or more restrictive on the ** OsFile, do nothing. Don't use the end_lock: exit path, as @@ -2991,10 +2986,7 @@ static int winCheckReservedLock(sqlite3_file *id, int *pResOut){ OSTRACE(("TEST-WR-LOCK file=%p, pResOut=%p\n", pFile->h, pResOut)); assert( id!=0 ); - if( pFile->ctrlFlags & WINFILE_NOLOCK ){ - rc = 0; - OSTRACE(("TEST-WR-LOCK file=%p, rc=%d (nop)\n", pFile->h, rc)); - }else if( pFile->locktype>=RESERVED_LOCK ){ + if( pFile->locktype>=RESERVED_LOCK ){ rc = 1; OSTRACE(("TEST-WR-LOCK file=%p, rc=%d (local)\n", pFile->h, rc)); }else{ @@ -3030,10 +3022,6 @@ static int winUnlock(sqlite3_file *id, int locktype){ assert( locktype<=SHARED_LOCK ); OSTRACE(("UNLOCK file=%p, oldLock=%d(%d), newLock=%d\n", pFile->h, pFile->locktype, pFile->sharedLockByte, locktype)); - if( pFile->ctrlFlags & WINFILE_NOLOCK ){ - OSTRACE(("UNLOCK-NOP file=%p, rc=SQLITE_OK\n", pFile->h)); - return SQLITE_OK; - } type = pFile->locktype; if( type>=EXCLUSIVE_LOCK ){ winUnlockFile(&pFile->h, SHARED_FIRST, 0, SHARED_SIZE, 0); @@ -4704,9 +4692,6 @@ static int winOpen( if( sqlite3_uri_boolean(zName, "psow", SQLITE_POWERSAFE_OVERWRITE) ){ pFile->ctrlFlags |= WINFILE_PSOW; } - if( sqlite3_uri_boolean(zName, "nolock", 0) ){ - pFile->ctrlFlags |= WINFILE_NOLOCK; - } pFile->lastErrno = NO_ERROR; pFile->zPath = zName; #if SQLITE_MAX_MMAP_SIZE>0 diff --git a/src/pager.c b/src/pager.c index 15d21cd9b1..9ab7cfbe54 100644 --- a/src/pager.c +++ b/src/pager.c @@ -626,7 +626,8 @@ struct Pager { u8 ckptSyncFlags; /* SYNC_NORMAL or SYNC_FULL for checkpoint */ u8 walSyncFlags; /* SYNC_NORMAL or SYNC_FULL for wal writes */ u8 syncFlags; /* SYNC_NORMAL or SYNC_FULL otherwise */ - u8 tempFile; /* zFilename is a temporary file */ + u8 tempFile; /* zFilename is a temporary or immutable file */ + u8 noLock; /* Do not lock (except in WAL mode) */ u8 readOnly; /* True for a read-only database */ u8 memDb; /* True to inhibit all file I/O */ @@ -1089,9 +1090,9 @@ static int pagerUnlockDb(Pager *pPager, int eLock){ assert( !pPager->exclusiveMode || pPager->eLock==eLock ); assert( eLock==NO_LOCK || eLock==SHARED_LOCK ); assert( eLock!=NO_LOCK || pagerUseWal(pPager)==0 ); - if( isOpen(pPager->fd) && !pPager->tempFile ){ + if( isOpen(pPager->fd) ){ assert( pPager->eLock>=eLock ); - rc = sqlite3OsUnlock(pPager->fd, eLock); + rc = pPager->noLock ? SQLITE_OK : sqlite3OsUnlock(pPager->fd, eLock); if( pPager->eLock!=UNKNOWN_LOCK ){ pPager->eLock = (u8)eLock; } @@ -1115,7 +1116,7 @@ static int pagerLockDb(Pager *pPager, int eLock){ assert( eLock==SHARED_LOCK || eLock==RESERVED_LOCK || eLock==EXCLUSIVE_LOCK ); if( pPager->eLockeLock==UNKNOWN_LOCK ){ - rc = sqlite3OsLock(pPager->fd, eLock); + rc = pPager->noLock ? SQLITE_OK : sqlite3OsLock(pPager->fd, eLock); if( rc==SQLITE_OK && (pPager->eLock!=UNKNOWN_LOCK||eLock==EXCLUSIVE_LOCK) ){ pPager->eLock = (u8)eLock; IOTRACE(("LOCK %p %d\n", pPager, eLock)) @@ -4699,6 +4700,7 @@ int sqlite3PagerOpen( } } #endif + pPager->noLock = sqlite3_uri_boolean(zFilename, "nolock", 0); if( (iDc & SQLITE_IOCAP_IMMUTABLE)!=0 || sqlite3_uri_boolean(zFilename, "immutable", 0) ){ vfsFlags |= SQLITE_OPEN_READONLY; @@ -4714,11 +4716,14 @@ int sqlite3PagerOpen( ** This branch is also run for an in-memory database. An in-memory ** database is the same as a temp-file that is never written out to ** disk and uses an in-memory rollback journal. + ** + ** This branch also runs for files marked as immutable. */ act_like_temp_file: tempFile = 1; - pPager->eState = PAGER_READER; - pPager->eLock = EXCLUSIVE_LOCK; + pPager->eState = PAGER_READER; /* Pretend we already have a lock */ + pPager->eLock = EXCLUSIVE_LOCK; /* Pretend we are in EXCLUSIVE locking mode */ + pPager->noLock = 1; /* Do no locking */ readOnly = (vfsFlags&SQLITE_OPEN_READONLY); } From e03d76254c850555579aa203af78d3e75671383b Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 1 May 2014 10:19:16 +0000 Subject: [PATCH 34/71] Update a test case in wal2.test that explicitly corrupts a checksum in the wal file to account for the fact that the first byte of said checksum may initially be 0xFF. FossilOrigin-Name: 2b935bdea1452505f36dc8c7aad49e6c42f4eceb --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/wal2.test | 8 +++++++- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 88f6a39dac..c24d03cdc7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sin\scalculating\sthe\scosts\sof\s"OR"\sscans. -D 2014-04-30T18:11:55.566 +C Update\sa\stest\scase\sin\swal2.test\sthat\sexplicitly\scorrupts\sa\schecksum\sin\sthe\swal\sfile\sto\saccount\sfor\sthe\sfact\sthat\sthe\sfirst\sbyte\sof\ssaid\schecksum\smay\sinitially\sbe\s0xFF. +D 2014-05-01T10:19:16.340 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1057,7 +1057,7 @@ F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test ea8778d5b0df200adef2ca7c00c3c37d4375f772 F test/wal.test 885f32b2b390b30b4aa3dbb0e568f8f78d40f5cc -F test/wal2.test a8e3963abf6b232cf0b852b09b53665ef34007af +F test/wal2.test 1f841d2048080d32f552942e333fd99ce541dada F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c F test/wal5.test 8f888b50f66b78821e61ed0e233ded5de378224b @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 90e36676476e8db00658772e6c938242f766d306 -R cf0f5a4a25fa17ee63b35d342904cb95 +P 9bbca48b42e4fe16f2188e18dc736da30a96435c +R 82092de2428e80a3b03a937e23116b0e U dan -Z 15897e765882c5d27e5f655ffd8d3c94 +Z f6ffad5e4f0572fc8e2909e59c29de84 diff --git a/manifest.uuid b/manifest.uuid index 274547b53f..98d646217d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9bbca48b42e4fe16f2188e18dc736da30a96435c \ No newline at end of file +2b935bdea1452505f36dc8c7aad49e6c42f4eceb \ No newline at end of file diff --git a/test/wal2.test b/test/wal2.test index b331d5ed10..9d45444d6a 100644 --- a/test/wal2.test +++ b/test/wal2.test @@ -811,7 +811,13 @@ do_test wal2-7.1.1 { do_test wal2-7.1.2 { forcecopy test.db test2.db forcecopy test.db-wal test2.db-wal - hexio_write test2.db-wal 48 FF + # The first 32 bytes of the WAL file contain the WAL header. Offset 48 + # is the first byte of the checksum for the first frame in the WAL. + # The following three lines replaces the contents of that byte with + # a different value. + set newval FF + if {$newval == [hexio_read test2.db-wal 48 1]} { set newval 00 } + hexio_write test2.db-wal 48 $newval } {1} do_test wal2-7.1.3 { sqlite3 db2 test2.db From c5f246eb3747ce3ab28aff821774493560e84611 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 1 May 2014 20:24:21 +0000 Subject: [PATCH 35/71] Add #ifdefs for test coverage. Add a testcase(). FossilOrigin-Name: be2702ce35c713b33c9b7689643b45fb0de6af2a --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/analyze.c | 8 +++++++- src/where.c | 1 + 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index c24d03cdc7..a856daec13 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sa\stest\scase\sin\swal2.test\sthat\sexplicitly\scorrupts\sa\schecksum\sin\sthe\swal\sfile\sto\saccount\sfor\sthe\sfact\sthat\sthe\sfirst\sbyte\sof\ssaid\schecksum\smay\sinitially\sbe\s0xFF. -D 2014-05-01T10:19:16.340 +C Add\s#ifdefs\sfor\stest\scoverage.\s\sAdd\sa\stestcase(). +D 2014-05-01T20:24:21.190 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -159,7 +159,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c b00900877f766f116f9e16116f1ccacdc21d82f1 -F src/analyze.c 92f1495304dd33b4f9e0b0e5aa030b068ada504d +F src/analyze.c 3596f863bb80126fe56ba217df5932749271efc8 F src/attach.c 3801129015ef59d76bf23c95ef9b0069d18a0c52 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 @@ -292,7 +292,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 3eaf3d241d86452c0f21aa3fe2b5df25d8c99a24 +F src/where.c 9e67d6c48006445d964bc835ea82500c190c3a6f F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 9bbca48b42e4fe16f2188e18dc736da30a96435c -R 82092de2428e80a3b03a937e23116b0e -U dan -Z f6ffad5e4f0572fc8e2909e59c29de84 +P 2b935bdea1452505f36dc8c7aad49e6c42f4eceb +R a1dff26029b41608141f15cdd82c4ee9 +U drh +Z 39b43641ac7d3a2ec2ad0946638e023c diff --git a/manifest.uuid b/manifest.uuid index 98d646217d..c97cdf989b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2b935bdea1452505f36dc8c7aad49e6c42f4eceb \ No newline at end of file +be2702ce35c713b33c9b7689643b45fb0de6af2a \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 4fbaaa5173..2952b364c3 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1390,9 +1390,15 @@ static void decodeIntArray( v = v*10 + c - '0'; z++; } +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 if( aOut ){ aOut[i] = v; - }else{ + }else +#else + assert( aOut==0 ); + UNUSED_PARAMETER(aOut); +#endif + { aLog[i] = sqlite3LogEst(v); } if( *z==' ' ) z++; diff --git a/src/where.c b/src/where.c index 3b96e5fbea..238d1ef42a 100644 --- a/src/where.c +++ b/src/where.c @@ -4200,6 +4200,7 @@ static int whereLoopAddBtreeIndex( assert( pNew->nOut==saved_nOut ); if( pTerm->truthProb<=0 && iCol>=0 ){ assert( (eOp & WO_IN) || nIn==0 ); + testcase( eOp & WO_IN ); pNew->nOut += pTerm->truthProb; pNew->nOut -= nIn; pNew->wsFlags |= WHERE_LIKELIHOOD; From 38524132466e77081fc2262122d86d655eb36be3 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 1 May 2014 20:26:48 +0000 Subject: [PATCH 36/71] Fix an obscure problem to do with temp register allocation that could occur if more than one simple SELECT within a compound SELECT uses a partial sort. FossilOrigin-Name: 427409ae106cdab7892a6b50fe30c5f52de5addc --- manifest | 18 ++++++------- manifest.uuid | 2 +- src/select.c | 12 ++++----- src/vdbe.c | 1 + test/selectA.test | 65 +++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 81 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index a856daec13..c7b6bc3a26 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\s#ifdefs\sfor\stest\scoverage.\s\sAdd\sa\stestcase(). -D 2014-05-01T20:24:21.190 +C Fix\san\sobscure\sproblem\sto\sdo\swith\stemp\sregister\sallocation\sthat\scould\soccur\sif\smore\sthan\sone\ssimple\sSELECT\swithin\sa\scompound\sSELECT\suses\sa\spartial\ssort. +D 2014-05-01T20:26:48.793 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -218,7 +218,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be -F src/select.c ed459f7f478a1e533d19c4b953693b3ffa2efd15 +F src/select.c 089c4d46f067a5cccae93524c6377f981ba99bd9 F src/shell.c 2afe7a7154e97be0c74c5feacf09626bda8493be F src/sqlite.h.in bde98816e1ba0c9ffef50afe7b32f4e5a8f54fe0 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -279,7 +279,7 @@ F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c F src/util.c 2b5fb283a190aacdb286f7835a447c45b345b83c F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 699693bea6710ed436392c928b02cb4e91944137 +F src/vdbe.c 7f359193bf2366cc914a9ece093ebf284e56acdc F src/vdbe.h 394464909ed682334aa3d5831aae0c2fe2abef94 F src/vdbeInt.h e6d83e5bfd62fc6685ba1ed6153f7099f82de9f7 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 @@ -789,7 +789,7 @@ F test/select6.test e76bd10a56988f15726c097a5d5a7966fe82d3b2 F test/select7.test 7fd2ef598cfabb6b9ff6ac13973b91d0527df49d F test/select8.test 391de11bdd52339c30580dabbbbe97e3e9a3c79d F test/select9.test aebc2bb0c3bc44606125033cbcaac2c8d1f33a95 -F test/selectA.test 77adaffe9704cb80e301ebaeff4b107b58d435c5 +F test/selectA.test 64b88a80271c1710966e50e633380696b60a12a4 F test/selectB.test 954e4e49cf1f896d61794e440669e03a27ceea25 F test/selectC.test 871fb55d884d3de5943c4057ebd22c2459e71977 F test/selectD.test b0f02a04ef7737decb24e08be2c39b9664b43394 @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 2b935bdea1452505f36dc8c7aad49e6c42f4eceb -R a1dff26029b41608141f15cdd82c4ee9 -U drh -Z 39b43641ac7d3a2ec2ad0946638e023c +P be2702ce35c713b33c9b7689643b45fb0de6af2a +R 31df819a20c9dcbfb809510504bd5463 +U dan +Z aa2318c066a07655ea4b90fb2a4008ea diff --git a/manifest.uuid b/manifest.uuid index c97cdf989b..76ebe330ab 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -be2702ce35c713b33c9b7689643b45fb0de6af2a \ No newline at end of file +427409ae106cdab7892a6b50fe30c5f52de5addc \ No newline at end of file diff --git a/src/select.c b/src/select.c index dfca6d3f83..5fff010f34 100644 --- a/src/select.c +++ b/src/select.c @@ -466,15 +466,17 @@ static void pushOntoSorter( ){ Vdbe *v = pParse->pVdbe; int nExpr = pSort->pOrderBy->nExpr; - int regBase = sqlite3GetTempRange(pParse, nExpr+2); - int regRecord = sqlite3GetTempReg(pParse); + int regRecord = ++pParse->nMem; + int regBase = pParse->nMem+1; int nOBSat = pSort->nOBSat; int op; + + pParse->nMem += nExpr+2; /* nExpr+2 registers allocated at regBase */ sqlite3ExprCacheClear(pParse); sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, 0); sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+1, 1); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nExpr+2-nOBSat, regRecord); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nExpr+2-nOBSat,regRecord); if( nOBSat>0 ){ int regPrevKey; /* The first nOBSat columns of the previous row */ int addrFirst; /* Address of the OP_IfNot opcode */ @@ -511,10 +513,6 @@ static void pushOntoSorter( op = OP_IdxInsert; } sqlite3VdbeAddOp2(v, op, pSort->iECursor, regRecord); - if( nOBSat==0 ){ - sqlite3ReleaseTempReg(pParse, regRecord); - sqlite3ReleaseTempRange(pParse, regBase, nExpr+2); - } if( pSelect->iLimit ){ int addr1, addr2; int iLimit; diff --git a/src/vdbe.c b/src/vdbe.c index 4ef6d0738c..a64e5bf5e1 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -4232,6 +4232,7 @@ case OP_SorterData: { pC = p->apCsr[pOp->p1]; assert( isSorter(pC) ); rc = sqlite3VdbeSorterRowkey(pC, pOut); + assert( rc!=SQLITE_OK || (pOut->flags & MEM_Blob) ); break; } diff --git a/test/selectA.test b/test/selectA.test index ca2ec38da4..6e593e8e22 100644 --- a/test/selectA.test +++ b/test/selectA.test @@ -21,6 +21,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +set testprefix selectA ifcapable !compound { finish_test @@ -1310,4 +1311,68 @@ do_execsql_test selectA-3.98 { SELECT n FROM xyz ORDER BY +n; } {MAD MAD+ MAD++} +#------------------------------------------------------------------------- +# At one point the following code exposed a temp register reuse problem. +# +proc f {args} { return 1 } +db func f f + +do_execsql_test 4.1.1 { + CREATE TABLE t4(a, b); + CREATE TABLE t5(c, d); + + INSERT INTO t5 VALUES(1, 'x'); + INSERT INTO t5 VALUES(2, 'x'); + INSERT INTO t4 VALUES(3, 'x'); + INSERT INTO t4 VALUES(4, 'x'); + + CREATE INDEX i1 ON t4(a); + CREATE INDEX i2 ON t5(c); +} + +do_eqp_test 4.1.2 { + SELECT c, d FROM t5 + UNION ALL + SELECT a, b FROM t4 WHERE f()==f() + ORDER BY 1,2 +} { + 1 0 0 {SCAN TABLE t5 USING INDEX i2} + 1 0 0 {USE TEMP B-TREE FOR RIGHT PART OF ORDER BY} + 2 0 0 {SCAN TABLE t4 USING INDEX i1} + 2 0 0 {USE TEMP B-TREE FOR RIGHT PART OF ORDER BY} + 0 0 0 {COMPOUND SUBQUERIES 1 AND 2 (UNION ALL)} +} + +do_execsql_test 4.1.3 { + SELECT c, d FROM t5 + UNION ALL + SELECT a, b FROM t4 WHERE f()==f() + ORDER BY 1,2 +} { + 1 x 2 x 3 x 4 x +} + +do_execsql_test 4.2.1 { + CREATE TABLE t6(a, b); + CREATE TABLE t7(c, d); + + INSERT INTO t7 VALUES(2, 9); + INSERT INTO t6 VALUES(3, 0); + INSERT INTO t6 VALUES(4, 1); + INSERT INTO t7 VALUES(5, 6); + INSERT INTO t6 VALUES(6, 0); + INSERT INTO t7 VALUES(7, 6); + + CREATE INDEX i6 ON t6(a); + CREATE INDEX i7 ON t7(c); +} + +do_execsql_test 4.2.2 { + SELECT c, f(d,c,d,c,d) FROM t7 + UNION ALL + SELECT a, b FROM t6 + ORDER BY 1,2 +} {/2 . 3 . 4 . 5 . 6 . 7 ./} + + finish_test From e724d3d26a6db490377d85b6015d6f85768b26dc Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 2 May 2014 00:09:40 +0000 Subject: [PATCH 37/71] Add a comment explaining why WhereLoop cost adjustments are omitted for skip-scan loops. FossilOrigin-Name: 3bc43594aaeee9225c0590677fcce480bedcb37b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 11 +++++++++++ 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index c7b6bc3a26..af2b3b36ec 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sobscure\sproblem\sto\sdo\swith\stemp\sregister\sallocation\sthat\scould\soccur\sif\smore\sthan\sone\ssimple\sSELECT\swithin\sa\scompound\sSELECT\suses\sa\spartial\ssort. -D 2014-05-01T20:26:48.793 +C Add\sa\scomment\sexplaining\swhy\sWhereLoop\scost\sadjustments\sare\somitted\sfor\nskip-scan\sloops. +D 2014-05-02T00:09:40.134 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -292,7 +292,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 9e67d6c48006445d964bc835ea82500c190c3a6f +F src/where.c 91dfd382273c3f67fcdc0ac4728f9140f91c6ab3 F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P be2702ce35c713b33c9b7689643b45fb0de6af2a -R 31df819a20c9dcbfb809510504bd5463 -U dan -Z aa2318c066a07655ea4b90fb2a4008ea +P 427409ae106cdab7892a6b50fe30c5f52de5addc +R ee65c8e54ebb299c9693aee246a7e200 +U drh +Z 5a37e5aacc1e047352493d523b6fb849 diff --git a/manifest.uuid b/manifest.uuid index 76ebe330ab..6c6f1f805d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -427409ae106cdab7892a6b50fe30c5f52de5addc \ No newline at end of file +3bc43594aaeee9225c0590677fcce480bedcb37b \ No newline at end of file diff --git a/src/where.c b/src/where.c index 238d1ef42a..c5892590bc 100644 --- a/src/where.c +++ b/src/where.c @@ -3783,6 +3783,17 @@ static int whereLoopCheaperProperSubset( ** To say "WhereLoop X is a proper subset of Y" means that X uses fewer ** WHERE clause terms than Y and that every WHERE clause term used by X is ** also used by Y. +** +** This adjustment is omitted for SKIPSCAN loops. In a SKIPSCAN loop, the +** WhereLoop.nLTerm field is not an accurate measure of the number of WHERE +** clause terms covered, since some of the first nLTerm entries in aLTerm[] +** will be NULL (because they are skipped). That makes it more difficult +** to compare the loops. We could add extra code to do the comparison, and +** perhaps we will someday. But SKIPSCAN is sufficiently uncommon, and this +** adjustment is sufficient minor, that it is very difficult to construct +** a test case where the extra code would improve the query plan. Better +** to avoid the added complexity and just omit cost adjustments to SKIPSCAN +** loops. */ static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ if( (pTemplate->wsFlags & WHERE_INDEXED)==0 ) return; From a153643b0731de848cb07cbe697ebef0e1ae4e61 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 2 May 2014 14:54:56 +0000 Subject: [PATCH 38/71] Fix a broken test case in fuzz.test. FossilOrigin-Name: faa469355eabb2c407f24638b090725448aac37f --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/fuzz.test | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index af2b3b36ec..c958eec031 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\scomment\sexplaining\swhy\sWhereLoop\scost\sadjustments\sare\somitted\sfor\nskip-scan\sloops. -D 2014-05-02T00:09:40.134 +C Fix\sa\sbroken\stest\scase\sin\sfuzz.test. +D 2014-05-02T14:54:56.266 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -582,7 +582,7 @@ F test/func3.test dbccee9133cfef1473c59ec07b5f0262b9d72f9a F test/func4.test 6beacdfcb0e18c358e6c2dcacf1b65d1fa80955f F test/func5.test cdd224400bc3e48d891827cc913a57051a426fa4 F test/fuzz-oss1.test 4912e528ec9cf2f42134456933659d371c9e0d74 -F test/fuzz.test 77fd50afc12847af50fcf1941679d90adebadde6 +F test/fuzz.test 96083052bf5765e4518c1ba686ce2bab785670d1 F test/fuzz2.test 207d0f9d06db3eaf47a6b7bfc835b8e2fc397167 F test/fuzz3.test efd384b896c647b61a2c1848ba70d42aad60a7b3 F test/fuzz_common.tcl a87dfbb88c2a6b08a38e9a070dabd129e617b45b @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 427409ae106cdab7892a6b50fe30c5f52de5addc -R ee65c8e54ebb299c9693aee246a7e200 -U drh -Z 5a37e5aacc1e047352493d523b6fb849 +P 3bc43594aaeee9225c0590677fcce480bedcb37b +R cb5319eb84ea1cdb4a55d1e33cf3d763 +U dan +Z 412ddec016208716851032f358e0939e diff --git a/manifest.uuid b/manifest.uuid index 6c6f1f805d..52fb89ae57 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3bc43594aaeee9225c0590677fcce480bedcb37b \ No newline at end of file +faa469355eabb2c407f24638b090725448aac37f \ No newline at end of file diff --git a/test/fuzz.test b/test/fuzz.test index e1b22ae804..0deed3b636 100644 --- a/test/fuzz.test +++ b/test/fuzz.test @@ -285,7 +285,7 @@ do_test fuzz-1.18 { ) )) } -} {0 -4294967298} +} {0 {{}}} # At one point the following INSERT statement caused an assert() to fail. # From fdf9f045b7c33281d5a027f2427e9431c31813c0 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 5 May 2014 17:43:28 +0000 Subject: [PATCH 39/71] Fix several header file comments. No changes to code. FossilOrigin-Name: dd6eb5d1eb4d50023a008fac116497ede29600c1 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_setup.h | 2 +- src/os_win.h | 18 +++++++++--------- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index 0a0e83ebbd..bbea0afd3a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\supdates\sfrom\strunk. -D 2014-05-02T21:38:02.443 +C Fix\sseveral\sheader\sfile\scomments.\s\sNo\schanges\sto\scode. +D 2014-05-05T17:43:28.823 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -204,10 +204,10 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 60d419395e32a8029fa380a80a3da2e9030f635e F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_setup.h 3f9e87d981eef070c8a11e76b4704842a0e5870f +F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c ae4b5240af4619d711301d7992396e182585269f F src/os_win.c 089fcb45dabe90a4c87369cc69846838198dd221 -F src/os_win.h ac253253d080dc3dd261efe5e98626c4b95e5a26 +F src/os_win.h ca9c83efaf41298691553e1d0f7e11328168d8b8 F src/pager.c ab62a24218d87dda1be641f6c5ad291bff78fd94 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 @@ -1168,7 +1168,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 09cf240a7fb4022aeb1d7ae2a2455b5f97b53d68 faa469355eabb2c407f24638b090725448aac37f -R 332509ae33dff41f389350276535e395 +P c4d1d8a0db48f523d1624f2468261c171152c0f7 +R e81e8bbcb0f41d36dce23eda3c309a85 U mistachkin -Z 6c2182b94bf292df6ecab4543f02e83a +Z 2824130e829ad3dc4daa36d25ec17b58 diff --git a/manifest.uuid b/manifest.uuid index b20cd682c5..fd4cdfc69f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c4d1d8a0db48f523d1624f2468261c171152c0f7 \ No newline at end of file +dd6eb5d1eb4d50023a008fac116497ede29600c1 \ No newline at end of file diff --git a/src/os_setup.h b/src/os_setup.h index 5b010b894e..68de1446ed 100644 --- a/src/os_setup.h +++ b/src/os_setup.h @@ -22,7 +22,7 @@ ** ** After the following block of preprocess macros, all of SQLITE_OS_UNIX, ** SQLITE_OS_WIN, and SQLITE_OS_OTHER will defined to either 1 or 0. One of -** the four will be 1. The other three will be 0. +** the three will be 1. The other two will be 0. */ #if defined(SQLITE_OS_OTHER) # if SQLITE_OS_OTHER==1 diff --git a/src/os_win.h b/src/os_win.h index 6ccd9f4828..cb1ff80bd9 100644 --- a/src/os_win.h +++ b/src/os_win.h @@ -16,7 +16,7 @@ #define _OS_WIN_H_ /* -** Include the Windows SDK header file. +** Include the primary Windows SDK header file. */ #include "windows.h" @@ -28,8 +28,8 @@ /* ** Determine if we are dealing with Windows NT. ** -** We ought to be able to determine if we are compiling for win98 or winNT -** using the _WIN32_WINNT macro as follows: +** We ought to be able to determine if we are compiling for Windows 9x or +** Windows NT using the _WIN32_WINNT macro as follows: ** ** #if defined(_WIN32_WINNT) ** # define SQLITE_OS_WINNT 1 @@ -37,18 +37,18 @@ ** # define SQLITE_OS_WINNT 0 ** #endif ** -** However, vs2005 does not set _WIN32_WINNT by default, as it ought to, -** so the above test does not work. We'll just assume that everything is -** winNT unless the programmer explicitly says otherwise by setting -** SQLITE_OS_WINNT to 0. +** However, Visual Studio 2005 does not set _WIN32_WINNT by default, as +** it ought to, so the above test does not work. We'll just assume that +** everything is Windows NT unless the programmer explicitly says otherwise +** by setting SQLITE_OS_WINNT to 0. */ #if SQLITE_OS_WIN && !defined(SQLITE_OS_WINNT) # define SQLITE_OS_WINNT 1 #endif /* -** Determine if we are dealing with WindowsCE - which has a much -** reduced API. +** Determine if we are dealing with Windows CE - which has a much reduced +** API. */ #if defined(_WIN32_WCE) # define SQLITE_OS_WINCE 1 From 1a8a0d3f1a4b35dbe9bde09026c6358cc9b2619c Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 5 May 2014 20:21:52 +0000 Subject: [PATCH 40/71] Attempt to fix the build on beagleboard. FossilOrigin-Name: 3ac75725a66730566105ad184a988a8f80f51055 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/os_win.h | 2 +- tool/mksqlite3c.tcl | 4 +++- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 2785d56cb3..a49aee7bc9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Centralize\sand\sreform\suse\sof\sthe\sWin32\srelated\sinclude\sfiles. -D 2014-05-05T18:24:47.522 +C Attempt\sto\sfix\sthe\sbuild\son\sbeagleboard. +D 2014-05-05T20:21:52.937 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -207,7 +207,7 @@ F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c ae4b5240af4619d711301d7992396e182585269f F src/os_win.c 485d06a93965f306c7281fca0937829292367234 -F src/os_win.h ca9c83efaf41298691553e1d0f7e11328168d8b8 +F src/os_win.h 057344a6720b4c8405d9bd98f58cb37a6ee46c25 F src/pager.c ab62a24218d87dda1be641f6c5ad291bff78fd94 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 @@ -1137,7 +1137,7 @@ F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e F tool/mkpragmatab.tcl 78a77b2c554d534c6f2dc903130186ed15715460 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl 1712d3d71256ca1f297046619c89e77a4d7c8f6d -F tool/mksqlite3c.tcl 1be89affd8ad71a31b6e5d49cc7bf790db7686ff +F tool/mksqlite3c.tcl ba274df71f5e6534b0a913c7c48eabfcbd0934b6 F tool/mksqlite3h.tcl ba24038056f51fde07c0079c41885ab85e2cff12 F tool/mksqlite3internalh.tcl b6514145a7d5321b47e64e19b8116cc44f973eb1 F tool/mkvsix.tcl 6477fb9dab838b7eea1eed50658ff1cda04850b5 @@ -1168,7 +1168,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 4760504396bcf44528eb8de9ed7aa5d9659c54b6 a3cfd81339267c87db511cedf35ec209ee1f1a72 -R 938ab94058764b9d56f0ca54851e9d8f -U mistachkin -Z 010c73dcd5fc4fed61a248b3b687b42d +P 857f2baf9b722e4eb1ac0d50fe03e32d5176185b +R 13f47e22fa817432bd25922bb1ff1b2f +U drh +Z 55108ca91e730f32aa309dbf16d83499 diff --git a/manifest.uuid b/manifest.uuid index 691d0d795d..f90aaab944 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -857f2baf9b722e4eb1ac0d50fe03e32d5176185b \ No newline at end of file +3ac75725a66730566105ad184a988a8f80f51055 \ No newline at end of file diff --git a/src/os_win.h b/src/os_win.h index cb1ff80bd9..d662cd467d 100644 --- a/src/os_win.h +++ b/src/os_win.h @@ -22,7 +22,7 @@ #ifdef __CYGWIN__ # include -# include /* amalgamator: keep */ +# include /* amalgamator: dontcache */ #endif /* diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl index 9b222bdd22..64207727be 100644 --- a/tool/mksqlite3c.tcl +++ b/tool/mksqlite3c.tcl @@ -170,7 +170,9 @@ proc copy_file {filename} { if {$linemacros} {puts $out "#line [expr {$ln+1}] \"$filename\""} } } elseif {![info exists seen_hdr($hdr)]} { - set seen_hdr($hdr) 1 + if {![regexp {/\*\s+amalgamator:\s+dontcache\s+\*/} $line]} { + set seen_hdr($hdr) 1 + } puts $out $line } elseif {[regexp {/\*\s+amalgamator:\s+keep\s+\*/} $line]} { # This include file must be kept because there was a "keep" From f6a2342651595b22fffb3076c718de4250456afb Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 5 May 2014 20:24:34 +0000 Subject: [PATCH 41/71] Work on cleaning up the MSVC batch build tool integration with Visual Studio 2013. FossilOrigin-Name: b5a72dfd59fba6a9938ac1afdec06a2e6809fd2d --- Makefile.msc | 16 ++++++++++------ manifest | 14 +++++++------- manifest.uuid | 2 +- tool/build-all-msvc.bat | 9 ++++++--- 4 files changed, 24 insertions(+), 17 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index 5d6460b7d5..112d4b6e5f 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -510,16 +510,20 @@ LTLIBOPTS = /MACHINE:$(PLATFORM) !IF $(FOR_WINRT)!=0 LTLINKOPTS = $(LTLINKOPTS) /APPCONTAINER !IF "$(VISUALSTUDIOVERSION)"=="12.0" +!IFNDEF STORELIBPATH !IF "$(PLATFORM)"=="x86" -LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(VCINSTALLDIR)\lib\store" +STORELIBPATH = $(NCRTLIBPATH)\store !ELSEIF "$(PLATFORM)"=="x64" -LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(VCINSTALLDIR)\lib\store\amd64" +STORELIBPATH = $(NCRTLIBPATH)\store\amd64 !ELSEIF "$(PLATFORM)"=="ARM" -LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(VCINSTALLDIR)\lib\store\arm" +STORELIBPATH = $(NCRTLIBPATH)\store\arm !ELSE -LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(VCINSTALLDIR)\lib\store" +STORELIBPATH = $(NCRTLIBPATH)\store !ENDIF !ENDIF +STORELIBPATH = $(STORELIBPATH:\\=\) +LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(STORELIBPATH)" +!ENDIF !ENDIF # If either debugging or symbols are enabled, enable PDBs. @@ -1370,8 +1374,8 @@ clean: del /Q *.lo *.ilk *.lib *.obj *.pdb sqlite3.exe libsqlite3.lib del /Q *.cod *.da *.bb *.bbg gmon.out del /Q sqlite3.h opcodes.c opcodes.h - del /Q lemon.exe lempar.c parse.* - del /Q mkkeywordhash.exe keywordhash.h + del /Q lemon.* lempar.c parse.* + del /Q mkkeywordhash.* keywordhash.h del /Q notasharedlib.* -rmdir /Q/S .deps -rmdir /Q/S .libs diff --git a/manifest b/manifest index 44f5a64a12..0392d96bc7 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Enhance\sthe\sWin32\sVFS\sI/O\sretry\slogic. -D 2014-05-02T22:39:54.345 +C Work\son\scleaning\sup\sthe\sMSVC\sbatch\sbuild\stool\sintegration\swith\sVisual\sStudio\s2013. +D 2014-05-05T20:24:34.916 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc fdacba6fb574868c71fde6db6b77789a383a4c19 +F Makefile.msc 1e8c0f7e80f73b36d04520844e1e6c335cfe2000 F Makefile.vxworks db21ed42a01d5740e656b16f92cb5d8d5e5dd315 F README.md 64f270c43c38c46de749e419c22f0ae2f4499fe8 F VERSION 9f823c026c6a32fc5f84d212a8aae0a221dba45c @@ -1116,7 +1116,7 @@ F test/without_rowid5.test b4a639a367f04d382d20e8f44fc1be4f2d57d107 F test/wordcount.c 9915e06cb33d8ca8109b8700791afe80d305afda F test/zeroblob.test caaecfb4f908f7bc086ed238668049f96774d688 F test/zerodamage.test cf6748bad89553cc1632be51a6f54e487e4039ac -F tool/build-all-msvc.bat e0917e787df675b020d250d60a00de8abaa4e30a x +F tool/build-all-msvc.bat 3ffc57eadcf6229ee8b940dde2cb2eac03813bf3 x F tool/build-shell.sh 950f47c6174f1eea171319438b93ba67ff5bf367 F tool/checkSpacing.c 810e51703529a204fc4e1eb060e9ab663e3c06d2 F tool/diffdb.c 7524b1b5df217c20cd0431f6789851a4e0cb191b @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P faa469355eabb2c407f24638b090725448aac37f 598a3875ce98015e9a38db54bd0cb6290424c3d8 -R ea09ee65f031a4f810c7f319ecf812cc +P 4760504396bcf44528eb8de9ed7aa5d9659c54b6 +R 430719bf3d22eab8999716ab9edcc7d3 U mistachkin -Z f9afa829a6179e568b33b0a29e5a3328 +Z 9015c1d19aac4c8ce51a7c46f2b60a96 diff --git a/manifest.uuid b/manifest.uuid index fea2bea1a8..22c786fba4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4760504396bcf44528eb8de9ed7aa5d9659c54b6 \ No newline at end of file +b5a72dfd59fba6a9938ac1afdec06a2e6809fd2d \ No newline at end of file diff --git a/tool/build-all-msvc.bat b/tool/build-all-msvc.bat index 6e0aeb572c..fcb4d2c8c0 100755 --- a/tool/build-all-msvc.bat +++ b/tool/build-all-msvc.bat @@ -238,6 +238,7 @@ GOTO set_vcvarsall_done :set_vcvarsall_phone SET VCVARSALL=%VCINSTALLDIR%\WPSDK\WP80\vcvarsphoneall.bat :set_vcvarsall_done +SET VCVARSALL=%VCVARSALL:\\=\% REM REM NOTE: This is the outer loop. There should be exactly one iteration per @@ -268,6 +269,7 @@ FOR %%P IN (%PLATFORMS%) DO ( CALL :fn_UnsetVariable DevEnvDir CALL :fn_UnsetVariable ExtensionSdkDir CALL :fn_UnsetVariable Framework35Version + CALL :fn_UnsetVariable Framework40Version CALL :fn_UnsetVariable FrameworkDir CALL :fn_UnsetVariable FrameworkDir32 CALL :fn_UnsetVariable FrameworkVersion @@ -283,6 +285,8 @@ FOR %%P IN (%PLATFORMS%) DO ( CALL :fn_UnsetVariable WindowsSdkDir CALL :fn_UnsetVariable WindowsSdkDir_35 CALL :fn_UnsetVariable WindowsSdkDir_old + CALL :fn_UnsetVariable WindowsSDK_ExecutablePath_x86 + CALL :fn_UnsetVariable WindowsSDK_ExecutablePath_x64 REM REM NOTE: Reset the PATH here to the absolute bare minimum required. @@ -374,10 +378,9 @@ FOR %%P IN (%PLATFORMS%) DO ( REM REM NOTE: The Windows 8.1 SDK has a slightly different directory - REM naming convention. Currently, this tool assumes that - REM the Windows 8.1 SDK should only be used with MSVC 2013. + REM naming convention. REM - IF "%VisualStudioVersion%" == "12.0" ( + IF DEFINED USE_WINV63_NSDKLIBPATH ( CALL :fn_AppendVariable NSDKLIBPATH \lib\winv6.3\um\x86 ) ELSE ( CALL :fn_AppendVariable NSDKLIBPATH \lib\win8\um\x86 From 220942b7576815b3034148232c565957662c31d7 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 5 May 2014 21:08:47 +0000 Subject: [PATCH 42/71] More modularization work on the MSVC batch build tool. FossilOrigin-Name: 9a06773acc181e981f61f476a8e56417b98beba9 --- manifest | 14 +++++++------- manifest.uuid | 2 +- tool/build-all-msvc.bat | 18 ++++++++++++++++-- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index f693fb7396..bd0a456f88 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Create\snew\sheader\sfile\s"os_setup.h"\sused\sto\sdetect\sthe\soperating\ssystem\nat\scompile-time,\sand\s"os_win.h"\sthat\scontains\swindows-specific\sheader\sin\nformation. -D 2014-05-05T20:32:15.817 +C More\smodularization\swork\son\sthe\sMSVC\sbatch\sbuild\stool. +D 2014-05-05T21:08:47.656 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1118,7 +1118,7 @@ F test/without_rowid5.test b4a639a367f04d382d20e8f44fc1be4f2d57d107 F test/wordcount.c 9915e06cb33d8ca8109b8700791afe80d305afda F test/zeroblob.test caaecfb4f908f7bc086ed238668049f96774d688 F test/zerodamage.test cf6748bad89553cc1632be51a6f54e487e4039ac -F tool/build-all-msvc.bat 3ffc57eadcf6229ee8b940dde2cb2eac03813bf3 x +F tool/build-all-msvc.bat b4f0b5577fd248529c3247c06309226c35db9ff6 x F tool/build-shell.sh 950f47c6174f1eea171319438b93ba67ff5bf367 F tool/checkSpacing.c 810e51703529a204fc4e1eb060e9ab663e3c06d2 F tool/diffdb.c 7524b1b5df217c20cd0431f6789851a4e0cb191b @@ -1168,7 +1168,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P b5a72dfd59fba6a9938ac1afdec06a2e6809fd2d 3ac75725a66730566105ad184a988a8f80f51055 -R 8d3da72aab741dd9279195207e6c8bd7 -U drh -Z 892b3ce35265d0d6387ad7a7c3c1d88b +P bd92e0f80c1158d810b0d47e38f2fe06ff24bb8c +R 219607b5b5ba75498daf10431b648c43 +U mistachkin +Z de5605f1e301dc16c5db1ce3f5edb9e1 diff --git a/manifest.uuid b/manifest.uuid index feb33def06..e6d8119bed 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bd92e0f80c1158d810b0d47e38f2fe06ff24bb8c \ No newline at end of file +9a06773acc181e981f61f476a8e56417b98beba9 \ No newline at end of file diff --git a/tool/build-all-msvc.bat b/tool/build-all-msvc.bat index fcb4d2c8c0..b34fe0c829 100755 --- a/tool/build-all-msvc.bat +++ b/tool/build-all-msvc.bat @@ -146,6 +146,17 @@ IF NOT DEFINED CONFIGURATIONS ( %_VECHO% Configurations = '%CONFIGURATIONS%' +REM +REM NOTE: If the command used to invoke NMAKE is not already set, use the +REM default. +REM +IF NOT DEFINED NMAKE_CMD ( + SET NMAKE_CMD=nmake -B -f Makefile.msc +) + +%_VECHO% NmakeCmd = '%NMAKE_CMD%' +%_VECHO% NmakeArgs = '%NMAKE_ARGS%' + REM REM NOTE: Setup environment variables to translate between the MSVC platform REM names and the names to be used for the platform-specific binary @@ -303,6 +314,8 @@ FOR %%P IN (%PLATFORMS%) DO ( REM environment variables to be picked up by the MSVC makefile REM itself. REM + %_AECHO% Building the "%%B" configuration for platform "%%D"... + IF /I "%%B" == "Debug" ( SET DEBUG=2 SET MEMDEBUG=1 @@ -395,7 +408,7 @@ FOR %%P IN (%PLATFORMS%) DO ( REM file, etc. REM IF NOT DEFINED NOCLEAN ( - %__ECHO% nmake -f Makefile.msc clean + %__ECHO% %NMAKE_CMD% clean IF ERRORLEVEL 1 ( ECHO Failed to clean for platform %%P. @@ -407,6 +420,7 @@ FOR %%P IN (%PLATFORMS%) DO ( REM need to remove the build output for the files we are REM specifically wanting to build for each platform. REM + %_AECHO% Cleaning final output files only... %__ECHO% DEL /Q *.lo sqlite3.dll sqlite3.lib sqlite3.pdb ) @@ -417,7 +431,7 @@ FOR %%P IN (%PLATFORMS%) DO ( REM Also, disable looking for and/or linking to the native Tcl REM runtime library. REM - %__ECHO% nmake -f Makefile.msc sqlite3.dll XCOMPILE=1 USE_NATIVE_LIBPATHS=1 NO_TCL=1 %NMAKE_ARGS% + %__ECHO% %NMAKE_CMD% sqlite3.dll XCOMPILE=1 USE_NATIVE_LIBPATHS=1 NO_TCL=1 %NMAKE_ARGS% IF ERRORLEVEL 1 ( ECHO Failed to build %%B "sqlite3.dll" for platform %%P. From 7dd9fb65e26606025ddd3ef4ba7f8f8b7842393b Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 5 May 2014 22:43:17 +0000 Subject: [PATCH 43/71] When building for Windows 8.0 using Visual Studio 2013, make sure the cross-compilation native library path is set correctly. FossilOrigin-Name: c1ab9092e29bc9d172c1f1a3becbcc83b79f2080 --- Makefile.msc | 4 ++-- manifest | 14 +++++++------- manifest.uuid | 2 +- tool/build-all-msvc.bat | 5 ++++- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index 1d59040844..2b970e3097 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -962,7 +962,7 @@ lempar.c: $(TOP)\src\lempar.c copy $(TOP)\src\lempar.c . lemon.exe: $(TOP)\tool\lemon.c lempar.c - $(BCC) -Daccess=_access -Fe$@ $(TOP)\tool\lemon.c /link $(NLTLIBPATHS) + $(BCC) -Daccess=_access -Fe$@ $(TOP)\tool\lemon.c /link $(NLTLINKOPTS) $(NLTLIBPATHS) # Rules to build individual *.lo files from generated *.c files. This # applies to: @@ -1230,7 +1230,7 @@ sqlite3.h: $(TOP)\src\sqlite.h.in $(TOP)\manifest.uuid $(TOP)\VERSION $(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP:\=/) > sqlite3.h mkkeywordhash.exe: $(TOP)\tool\mkkeywordhash.c - $(BCC) -Fe$@ $(OPT_FEATURE_FLAGS) $(OPTS) $(TOP)\tool\mkkeywordhash.c /link $(NLTLIBPATHS) + $(BCC) -Fe$@ $(OPT_FEATURE_FLAGS) $(OPTS) $(TOP)\tool\mkkeywordhash.c /link $(NLTLINKOPTS) $(NLTLIBPATHS) keywordhash.h: $(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe .\mkkeywordhash.exe > keywordhash.h diff --git a/manifest b/manifest index bd0a456f88..089fba14e7 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C More\smodularization\swork\son\sthe\sMSVC\sbatch\sbuild\stool. -D 2014-05-05T21:08:47.656 +C When\sbuilding\sfor\sWindows\s8.0\susing\sVisual\sStudio\s2013,\smake\ssure\sthe\scross-compilation\snative\slibrary\spath\sis\sset\scorrectly. +D 2014-05-05T22:43:17.621 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc cf9737350840c56f1e46204aa310227b8183c221 +F Makefile.msc 7e6c495d9a145054a09f518781916c7503f7a8e9 F Makefile.vxworks 034289efa9d591b04b1a73598623119c306cbba0 F README.md 64f270c43c38c46de749e419c22f0ae2f4499fe8 F VERSION 9f823c026c6a32fc5f84d212a8aae0a221dba45c @@ -1118,7 +1118,7 @@ F test/without_rowid5.test b4a639a367f04d382d20e8f44fc1be4f2d57d107 F test/wordcount.c 9915e06cb33d8ca8109b8700791afe80d305afda F test/zeroblob.test caaecfb4f908f7bc086ed238668049f96774d688 F test/zerodamage.test cf6748bad89553cc1632be51a6f54e487e4039ac -F tool/build-all-msvc.bat b4f0b5577fd248529c3247c06309226c35db9ff6 x +F tool/build-all-msvc.bat a0534c971b86fe95f1983f445db5b896d3394818 x F tool/build-shell.sh 950f47c6174f1eea171319438b93ba67ff5bf367 F tool/checkSpacing.c 810e51703529a204fc4e1eb060e9ab663e3c06d2 F tool/diffdb.c 7524b1b5df217c20cd0431f6789851a4e0cb191b @@ -1168,7 +1168,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P bd92e0f80c1158d810b0d47e38f2fe06ff24bb8c -R 219607b5b5ba75498daf10431b648c43 +P 9a06773acc181e981f61f476a8e56417b98beba9 +R e91b1cf1dddb5793408fd6de2be74034 U mistachkin -Z de5605f1e301dc16c5db1ce3f5edb9e1 +Z b713eaa9517cf3b104e3dc82bdd65124 diff --git a/manifest.uuid b/manifest.uuid index e6d8119bed..af81c8bc82 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9a06773acc181e981f61f476a8e56417b98beba9 \ No newline at end of file +c1ab9092e29bc9d172c1f1a3becbcc83b79f2080 \ No newline at end of file diff --git a/tool/build-all-msvc.bat b/tool/build-all-msvc.bat index b34fe0c829..1fb61d4df8 100755 --- a/tool/build-all-msvc.bat +++ b/tool/build-all-msvc.bat @@ -277,6 +277,7 @@ FOR %%P IN (%PLATFORMS%) DO ( REM and/or Visual Studio. This block may need to be updated in the REM future to account for additional environment variables. REM + CALL :fn_UnsetVariable CommandPromptType CALL :fn_UnsetVariable DevEnvDir CALL :fn_UnsetVariable ExtensionSdkDir CALL :fn_UnsetVariable Framework35Version @@ -314,7 +315,7 @@ FOR %%P IN (%PLATFORMS%) DO ( REM environment variables to be picked up by the MSVC makefile REM itself. REM - %_AECHO% Building the "%%B" configuration for platform "%%D"... + %_AECHO% Building the %%B configuration for platform %%P with name %%D... IF /I "%%B" == "Debug" ( SET DEBUG=2 @@ -395,6 +396,8 @@ FOR %%P IN (%PLATFORMS%) DO ( REM IF DEFINED USE_WINV63_NSDKLIBPATH ( CALL :fn_AppendVariable NSDKLIBPATH \lib\winv6.3\um\x86 + ) ELSE IF "%VisualStudioVersion%" == "12.0" ( + CALL :fn_AppendVariable NSDKLIBPATH \..\8.0\lib\win8\um\x86 ) ELSE ( CALL :fn_AppendVariable NSDKLIBPATH \lib\win8\um\x86 ) From 897f6833fc4482ea1bada1e9ddde538704a8dc56 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 6 May 2014 00:09:46 +0000 Subject: [PATCH 44/71] Refactor VSIX package creation tool to better support Visual Studio 2013. FossilOrigin-Name: 8be166af239cbdaff3937c4003a6dc6e8a96e53a --- manifest | 12 +-- manifest.uuid | 2 +- tool/mkvsix.tcl | 250 +++++++++++++++++++++++++++++++----------------- 3 files changed, 167 insertions(+), 97 deletions(-) diff --git a/manifest b/manifest index 44f50d85ea..eb814f0314 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\supdates\sfrom\strunk. -D 2014-05-05T22:50:19.792 +C Refactor\sVSIX\spackage\screation\stool\sto\sbetter\ssupport\sVisual\sStudio\s2013. +D 2014-05-06T00:09:46.488 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1140,7 +1140,7 @@ F tool/mksqlite3c-noext.tcl 1712d3d71256ca1f297046619c89e77a4d7c8f6d F tool/mksqlite3c.tcl ba274df71f5e6534b0a913c7c48eabfcbd0934b6 F tool/mksqlite3h.tcl ba24038056f51fde07c0079c41885ab85e2cff12 F tool/mksqlite3internalh.tcl b6514145a7d5321b47e64e19b8116cc44f973eb1 -F tool/mkvsix.tcl 0d439bb042d0f889210afa91674fe7bf07131d39 +F tool/mkvsix.tcl ee6cf791d22baeac5d488a30bac156e5098fabb9 F tool/offsets.c fe4262fdfa378e8f5499a42136d17bf3b98f6091 F tool/omittest.tcl 34d7ac01fe4fd18e3637f64abe12c40eca0f6b97 F tool/opcodeDoc.awk b3a2a3d5d3075b8bd90b7afe24283efdd586659c @@ -1168,7 +1168,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix a94fb9b1b1ef06efc2898975cdfcfa9643731f5e -P eaa8fc810fcff8171b6c5cff8c661a6faf0e023b c1ab9092e29bc9d172c1f1a3becbcc83b79f2080 -R 6e2bd0951e32bcb3346844cc25759e20 +P 668ed76ac6cfd5810aea7319c752d77fafaab319 +R 5812195e1a7ed7ccc7cd4024eddb3bfc U mistachkin -Z 072e655b0f9d22e16552a74286be445f +Z add6d3433971df64a412a72ae0f72bd9 diff --git a/manifest.uuid b/manifest.uuid index 78722cf280..8b91267341 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -668ed76ac6cfd5810aea7319c752d77fafaab319 \ No newline at end of file +8be166af239cbdaff3937c4003a6dc6e8a96e53a \ No newline at end of file diff --git a/tool/mkvsix.tcl b/tool/mkvsix.tcl index 787326f0f1..feb1bcefd9 100644 --- a/tool/mkvsix.tcl +++ b/tool/mkvsix.tcl @@ -66,11 +66,15 @@ # containing the root of the source tree for SQLite. The third argument is # optional and if present must contain the flavor the VSIX package to build. # Currently, the only supported package flavors are "WinRT", "WinRT81", and -# "WP80". The fourth argument is optional and if present must be a string -# containing a list of platforms to include in the VSIX package. The format -# of the platform list string is "platform1,platform2,platform3". Typically, -# when on Windows, this script is executed using commands similar to the -# following from a normal Windows command prompt: +# "WP80", "WP81", and "Win32". The fourth argument is optional and if present +# must be a string containing a list of platforms to include in the VSIX +# package. The platform list is "platform1,platform2,platform3". The fifth +# argument is optional and if present must contain the version of Visual Studio +# required by the package. Currently, the only supported versions are "2012" +# and "2013". The package flavors "WinRT81" and "WP81" are only supported when +# the Visual Studio version is "2013". Typically, when on Windows, this script +# is executed using commands similar to the following from a normal Windows +# command prompt: # # CD /D C:\dev\sqlite\core # tclsh85 tool\mkvsix.tcl C:\Temp @@ -100,7 +104,7 @@ proc fail { {error ""} {usage false} } { puts stdout "usage:\ [file tail [info nameofexecutable]]\ [file tail [info script]] \[sourceDirectory\]\ -\[packageFlavor\] \[platformNames\]" +\[packageFlavor\] \[platformNames\] \[vsVersion\]" exit 1 } @@ -170,13 +174,50 @@ proc writeFile { fileName data } { return "" } -proc substFile { fileName } { +proc getMinVsVersionXmlChunk { vsVersion } { + switch -exact $vsVersion { + 2012 { + return [appendArgs \ + "\r\n " {MinVSVersion="11.0"}] + } + 2013 { + return [appendArgs \ + "\r\n " {MinVSVersion="12.0"}] + } + default { + return "" + } + } +} + +proc getExtraFileListXmlChunk { packageFlavor vsVersion } { # - # NOTE: Performs all Tcl command, variable, and backslash substitutions in - # the specified file and then rewrites the contents of that same file - # with the substituted data. + # NOTE: Neither Windows Phone 8.0 nor Windows Phone 8.1 require any extra + # attributes in their VSIX package SDK manifests. # - return [writeFile $fileName [uplevel 1 [list subst [readFile $fileName]]]] + if {[string equal $packageFlavor WP80] || \ + [string equal $packageFlavor WP81]} then { + return "" + } + + set appliesTo [expr {[string equal $packageFlavor Win32] ? \ + "VisualC" : "WindowsAppContainer"}] + + switch -exact $vsVersion { + 2012 { + return [appendArgs \ + "\r\n " AppliesTo=\" $appliesTo \" \ + "\r\n " {DependsOn="Microsoft.VCLibs, version=11.0"}] + } + 2013 { + return [appendArgs \ + "\r\n " AppliesTo=\" $appliesTo \" \ + "\r\n " {DependsOn="Microsoft.VCLibs, version=12.0"}] + } + default { + return "" + } + } } proc replaceFileNameTokens { fileName name buildName platformName } { @@ -188,6 +229,15 @@ proc replaceFileNameTokens { fileName name buildName platformName } { $name] $fileName] } +proc substFile { fileName } { + # + # NOTE: Performs all Tcl command, variable, and backslash substitutions in + # the specified file and then rewrites the contents of that same file + # with the substituted data. + # + return [writeFile $fileName [uplevel 1 [list subst [readFile $fileName]]]] +} + # # NOTE: This is the entry point for this script. # @@ -206,7 +256,7 @@ set rootName [file rootname [file tail $script]] # NOTE: Process and verify all the command line arguments. # set argc [llength $argv] -if {$argc < 1 || $argc > 4} then {fail} +if {$argc < 1 || $argc > 5} then {fail} set binaryDirectory [lindex $argv 0] @@ -251,93 +301,113 @@ if {[string length $packageFlavor] == 0} then { fail "invalid package flavor" } -if {[string equal -nocase $packageFlavor WinRT]} then { - set shortName SQLite.WinRT - set displayName "SQLite for Windows Runtime" - set targetPlatformIdentifier Windows - set targetPlatformVersion v8.0 - set minVsVersion [appendArgs \ - "\r\n " {MinVSVersion="11.0"}] - set extraSdkPath "" - set extraFileListAttributes [appendArgs \ - "\r\n " {AppliesTo="WindowsAppContainer"} \ - "\r\n " {DependsOn="Microsoft.VCLibs, version=11.0"}] -} elseif {[string equal -nocase $packageFlavor WinRT_2013]} then { - set shortName SQLite.WinRT.2013 - set displayName "SQLite for Windows Runtime (2013)" - set targetPlatformIdentifier Windows - set targetPlatformVersion v8.0 - set minVsVersion [appendArgs \ - "\r\n " {MinVSVersion="12.0"}] - set extraSdkPath "" - set extraFileListAttributes [appendArgs \ - "\r\n " {AppliesTo="WindowsAppContainer"} \ - "\r\n " {DependsOn="Microsoft.VCLibs, version=12.0"}] -} elseif {[string equal -nocase $packageFlavor WinRT81]} then { - set shortName SQLite.WinRT81 - set displayName "SQLite for Windows Runtime (Windows 8.1)" - set targetPlatformIdentifier Windows - set targetPlatformVersion v8.1 - set minVsVersion [appendArgs \ - "\r\n " {MinVSVersion="12.0"}] - set extraSdkPath "" - set extraFileListAttributes [appendArgs \ - "\r\n " {AppliesTo="WindowsAppContainer"} \ - "\r\n " {DependsOn="Microsoft.VCLibs, version=12.0"}] -} elseif {[string equal -nocase $packageFlavor WP80]} then { - set shortName SQLite.WP80 - set displayName "SQLite for Windows Phone" - set targetPlatformIdentifier "Windows Phone" - set targetPlatformVersion v8.0 - set minVsVersion [appendArgs \ - "\r\n " {MinVSVersion="11.0"}] - set extraSdkPath "\\..\\$targetPlatformIdentifier" - set extraFileListAttributes "" -} elseif {[string equal -nocase $packageFlavor WP80_2013]} then { - set shortName SQLite.WP80.2013 - set displayName "SQLite for Windows Phone (2013)" - set targetPlatformIdentifier "Windows Phone" - set targetPlatformVersion v8.0 - set minVsVersion [appendArgs \ - "\r\n " {MinVSVersion="12.0"}] - set extraSdkPath "\\..\\$targetPlatformIdentifier" - set extraFileListAttributes "" -} elseif {[string equal -nocase $packageFlavor WP81]} then { - set shortName SQLite.WP81 - set displayName "SQLite for Windows Phone 8.1" - set targetPlatformIdentifier "WindowsPhoneApp" - set targetPlatformVersion v8.1 - set minVsVersion [appendArgs \ - "\r\n " {MinVSVersion="12.0"}] - set extraSdkPath "\\..\\$targetPlatformIdentifier" - set extraFileListAttributes "" -} elseif {[string equal -nocase $packageFlavor Win32]} then { - set shortName SQLite.Win32 - set displayName "SQLite for Windows" - set targetPlatformIdentifier Windows - set targetPlatformVersion v8.0 - set minVsVersion [appendArgs \ - "\r\n " {MinVSVersion="11.0"}] - set extraSdkPath "" - set extraFileListAttributes [appendArgs \ - "\r\n " {AppliesTo="VisualC"} \ - "\r\n " {DependsOn="Microsoft.VCLibs, version=11.0"}] -} else { - fail [appendArgs \ - "unsupported package flavor, must be one of: " \ - "WinRT WinRT_2013 WinRT81 WP80 WP80_2013 WP81 Win32"] -} - if {$argc >= 4} then { set platformNames [list] foreach platformName [split [lindex $argv 3] ", "] { + set platformName [string trim $platformName] + if {[string length $platformName] > 0} then { lappend platformNames $platformName } } } +if {$argc >= 5} then { + set vsVersion [lindex $argv 4] +} else { + set vsVersion 2012 +} + +if {[string length $vsVersion] == 0} then { + fail "invalid Visual Studio version" +} + +if {$vsVersion ne "2012" && $vsVersion ne "2013"} then { + fail [appendArgs \ + "unsupported Visual Studio version, must be one of: " \ + [list 2012 2013]] +} + +set shortNames(WinRT,2012) SQLite.WinRT +set shortNames(WinRT,2013) SQLite.WinRT.2013 +set shortNames(WinRT81,2013) SQLite.WinRT81 +set shortNames(WP80,2012) SQLite.WP80 +set shortNames(WP80,2013) SQLite.WP80.2013 +set shortNames(WP81,2013) SQLite.WP81 +set shortNames(Win32,2012) SQLite.Win32 +set shortNames(Win32,2013) SQLite.Win32.2013 + +set displayNames(WinRT,2012) "SQLite for Windows Runtime" +set displayNames(WinRT,2013) "SQLite for Windows Runtime" +set displayNames(WinRT81,2013) "SQLite for Windows Runtime (Windows 8.1)" +set displayNames(WP80,2012) "SQLite for Windows Phone" +set displayNames(WP80,2013) "SQLite for Windows Phone" +set displayNames(WP81,2013) "SQLite for Windows Phone 8.1" +set displayNames(Win32,2012) "SQLite for Windows" +set displayNames(Win32,2013) "SQLite for Windows" + +if {[string equal $packageFlavor WinRT]} then { + set shortName $shortNames($packageFlavor,$vsVersion) + set displayName $displayNames($packageFlavor,$vsVersion) + set targetPlatformIdentifier Windows + set targetPlatformVersion v8.0 + set minVsVersion [getMinVsVersionXmlChunk $vsVersion] + set extraSdkPath "" + set extraFileListAttributes \ + [getExtraFileListXmlChunk $packageFlavor $vsVersion] +} elseif {[string equal $packageFlavor WinRT81]} then { + if {$vsVersion ne "2013"} then { + fail [appendArgs \ + "unsupported combination, package flavor " $packageFlavor \ + " is only supported with Visual Studio 2013"] + } + set shortName $shortNames($packageFlavor,$vsVersion) + set displayName $displayNames($packageFlavor,$vsVersion) + set targetPlatformIdentifier Windows + set targetPlatformVersion v8.1 + set minVsVersion [getMinVsVersionXmlChunk $vsVersion] + set extraSdkPath "" + set extraFileListAttributes \ + [getExtraFileListXmlChunk $packageFlavor $vsVersion] +} elseif {[string equal $packageFlavor WP80]} then { + set shortName $shortNames($packageFlavor,$vsVersion) + set displayName $displayNames($packageFlavor,$vsVersion) + set targetPlatformIdentifier "Windows Phone" + set targetPlatformVersion v8.0 + set minVsVersion [getMinVsVersionXmlChunk $vsVersion] + set extraSdkPath "\\..\\$targetPlatformIdentifier" + set extraFileListAttributes \ + [getExtraFileListXmlChunk $packageFlavor $vsVersion] +} elseif {[string equal $packageFlavor WP81]} then { + if {$vsVersion ne "2013"} then { + fail [appendArgs \ + "unsupported combination, package flavor " $packageFlavor \ + " is only supported with Visual Studio 2013"] + } + set shortName $shortNames($packageFlavor,$vsVersion) + set displayName $displayNames($packageFlavor,$vsVersion) + set targetPlatformIdentifier WindowsPhoneApp + set targetPlatformVersion v8.1 + set minVsVersion [getMinVsVersionXmlChunk $vsVersion] + set extraSdkPath "\\..\\$targetPlatformIdentifier" + set extraFileListAttributes \ + [getExtraFileListXmlChunk $packageFlavor $vsVersion] +} elseif {[string equal $packageFlavor Win32]} then { + set shortName $shortNames($packageFlavor,$vsVersion) + set displayName $displayNames($packageFlavor,$vsVersion) + set targetPlatformIdentifier Windows + set targetPlatformVersion v8.0 + set minVsVersion [getMinVsVersionXmlChunk $vsVersion] + set extraSdkPath "" + set extraFileListAttributes \ + [getExtraFileListXmlChunk $packageFlavor $vsVersion] +} else { + fail [appendArgs \ + "unsupported package flavor, must be one of: " \ + [list WinRT WinRT81 WP80 WP81 Win32]] +} + ############################################################################### # @@ -525,7 +595,7 @@ if {![info exists buildNames]} then { # overridden via the command line or the user-specific customizations # file. # -if {![info exists platformNames]} then { +if {![info exists platformNames] || [llength $platformNames] == 0} then { set platformNames [list x86 x64 ARM] } From 441aa095edbdcbdcf77765424026a9d263bd1ece Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 6 May 2014 21:26:34 +0000 Subject: [PATCH 45/71] Enhance the VSIX package creation tool to better support Visual Studio 2013. FossilOrigin-Name: c1fb04f61eb74d80d1b7607ae6904fe2e1717988 --- manifest | 12 +-- manifest.uuid | 2 +- tool/mkvsix.tcl | 199 ++++++++++++++++++++++++++++++++++-------------- 3 files changed, 150 insertions(+), 63 deletions(-) diff --git a/manifest b/manifest index 089fba14e7..5cc3df3b6b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sbuilding\sfor\sWindows\s8.0\susing\sVisual\sStudio\s2013,\smake\ssure\sthe\scross-compilation\snative\slibrary\spath\sis\sset\scorrectly. -D 2014-05-05T22:43:17.621 +C Enhance\sthe\sVSIX\spackage\screation\stool\sto\sbetter\ssupport\sVisual\sStudio\s2013. +D 2014-05-06T21:26:34.436 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1140,7 +1140,7 @@ F tool/mksqlite3c-noext.tcl 1712d3d71256ca1f297046619c89e77a4d7c8f6d F tool/mksqlite3c.tcl ba274df71f5e6534b0a913c7c48eabfcbd0934b6 F tool/mksqlite3h.tcl ba24038056f51fde07c0079c41885ab85e2cff12 F tool/mksqlite3internalh.tcl b6514145a7d5321b47e64e19b8116cc44f973eb1 -F tool/mkvsix.tcl 6477fb9dab838b7eea1eed50658ff1cda04850b5 +F tool/mkvsix.tcl 924dcdecda86969686833301c08f84cca2600d94 F tool/offsets.c fe4262fdfa378e8f5499a42136d17bf3b98f6091 F tool/omittest.tcl 34d7ac01fe4fd18e3637f64abe12c40eca0f6b97 F tool/opcodeDoc.awk b3a2a3d5d3075b8bd90b7afe24283efdd586659c @@ -1168,7 +1168,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 9a06773acc181e981f61f476a8e56417b98beba9 -R e91b1cf1dddb5793408fd6de2be74034 +P c1ab9092e29bc9d172c1f1a3becbcc83b79f2080 +R a58107e75319df920bcb764f3109ea62 U mistachkin -Z b713eaa9517cf3b104e3dc82bdd65124 +Z 469c7a319e5e989d73df4c062d995fca diff --git a/manifest.uuid b/manifest.uuid index af81c8bc82..4bcb53df54 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c1ab9092e29bc9d172c1f1a3becbcc83b79f2080 \ No newline at end of file +c1fb04f61eb74d80d1b7607ae6904fe2e1717988 \ No newline at end of file diff --git a/tool/mkvsix.tcl b/tool/mkvsix.tcl index 65fa731245..712cf32278 100644 --- a/tool/mkvsix.tcl +++ b/tool/mkvsix.tcl @@ -65,12 +65,15 @@ # argument is optional and if present must contain the name of the directory # containing the root of the source tree for SQLite. The third argument is # optional and if present must contain the flavor the VSIX package to build. -# Currently, the only supported package flavors are "WinRT", "WinRT81", and -# "WP80". The fourth argument is optional and if present must be a string -# containing a list of platforms to include in the VSIX package. The format -# of the platform list string is "platform1,platform2,platform3". Typically, -# when on Windows, this script is executed using commands similar to the -# following from a normal Windows command prompt: +# Currently, the only supported package flavors are "WinRT", "WinRT81", "WP80", +# and "Win32". The fourth argument is optional and if present must be a string +# containing a list of platforms to include in the VSIX package. The platform +# list is "platform1,platform2,platform3". The fifth argument is optional and +# if present must contain the version of Visual Studio required by the package. +# Currently, the only supported versions are "2012" and "2013". The package +# flavor "WinRT81" is only supported when the Visual Studio version is "2013". +# Typically, when on Windows, this script is executed using commands similar to +# the following from a normal Windows command prompt: # # CD /D C:\dev\sqlite\core # tclsh85 tool\mkvsix.tcl C:\Temp @@ -100,7 +103,7 @@ proc fail { {error ""} {usage false} } { puts stdout "usage:\ [file tail [info nameofexecutable]]\ [file tail [info script]] \[sourceDirectory\]\ -\[packageFlavor\] \[platformNames\]" +\[packageFlavor\] \[platformNames\] \[vsVersion\]" exit 1 } @@ -170,13 +173,49 @@ proc writeFile { fileName data } { return "" } -proc substFile { fileName } { +proc getMinVsVersionXmlChunk { vsVersion } { + switch -exact $vsVersion { + 2012 { + return [appendArgs \ + "\r\n " {MinVSVersion="11.0"}] + } + 2013 { + return [appendArgs \ + "\r\n " {MinVSVersion="12.0"}] + } + default { + return "" + } + } +} + +proc getExtraFileListXmlChunk { packageFlavor vsVersion } { # - # NOTE: Performs all Tcl command, variable, and backslash substitutions in - # the specified file and then rewrites the contents of that same file - # with the substituted data. + # NOTE: Windows Phone 8.0 does not require any extra attributes in its VSIX + # package SDK manifests. # - return [writeFile $fileName [uplevel 1 [list subst [readFile $fileName]]]] + if {[string equal $packageFlavor WP80]} then { + return "" + } + + set appliesTo [expr {[string equal $packageFlavor Win32] ? \ + "VisualC" : "WindowsAppContainer"}] + + switch -exact $vsVersion { + 2012 { + return [appendArgs \ + "\r\n " AppliesTo=\" $appliesTo \" \ + "\r\n " {DependsOn="Microsoft.VCLibs, version=11.0"}] + } + 2013 { + return [appendArgs \ + "\r\n " AppliesTo=\" $appliesTo \" \ + "\r\n " {DependsOn="Microsoft.VCLibs, version=12.0"}] + } + default { + return "" + } + } } proc replaceFileNameTokens { fileName name buildName platformName } { @@ -188,6 +227,15 @@ proc replaceFileNameTokens { fileName name buildName platformName } { $name] $fileName] } +proc substFile { fileName } { + # + # NOTE: Performs all Tcl command, variable, and backslash substitutions in + # the specified file and then rewrites the contents of that same file + # with the substituted data. + # + return [writeFile $fileName [uplevel 1 [list subst [readFile $fileName]]]] +} + # # NOTE: This is the entry point for this script. # @@ -206,7 +254,7 @@ set rootName [file rootname [file tail $script]] # NOTE: Process and verify all the command line arguments. # set argc [llength $argv] -if {$argc < 1 || $argc > 4} then {fail} +if {$argc < 1 || $argc > 5} then {fail} set binaryDirectory [lindex $argv 0] @@ -251,58 +299,97 @@ if {[string length $packageFlavor] == 0} then { fail "invalid package flavor" } -if {[string equal -nocase $packageFlavor WinRT]} then { - set shortName SQLite.WinRT - set displayName "SQLite for Windows Runtime" - set targetPlatformIdentifier Windows - set targetPlatformVersion v8.0 - set minVsVersion 11.0 - set extraSdkPath "" - set extraFileListAttributes [appendArgs \ - "\r\n " {AppliesTo="WindowsAppContainer"} \ - "\r\n " {DependsOn="Microsoft.VCLibs, version=11.0"}] -} elseif {[string equal -nocase $packageFlavor WinRT81]} then { - set shortName SQLite.WinRT81 - set displayName "SQLite for Windows Runtime (Windows 8.1)" - set targetPlatformIdentifier Windows - set targetPlatformVersion v8.1 - set minVsVersion 12.0 - set extraSdkPath "" - set extraFileListAttributes [appendArgs \ - "\r\n " {AppliesTo="WindowsAppContainer"} \ - "\r\n " {DependsOn="Microsoft.VCLibs, version=12.0"}] -} elseif {[string equal -nocase $packageFlavor WP80]} then { - set shortName SQLite.WP80 - set displayName "SQLite for Windows Phone" - set targetPlatformIdentifier "Windows Phone" - set targetPlatformVersion v8.0 - set minVsVersion 11.0 - set extraSdkPath "\\..\\$targetPlatformIdentifier" - set extraFileListAttributes "" -} elseif {[string equal -nocase $packageFlavor Win32]} then { - set shortName SQLite.Win32 - set displayName "SQLite for Windows" - set targetPlatformIdentifier Windows - set targetPlatformVersion v8.0 - set minVsVersion 11.0 - set extraSdkPath "" - set extraFileListAttributes [appendArgs \ - "\r\n " {AppliesTo="VisualC"} \ - "\r\n " {DependsOn="Microsoft.VCLibs, version=11.0"}] -} else { - fail "unsupported package flavor, must be one of: WinRT WinRT81 WP80 Win32" -} - if {$argc >= 4} then { set platformNames [list] foreach platformName [split [lindex $argv 3] ", "] { + set platformName [string trim $platformName] + if {[string length $platformName] > 0} then { lappend platformNames $platformName } } } +if {$argc >= 5} then { + set vsVersion [lindex $argv 4] +} else { + set vsVersion 2012 +} + +if {[string length $vsVersion] == 0} then { + fail "invalid Visual Studio version" +} + +if {$vsVersion ne "2012" && $vsVersion ne "2013"} then { + fail [appendArgs \ + "unsupported Visual Studio version, must be one of: " \ + [list 2012 2013]] +} + +set shortNames(WinRT,2012) SQLite.WinRT +set shortNames(WinRT,2013) SQLite.WinRT.2013 +set shortNames(WinRT81,2013) SQLite.WinRT81 +set shortNames(WP80,2012) SQLite.WP80 +set shortNames(WP80,2013) SQLite.WP80.2013 +set shortNames(Win32,2012) SQLite.Win32 +set shortNames(Win32,2013) SQLite.Win32.2013 + +set displayNames(WinRT,2012) "SQLite for Windows Runtime" +set displayNames(WinRT,2013) "SQLite for Windows Runtime" +set displayNames(WinRT81,2013) "SQLite for Windows Runtime (Windows 8.1)" +set displayNames(WP80,2012) "SQLite for Windows Phone" +set displayNames(WP80,2013) "SQLite for Windows Phone" +set displayNames(Win32,2012) "SQLite for Windows" +set displayNames(Win32,2013) "SQLite for Windows" + +if {[string equal $packageFlavor WinRT]} then { + set shortName $shortNames($packageFlavor,$vsVersion) + set displayName $displayNames($packageFlavor,$vsVersion) + set targetPlatformIdentifier Windows + set targetPlatformVersion v8.0 + set minVsVersion [getMinVsVersionXmlChunk $vsVersion] + set extraSdkPath "" + set extraFileListAttributes \ + [getExtraFileListXmlChunk $packageFlavor $vsVersion] +} elseif {[string equal $packageFlavor WinRT81]} then { + if {$vsVersion ne "2013"} then { + fail [appendArgs \ + "unsupported combination, package flavor " $packageFlavor \ + " is only supported with Visual Studio 2013"] + } + set shortName $shortNames($packageFlavor,$vsVersion) + set displayName $displayNames($packageFlavor,$vsVersion) + set targetPlatformIdentifier Windows + set targetPlatformVersion v8.1 + set minVsVersion [getMinVsVersionXmlChunk $vsVersion] + set extraSdkPath "" + set extraFileListAttributes \ + [getExtraFileListXmlChunk $packageFlavor $vsVersion] +} elseif {[string equal $packageFlavor WP80]} then { + set shortName $shortNames($packageFlavor,$vsVersion) + set displayName $displayNames($packageFlavor,$vsVersion) + set targetPlatformIdentifier "Windows Phone" + set targetPlatformVersion v8.0 + set minVsVersion [getMinVsVersionXmlChunk $vsVersion] + set extraSdkPath "\\..\\$targetPlatformIdentifier" + set extraFileListAttributes \ + [getExtraFileListXmlChunk $packageFlavor $vsVersion] +} elseif {[string equal $packageFlavor Win32]} then { + set shortName $shortNames($packageFlavor,$vsVersion) + set displayName $displayNames($packageFlavor,$vsVersion) + set targetPlatformIdentifier Windows + set targetPlatformVersion v8.0 + set minVsVersion [getMinVsVersionXmlChunk $vsVersion] + set extraSdkPath "" + set extraFileListAttributes \ + [getExtraFileListXmlChunk $packageFlavor $vsVersion] +} else { + fail [appendArgs \ + "unsupported package flavor, must be one of: " \ + [list WinRT WinRT81 WP80 Win32]] +} + ############################################################################### # @@ -490,7 +577,7 @@ if {![info exists buildNames]} then { # overridden via the command line or the user-specific customizations # file. # -if {![info exists platformNames]} then { +if {![info exists platformNames] || [llength $platformNames] == 0} then { set platformNames [list x86 x64 ARM] } From 6117272e03bf37fb4754205bc13f0826b35bc788 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 6 May 2014 21:37:10 +0000 Subject: [PATCH 46/71] Update the template VSIX package file as well. FossilOrigin-Name: 99d96765cc378fde7b285f4577ea2b5d130d9a61 --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/win/sqlite.vsix | Bin 32825 -> 32820 bytes 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 5cc3df3b6b..91f6b157b3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\sVSIX\spackage\screation\stool\sto\sbetter\ssupport\sVisual\sStudio\s2013. -D 2014-05-06T21:26:34.436 +C Update\sthe\stemplate\sVSIX\spackage\sfile\sas\swell. +D 2014-05-06T21:37:10.782 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1167,8 +1167,8 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 -F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P c1ab9092e29bc9d172c1f1a3becbcc83b79f2080 -R a58107e75319df920bcb764f3109ea62 +F tool/win/sqlite.vsix a94fb9b1b1ef06efc2898975cdfcfa9643731f5e +P c1fb04f61eb74d80d1b7607ae6904fe2e1717988 +R e86d9d0c4e9d221c23e483062a69a0ee U mistachkin -Z 469c7a319e5e989d73df4c062d995fca +Z 61035ffe1ca5b1a658207d61a8a19a18 diff --git a/manifest.uuid b/manifest.uuid index 4bcb53df54..eaa07f7373 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c1fb04f61eb74d80d1b7607ae6904fe2e1717988 \ No newline at end of file +99d96765cc378fde7b285f4577ea2b5d130d9a61 \ No newline at end of file diff --git a/tool/win/sqlite.vsix b/tool/win/sqlite.vsix index ac4afb3f4bc2aa7c4419376732e5a7b3910d1070..de4d0a1f96ad3b53b899a7bcfbba53f23a1d4990 100644 GIT binary patch delta 313 zcmV-90mlBhfC99D0OuO<`_nW@U49E_iKh zRgh0h!$1(l?}dJcWy#e>>ZK#bm%`XWEq3S8|QbemE~Y0oaTn%_((vn3jMK$hu5rMX28f2R6=tSpo|+cp1k z&?E6gzKOLgD~sljMljq5(yIX^5M~`ES(~)C%Z4o z>h=0f?;lVcLX-FbvkrS(1PuUykwjgh(dOF#001|Wc7HGjqdpA6kXp}^t$#ZK36tr6 LN(K*n00000h#!t5 delta 327 zcmV-N0l5COfC9OI0OuO<`_nW@U49E_iKh zRgg_f!$1&*?}h${W$D#KZ=zjGK?{Nvq0ozZh{>cG*)L~jO-1tG-H^t%dwbxSecze+ zVsEtg1`6n`l2_TK6ktnNq1`DNxxToTVsST}%vY%42?I|jBE5SIe_j_q&xHXcXTQ}E zJ2@CHTmsxYYkz0f$~Q)}%A9&PQEY?%E`p0{nAg)fZ^nWJ4kYG}8c?a;FS$0QC1 z?h?bEsTdnBnF&84L+FO-P4qtHhIQ18Uc+H9DEUC-NS;|xEval$e#wy_Ij^4%t(c|# zxOQav_CdVezRUh93HcbB;15ts0|b-5e;BhRdp`sXQ(e?Tdi!qm-~a#sOp|ecFb5W4 ZChz5K&XcHrI{^-px001^@mD~UT From 62e603a904e8d4c48ba70df5ef1376dcfdb27e73 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 7 May 2014 15:09:24 +0000 Subject: [PATCH 47/71] Add test cases and documentation for the nolock and immutable query parameters. FossilOrigin-Name: 19d56cbaca222b32e2e354063149cccd89441d73 --- manifest | 15 ++--- manifest.uuid | 2 +- src/sqlite.h.in | 29 +++++++++- src/test_vfs.c | 73 +++++++++++++++++------- test/nolock.test | 143 +++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 232 insertions(+), 30 deletions(-) create mode 100644 test/nolock.test diff --git a/manifest b/manifest index 6ac75f7987..f945a392c4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Take\sout\sthe\sspecial\shandling\sof\snolock=true\sin\sos_win.c\sand\sadd\sit\sto\npager.c,\sso\sthat\sit\sworks\sfor\sall\sVFSes.\s\sAdd\sthe\spPager->noLock\sboolean\nfor\sclarity. -D 2014-05-01T01:49:22.279 +C Add\stest\scases\sand\sdocumentation\sfor\sthe\snolock\sand\simmutable\squery\nparameters. +D 2014-05-07T15:09:24.618 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -220,7 +220,7 @@ F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be F src/select.c ed459f7f478a1e533d19c4b953693b3ffa2efd15 F src/shell.c 2afe7a7154e97be0c74c5feacf09626bda8493be -F src/sqlite.h.in ad2bbeb6a41c228f6a9dbf24df62e4d3eff79ee5 +F src/sqlite.h.in 564fc23db33870b5096b20d72df7491ce0b8b74f F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc F src/sqliteInt.h b2947801eccefd7ba3e5f14e1353289351a83cf3 @@ -270,7 +270,7 @@ F src/test_superlock.c 2b97936ca127d13962c3605dbc9a4ef269c424cd F src/test_syscall.c 2e21ca7f7dc54a028f1967b63f1e76155c356f9b F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb -F src/test_vfs.c e72f555ef7a59080f898fcf1a233deb9eb704ea9 +F src/test_vfs.c f84075a388527892ff184988f43b69ce69b8083c F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/tokenize.c 6da2de6e12218ccb0aea5184b56727d011f4bee7 @@ -713,6 +713,7 @@ F test/multiplex3.test d228f59eac91839a977eac19f21d053f03e4d101 F test/mutex1.test 78b2b9bb320e51d156c4efdb71b99b051e7a4b41 F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660 F test/nan.test e9648b9d007c7045242af35e11a984d4b169443a +F test/nolock.test 77b278317fa765b1ada5f88d59219cdeb8364f8f F test/notify1.test 669b2b743618efdc18ca4b02f45423d5d2304abf F test/notify2.test ce23eb522c9e1fff6443f96376fe67872202061c F test/notify3.test 10ff25cde502e72a92053a2f215d64bece4ef934 @@ -1166,7 +1167,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 1829c38c3233c8cb194fae6d560d35a8916b1348 -R 9585bcff14fbe775451a3a647d896079 +P 725c1c14be331bce9bf074960dbfa7b6d4c1ba87 +R 89db3fd7a34cc009fbfa4be31d82927f U drh -Z 3dae204f6c4a7be4483961b87ae0c17a +Z 2af631f996ea658459fe7078e363eaf4 diff --git a/manifest.uuid b/manifest.uuid index d015fa2fa6..164fe70792 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -725c1c14be331bce9bf074960dbfa7b6d4c1ba87 \ No newline at end of file +19d56cbaca222b32e2e354063149cccd89441d73 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 81220a8822..9be07fc1e6 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -2778,6 +2778,30 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** ^If sqlite3_open_v2() is used and the "cache" parameter is present in ** a URI filename, its value overrides any behavior requested by setting ** SQLITE_OPEN_PRIVATECACHE or SQLITE_OPEN_SHAREDCACHE flag. +** +**
  • psow: ^The psow parameter may be "true" (or "on" or "yes" or +** "1") or "false" (or "off" or "no" or "0") to indicate that the +** [powersafe overwrite] property does or does not apply to the +** storage media on which the database file resides. ^The psow query +** parameter only works for the built-in unix and Windows VFSes. +** +**
  • nolock: ^The nolock parameter is a boolean query parameter +** which if set disables file locking in rollback journal modes. This +** is useful for accessing a database on a filesystem that does not +** support locking. Caution: Database corruption might result if two +** or more processes write to the same database and any one of those +** processes uses nolock=1. +** +**
  • immutable: ^The immutable parameter is a boolean query +** parameter that indicates that the database file is stored on +** read-only media. ^When immutable is set, SQLite assumes that the +** database file cannot be changed, even by a process with higher +** privilege, and so the database is opened read-only and all locking +** and change detection is disabled. Caution: Setting the immutable +** property on a database file that does in fact change can result +** in incorrect query results and/or [SQLITE_CORRUPT] errors. +** See also: [SQLITE_IOCAP_IMMUTABLE]. +** ** ** ** ^Specifying an unknown parameter in the query component of a URI is not an @@ -2807,8 +2831,9 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** Open file "data.db" in the current directory for read-only access. ** Regardless of whether or not shared-cache mode is enabled by ** default, use a private cache. -** file:/home/fred/data.db?vfs=unix-nolock -** Open file "/home/fred/data.db". Use the special VFS "unix-nolock". +** file:/home/fred/data.db?vfs=unix-dotfile +** Open file "/home/fred/data.db". Use the special VFS "unix-dotfile" +** that uses dot-files in place of posix advisory locking. ** file:data.db?mode=readonly ** An error. "readonly" is not a valid option for the "mode" parameter. ** diff --git a/src/test_vfs.c b/src/test_vfs.c index 613b0fce77..7ee2a93453 100644 --- a/src/test_vfs.c +++ b/src/test_vfs.c @@ -127,8 +127,10 @@ struct Testvfs { #define TESTVFS_FULLPATHNAME_MASK 0x00008000 #define TESTVFS_READ_MASK 0x00010000 #define TESTVFS_UNLOCK_MASK 0x00020000 +#define TESTVFS_LOCK_MASK 0x00040000 +#define TESTVFS_CKLOCK_MASK 0x00080000 -#define TESTVFS_ALL_MASK 0x0003FFFF +#define TESTVFS_ALL_MASK 0x000FFFFF #define TESTVFS_MAX_PAGES 1024 @@ -466,8 +468,15 @@ static int tvfsFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ ** Lock an tvfs-file. */ static int tvfsLock(sqlite3_file *pFile, int eLock){ - TestvfsFd *p = tvfsGetFd(pFile); - return sqlite3OsLock(p->pReal, eLock); + TestvfsFd *pFd = tvfsGetFd(pFile); + Testvfs *p = (Testvfs *)pFd->pVfs->pAppData; + if( p->pScript && p->mask&TESTVFS_LOCK_MASK ){ + char zLock[30]; + sqlite3_snprintf(sizeof(zLock),zLock,"%d",eLock); + tvfsExecTcl(p, "xLock", Tcl_NewStringObj(pFd->zFilename, -1), + Tcl_NewStringObj(zLock, -1), 0, 0); + } + return sqlite3OsLock(pFd->pReal, eLock); } /* @@ -476,6 +485,12 @@ static int tvfsLock(sqlite3_file *pFile, int eLock){ static int tvfsUnlock(sqlite3_file *pFile, int eLock){ TestvfsFd *pFd = tvfsGetFd(pFile); Testvfs *p = (Testvfs *)pFd->pVfs->pAppData; + if( p->pScript && p->mask&TESTVFS_UNLOCK_MASK ){ + char zLock[30]; + sqlite3_snprintf(sizeof(zLock),zLock,"%d",eLock); + tvfsExecTcl(p, "xUnlock", Tcl_NewStringObj(pFd->zFilename, -1), + Tcl_NewStringObj(zLock, -1), 0, 0); + } if( p->mask&TESTVFS_WRITE_MASK && tvfsInjectIoerr(p) ){ return SQLITE_IOERR_UNLOCK; } @@ -486,8 +501,13 @@ static int tvfsUnlock(sqlite3_file *pFile, int eLock){ ** Check if another file-handle holds a RESERVED lock on an tvfs-file. */ static int tvfsCheckReservedLock(sqlite3_file *pFile, int *pResOut){ - TestvfsFd *p = tvfsGetFd(pFile); - return sqlite3OsCheckReservedLock(p->pReal, pResOut); + TestvfsFd *pFd = tvfsGetFd(pFile); + Testvfs *p = (Testvfs *)pFd->pVfs->pAppData; + if( p->pScript && p->mask&TESTVFS_CKLOCK_MASK ){ + tvfsExecTcl(p, "xCheckReservedLock", Tcl_NewStringObj(pFd->zFilename, -1), + 0, 0, 0); + } + return sqlite3OsCheckReservedLock(pFd->pReal, pResOut); } /* @@ -1111,26 +1131,32 @@ static int testvfs_obj_cmd( break; } + /* TESTVFS filter METHOD-LIST + ** + ** Activate special processing for those methods contained in the list + */ case CMD_FILTER: { static struct VfsMethod { char *zName; int mask; } vfsmethod [] = { - { "xShmOpen", TESTVFS_SHMOPEN_MASK }, - { "xShmLock", TESTVFS_SHMLOCK_MASK }, - { "xShmBarrier", TESTVFS_SHMBARRIER_MASK }, - { "xShmUnmap", TESTVFS_SHMCLOSE_MASK }, - { "xShmMap", TESTVFS_SHMMAP_MASK }, - { "xSync", TESTVFS_SYNC_MASK }, - { "xDelete", TESTVFS_DELETE_MASK }, - { "xWrite", TESTVFS_WRITE_MASK }, - { "xRead", TESTVFS_READ_MASK }, - { "xTruncate", TESTVFS_TRUNCATE_MASK }, - { "xOpen", TESTVFS_OPEN_MASK }, - { "xClose", TESTVFS_CLOSE_MASK }, - { "xAccess", TESTVFS_ACCESS_MASK }, - { "xFullPathname", TESTVFS_FULLPATHNAME_MASK }, - { "xUnlock", TESTVFS_UNLOCK_MASK }, + { "xShmOpen", TESTVFS_SHMOPEN_MASK }, + { "xShmLock", TESTVFS_SHMLOCK_MASK }, + { "xShmBarrier", TESTVFS_SHMBARRIER_MASK }, + { "xShmUnmap", TESTVFS_SHMCLOSE_MASK }, + { "xShmMap", TESTVFS_SHMMAP_MASK }, + { "xSync", TESTVFS_SYNC_MASK }, + { "xDelete", TESTVFS_DELETE_MASK }, + { "xWrite", TESTVFS_WRITE_MASK }, + { "xRead", TESTVFS_READ_MASK }, + { "xTruncate", TESTVFS_TRUNCATE_MASK }, + { "xOpen", TESTVFS_OPEN_MASK }, + { "xClose", TESTVFS_CLOSE_MASK }, + { "xAccess", TESTVFS_ACCESS_MASK }, + { "xFullPathname", TESTVFS_FULLPATHNAME_MASK }, + { "xUnlock", TESTVFS_UNLOCK_MASK }, + { "xLock", TESTVFS_LOCK_MASK }, + { "xCheckReservedLock", TESTVFS_CKLOCK_MASK }, }; Tcl_Obj **apElem = 0; int nElem = 0; @@ -1162,6 +1188,12 @@ static int testvfs_obj_cmd( break; } + /* + ** TESTVFS script ?SCRIPT? + ** + ** Query or set the script to be run when filtered VFS events + ** occur. + */ case CMD_SCRIPT: { if( objc==3 ){ int nByte; @@ -1248,6 +1280,7 @@ static int testvfs_obj_cmd( { "safe_append", SQLITE_IOCAP_SAFE_APPEND }, { "undeletable_when_open", SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN }, { "powersafe_overwrite", SQLITE_IOCAP_POWERSAFE_OVERWRITE }, + { "immutable", SQLITE_IOCAP_IMMUTABLE }, { 0, 0 } }; Tcl_Obj *pRet; diff --git a/test/nolock.test b/test/nolock.test new file mode 100644 index 0000000000..98ed762cac --- /dev/null +++ b/test/nolock.test @@ -0,0 +1,143 @@ +# 2014-05-07 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file implements regression tests for SQLite library. The +# focus of this file is testing the nolock=1 and immutable=1 query +# parameters and the SQLITE_IOCAP_IMMUTABLE device characteristic. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +unset -nocomplain tvfs_calls +proc tvfs_reset {} { + global tvfs_calls + array set tvfs_calls {xLock 0 xUnlock 0 xCheckReservedLock 0 xAccess 0} +} +proc tvfs_callback {op args} { + global tvfs_calls + incr tvfs_calls($op) + return SQLITE_OK +} +tvfs_reset + +testvfs tvfs +tvfs script tvfs_callback +tvfs filter {xLock xUnlock xCheckReservedLock xAccess} + +############################################################################ +# Verify that the nolock=1 query parameter for URI filenames disables all +# calls to xLock and xUnlock for rollback databases. +# +do_test nolock-1.0 { + db close + forcedelete test.db + tvfs_reset + sqlite db test.db -vfs tvfs + db eval {CREATE TABLE t1(a,b,c); INSERT INTO t1 VALUES(1,2,3);} + list xLock $::tvfs_calls(xLock) xUnlock $::tvfs_calls(xUnlock) \ + xCheckReservedLock $::tvfs_calls(xCheckReservedLock) +} {xLock 7 xUnlock 5 xCheckReservedLock 0} + +do_test nolock-1.1 { + db close + forcedelete test.db + tvfs_reset + sqlite db file:test.db?nolock=0 -vfs tvfs -uri 1 + db eval {CREATE TABLE t1(a,b,c); INSERT INTO t1 VALUES(1,2,3);} + list xLock $::tvfs_calls(xLock) xUnlock $::tvfs_calls(xUnlock) \ + xCheckReservedLock $::tvfs_calls(xCheckReservedLock) +} {xLock 7 xUnlock 5 xCheckReservedLock 0} + +do_test nolock-1.2 { + db close + forcedelete test.db + tvfs_reset + sqlite db file:test.db?nolock=1 -vfs tvfs -uri 1 + db eval {CREATE TABLE t1(a,b,c); INSERT INTO t1 VALUES(1,2,3);} + list xLock $::tvfs_calls(xLock) xUnlock $::tvfs_calls(xUnlock) \ + xCheckReservedLock $::tvfs_calls(xCheckReservedLock) +} {xLock 0 xUnlock 0 xCheckReservedLock 0} + +############################################################################# +# Verify that immutable=1 disables both locking and xAccess calls to the +# journal files. +# +do_test nolock-2.0 { + db close + forcedelete test.db + # begin by creating a test database + sqlite3 db test.db + db eval { + CREATE TABLE t1(a,b); + INSERT INTO t1 VALUES('hello','world'); + CREATE TABLE t2(x,y); + INSERT INTO t2 VALUES(12345,67890); + SELECT * FROM t1, t2; + } +} {hello world 12345 67890} +do_test nolock-2.1 { + tvfs_reset + sqlite3 db2 test.db -vfs tvfs + db2 eval {SELECT * FROM t1, t2} +} {hello world 12345 67890} +do_test nolock-2.2 { + list xLock $::tvfs_calls(xLock) xUnlock $::tvfs_calls(xUnlock) \ + xCheckReservedLock $::tvfs_calls(xCheckReservedLock) \ + xAccess $::tvfs_calls(xAccess) +} {xLock 2 xUnlock 2 xCheckReservedLock 0 xAccess 4} + + +do_test nolock-2.11 { + db2 close + tvfs_reset + sqlite3 db2 file:test.db?immutable=0 -vfs tvfs -uri 1 + db2 eval {SELECT * FROM t1, t2} +} {hello world 12345 67890} +do_test nolock-2.12 { + list xLock $::tvfs_calls(xLock) xUnlock $::tvfs_calls(xUnlock) \ + xCheckReservedLock $::tvfs_calls(xCheckReservedLock) \ + xAccess $::tvfs_calls(xAccess) +} {xLock 2 xUnlock 2 xCheckReservedLock 0 xAccess 4} + + +do_test nolock-2.21 { + db2 close + tvfs_reset + sqlite3 db2 file:test.db?immutable=1 -vfs tvfs -uri 1 + db2 eval {SELECT * FROM t1, t2} +} {hello world 12345 67890} +do_test nolock-2.22 { + list xLock $::tvfs_calls(xLock) xUnlock $::tvfs_calls(xUnlock) \ + xCheckReservedLock $::tvfs_calls(xCheckReservedLock) \ + xAccess $::tvfs_calls(xAccess) +} {xLock 0 xUnlock 0 xCheckReservedLock 0 xAccess 0} + +############################################################################ +# Verify that the SQLITE_IOCAP_IMMUTABLE flag works +# +do_test nolock-3.1 { + db2 close + tvfs devchar immutable + tvfs_reset + sqlite3 db2 test.db -vfs tvfs + db2 eval {SELECT * FROM t1, t2} +} {hello world 12345 67890} +do_test nolock-3.2 { + list xLock $::tvfs_calls(xLock) xUnlock $::tvfs_calls(xUnlock) \ + xCheckReservedLock $::tvfs_calls(xCheckReservedLock) \ + xAccess $::tvfs_calls(xAccess) +} {xLock 0 xUnlock 0 xCheckReservedLock 0 xAccess 0} + +db2 close +db close +tvfs delete +finish_test From 6451c2b04e1f08175800d1c9359b634565c84ce6 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 7 May 2014 15:32:03 +0000 Subject: [PATCH 48/71] Fix nolock and immutable so that they work even if READONLY is requested. FossilOrigin-Name: e193aced2942e7405d0f45f90d4954b5663b4ba5 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/pager.c | 14 +++++++------- test/nolock.test | 42 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 57 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index f945a392c4..4823bece17 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stest\scases\sand\sdocumentation\sfor\sthe\snolock\sand\simmutable\squery\nparameters. -D 2014-05-07T15:09:24.618 +C Fix\snolock\sand\simmutable\sso\sthat\sthey\swork\seven\sif\sREADONLY\sis\srequested. +D 2014-05-07T15:32:03.879 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -206,7 +206,7 @@ F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c ae4b5240af4619d711301d7992396e182585269f F src/os_win.c e71678ac927d0a0fb11d993db20a9748eabf808e -F src/pager.c 6ed54e169933200b8d113674d5da46a5cbd32c6c +F src/pager.c f6bb1fa6cdf2062f2d8aec3e64db302bca519ab8 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 F src/pcache.c d8eafac28290d4bb80332005435db44991d07fc2 @@ -713,7 +713,7 @@ F test/multiplex3.test d228f59eac91839a977eac19f21d053f03e4d101 F test/mutex1.test 78b2b9bb320e51d156c4efdb71b99b051e7a4b41 F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660 F test/nan.test e9648b9d007c7045242af35e11a984d4b169443a -F test/nolock.test 77b278317fa765b1ada5f88d59219cdeb8364f8f +F test/nolock.test 0540dd96f39b8876e3ffdd8814fad0ea425efeee F test/notify1.test 669b2b743618efdc18ca4b02f45423d5d2304abf F test/notify2.test ce23eb522c9e1fff6443f96376fe67872202061c F test/notify3.test 10ff25cde502e72a92053a2f215d64bece4ef934 @@ -1167,7 +1167,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 725c1c14be331bce9bf074960dbfa7b6d4c1ba87 -R 89db3fd7a34cc009fbfa4be31d82927f +P 19d56cbaca222b32e2e354063149cccd89441d73 +R 330846b5261a6146788edf187c03b165 U drh -Z 2af631f996ea658459fe7078e363eaf4 +Z 1507cbe7cdb05905b7ab13bc4a3a8db3 diff --git a/manifest.uuid b/manifest.uuid index 164fe70792..ba9d26dfee 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -19d56cbaca222b32e2e354063149cccd89441d73 \ No newline at end of file +e193aced2942e7405d0f45f90d4954b5663b4ba5 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 9ab7cfbe54..a77dcecc4c 100644 --- a/src/pager.c +++ b/src/pager.c @@ -4676,8 +4676,8 @@ int sqlite3PagerOpen( ** + The largest page size that can be written atomically. */ if( rc==SQLITE_OK ){ + int iDc = sqlite3OsDeviceCharacteristics(pPager->fd); if( !readOnly ){ - int iDc = sqlite3OsDeviceCharacteristics(pPager->fd); setSectorSize(pPager); assert(SQLITE_DEFAULT_PAGE_SIZE<=SQLITE_MAX_DEFAULT_PAGE_SIZE); if( szPageDfltsectorSize ){ @@ -4700,12 +4700,12 @@ int sqlite3PagerOpen( } } #endif - pPager->noLock = sqlite3_uri_boolean(zFilename, "nolock", 0); - if( (iDc & SQLITE_IOCAP_IMMUTABLE)!=0 - || sqlite3_uri_boolean(zFilename, "immutable", 0) ){ - vfsFlags |= SQLITE_OPEN_READONLY; - goto act_like_temp_file; - } + } + pPager->noLock = sqlite3_uri_boolean(zFilename, "nolock", 0); + if( (iDc & SQLITE_IOCAP_IMMUTABLE)!=0 + || sqlite3_uri_boolean(zFilename, "immutable", 0) ){ + vfsFlags |= SQLITE_OPEN_READONLY; + goto act_like_temp_file; } } }else{ diff --git a/test/nolock.test b/test/nolock.test index 98ed762cac..331af08ad7 100644 --- a/test/nolock.test +++ b/test/nolock.test @@ -67,6 +67,24 @@ do_test nolock-1.2 { xCheckReservedLock $::tvfs_calls(xCheckReservedLock) } {xLock 0 xUnlock 0 xCheckReservedLock 0} +do_test nolock-1.3 { + db close + tvfs_reset + sqlite db file:test.db?nolock=0 -vfs tvfs -uri 1 -readonly 1 + db eval {SELECT * FROM t1} + list xLock $::tvfs_calls(xLock) xUnlock $::tvfs_calls(xUnlock) \ + xCheckReservedLock $::tvfs_calls(xCheckReservedLock) +} {xLock 2 xUnlock 2 xCheckReservedLock 0} + +do_test nolock-1.4 { + db close + tvfs_reset + sqlite db file:test.db?nolock=1 -vfs tvfs -uri 1 -readonly 1 + db eval {SELECT * FROM t1} + list xLock $::tvfs_calls(xLock) xUnlock $::tvfs_calls(xUnlock) \ + xCheckReservedLock $::tvfs_calls(xCheckReservedLock) +} {xLock 0 xUnlock 0 xCheckReservedLock 0} + ############################################################################# # Verify that immutable=1 disables both locking and xAccess calls to the # journal files. @@ -121,6 +139,18 @@ do_test nolock-2.22 { xAccess $::tvfs_calls(xAccess) } {xLock 0 xUnlock 0 xCheckReservedLock 0 xAccess 0} +do_test nolock-2.31 { + db2 close + tvfs_reset + sqlite3 db2 file:test.db?immutable=1 -vfs tvfs -uri 1 -readonly 1 + db2 eval {SELECT * FROM t1, t2} +} {hello world 12345 67890} +do_test nolock-2.32 { + list xLock $::tvfs_calls(xLock) xUnlock $::tvfs_calls(xUnlock) \ + xCheckReservedLock $::tvfs_calls(xCheckReservedLock) \ + xAccess $::tvfs_calls(xAccess) +} {xLock 0 xUnlock 0 xCheckReservedLock 0 xAccess 0} + ############################################################################ # Verify that the SQLITE_IOCAP_IMMUTABLE flag works # @@ -137,6 +167,18 @@ do_test nolock-3.2 { xAccess $::tvfs_calls(xAccess) } {xLock 0 xUnlock 0 xCheckReservedLock 0 xAccess 0} +do_test nolock-3.11 { + db2 close + tvfs_reset + sqlite3 db2 test.db -vfs tvfs -readonly 1 + db2 eval {SELECT * FROM t1, t2} +} {hello world 12345 67890} +do_test nolock-3.12 { + list xLock $::tvfs_calls(xLock) xUnlock $::tvfs_calls(xUnlock) \ + xCheckReservedLock $::tvfs_calls(xCheckReservedLock) \ + xAccess $::tvfs_calls(xAccess) +} {xLock 0 xUnlock 0 xCheckReservedLock 0 xAccess 0} + db2 close db close tvfs delete From eacc050f3059b197533fc2d4903cb1fb8108ec7d Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 7 May 2014 17:19:31 +0000 Subject: [PATCH 49/71] Make sure the group_concat() function returns an empty string, not a NULL, if it has at least one input row. Fix for ticket [55746f9e65f8587]. FossilOrigin-Name: 0deac8737545a020d344be96fff16660a7977ab8 --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/func.c | 2 +- test/func.test | 12 ++++++++++++ 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 28652fe3ac..9818296e6d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sSQLITE_IOCAP_IMMUTABLE\sbit\sas\sa\spossible\sreturn\svalue\sfrom\nthe\sxDeviceCharacteristics\smethod\sin\sthe\sVFS.\s\sAdd\sthe\s"nolock"\sand\n"immutable"\squery\sparameters\sto\sURI\sfilenames. -D 2014-05-07T15:46:04.082 +C Make\ssure\sthe\sgroup_concat()\sfunction\sreturns\san\sempty\sstring,\snot\sa\sNULL,\nif\sit\shas\sat\sleast\sone\sinput\srow.\s\sFix\sfor\sticket\s[55746f9e65f8587]. +D 2014-05-07T17:19:31.457 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -177,7 +177,7 @@ F src/delete.c bcf8f72126cea80fc3d5bc5494cf19b3f8935aaf F src/expr.c 4f9e497c66e2f25a4d139357a778c84d5713207c F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 5269ef07b100763134f71b889327c333bd0989cf -F src/func.c 2945bb2c4cdc0ac43733046285a4434310be1811 +F src/func.c 1db49570e01b31fd3cd00ff2c13da8f2f05e9051 F src/global.c 1d7bb7ea8254ae6a68ed9bfaf65fcb3d1690b486 F src/hash.c d139319967164f139c8d1bb8a11b14db9c4ba3cd F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 @@ -578,7 +578,7 @@ F test/fts4merge4.test c19c85ca1faa7b6d536832b49c12e1867235f584 F test/fts4noti.test aed33ba44808852dcb24bf70fa132e7bf530f057 F test/fts4unicode.test 01ec3fe2a7c3cfff3b4c0581b83caa11b33efa36 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d -F test/func.test c2cbfc23d554c5bf8678d0fb271aa4f8ef94839c +F test/func.test ae97561957aba6ca9e3a7b8a13aac41830d701ef F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test dbccee9133cfef1473c59ec07b5f0262b9d72f9a F test/func4.test 6beacdfcb0e18c358e6c2dcacf1b65d1fa80955f @@ -1169,8 +1169,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix a94fb9b1b1ef06efc2898975cdfcfa9643731f5e -P 99d96765cc378fde7b285f4577ea2b5d130d9a61 e193aced2942e7405d0f45f90d4954b5663b4ba5 -R 397fc32d1bd36ead839d4c78d231487f -T +closed e193aced2942e7405d0f45f90d4954b5663b4ba5 +P 1a0d7d3d9dd54b783e3a805961287dd01f94770c +R 34db7d5fd6dab33bdc52f4f3fa07974a U drh -Z 57c6364c482e6b92f356dd08394fdb62 +Z eb431a80b9a77174dcfd85ab3e28a5a4 diff --git a/manifest.uuid b/manifest.uuid index 9da16245c0..d19d28ec95 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1a0d7d3d9dd54b783e3a805961287dd01f94770c \ No newline at end of file +0deac8737545a020d344be96fff16660a7977ab8 \ No newline at end of file diff --git a/src/func.c b/src/func.c index 6be963580c..50f76c33de 100644 --- a/src/func.c +++ b/src/func.c @@ -1541,7 +1541,7 @@ static void groupConcatStep( } zVal = (char*)sqlite3_value_text(argv[0]); nVal = sqlite3_value_bytes(argv[0]); - if( nVal ) sqlite3StrAccumAppend(pAccum, zVal, nVal); + if( nVal || firstTerm ) sqlite3StrAccumAppend(pAccum, zVal, nVal); } } static void groupConcatFinalize(sqlite3_context *context){ diff --git a/test/func.test b/test/func.test index e3cec32153..98ae8ddeb5 100644 --- a/test/func.test +++ b/test/func.test @@ -1194,6 +1194,18 @@ do_test func-24.12 { WHEN 'program' THEN null ELSE t1 END) FROM tbl1 } } {,is,free,software} +# Tests to verify ticket http://www.sqlite.org/src/tktview/55746f9e65f8587c0 +do_test func-24.13 { + execsql { + SELECT typeof(group_concat(x)) FROM (SELECT '' AS x); + } +} {text} +do_test func-24.14 { + execsql { + SELECT typeof(group_concat(x,'')) + FROM (SELECT '' AS x UNION ALL SELECT ''); + } +} {text} # Use the test_isolation function to make sure that type conversions From 86a1119133963032a2c388a7ce5a300a904f5d00 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 7 May 2014 18:21:56 +0000 Subject: [PATCH 50/71] Ensure that the sqlite3StrAccumAppend() routine is never called with a NULL second argument. Doing so is harmless when N==0, but it causes an assert() to fail that was placed to quiet static analyzers. FossilOrigin-Name: f03fbf3700d9d4a654e3aa2e5caa810a8416bed9 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/func.c | 4 +++- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 9818296e6d..c8a928d230 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sthe\sgroup_concat()\sfunction\sreturns\san\sempty\sstring,\snot\sa\sNULL,\nif\sit\shas\sat\sleast\sone\sinput\srow.\s\sFix\sfor\sticket\s[55746f9e65f8587]. -D 2014-05-07T17:19:31.457 +C Ensure\sthat\sthe\ssqlite3StrAccumAppend()\sroutine\sis\snever\scalled\swith\sa\sNULL\nsecond\sargument.\s\sDoing\sso\sis\sharmless\swhen\sN==0,\sbut\sit\scauses\san\sassert()\nto\sfail\sthat\swas\splaced\sto\squiet\sstatic\sanalyzers. +D 2014-05-07T18:21:56.753 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -177,7 +177,7 @@ F src/delete.c bcf8f72126cea80fc3d5bc5494cf19b3f8935aaf F src/expr.c 4f9e497c66e2f25a4d139357a778c84d5713207c F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 5269ef07b100763134f71b889327c333bd0989cf -F src/func.c 1db49570e01b31fd3cd00ff2c13da8f2f05e9051 +F src/func.c 2077ccd5c77952fb654997eb81bc7be57c955ba2 F src/global.c 1d7bb7ea8254ae6a68ed9bfaf65fcb3d1690b486 F src/hash.c d139319967164f139c8d1bb8a11b14db9c4ba3cd F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 @@ -1169,7 +1169,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix a94fb9b1b1ef06efc2898975cdfcfa9643731f5e -P 1a0d7d3d9dd54b783e3a805961287dd01f94770c -R 34db7d5fd6dab33bdc52f4f3fa07974a +P 0deac8737545a020d344be96fff16660a7977ab8 +R aaf6cc2f3f5ec9bee45f0a384deaafae U drh -Z eb431a80b9a77174dcfd85ab3e28a5a4 +Z daf5890128d88bd8d8d454e0919fc158 diff --git a/manifest.uuid b/manifest.uuid index d19d28ec95..a75992b9d9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0deac8737545a020d344be96fff16660a7977ab8 \ No newline at end of file +f03fbf3700d9d4a654e3aa2e5caa810a8416bed9 \ No newline at end of file diff --git a/src/func.c b/src/func.c index 50f76c33de..91c20ff41e 100644 --- a/src/func.c +++ b/src/func.c @@ -1541,7 +1541,9 @@ static void groupConcatStep( } zVal = (char*)sqlite3_value_text(argv[0]); nVal = sqlite3_value_bytes(argv[0]); - if( nVal || firstTerm ) sqlite3StrAccumAppend(pAccum, zVal, nVal); + if( nVal || (zVal="", firstTerm) ){ + sqlite3StrAccumAppend(pAccum, zVal, nVal); + } } } static void groupConcatFinalize(sqlite3_context *context){ From d2d8ca600e5bb14050799f9246d29335dc87a476 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 7 May 2014 19:59:36 +0000 Subject: [PATCH 51/71] Fix the way parenthesis in MATCH expressions are handled by FTS if the tokenizer considers them to be token characters. FossilOrigin-Name: e21bf7a2ade6373e94ea403c665f78e1ad22143f --- ext/fts3/fts3_expr.c | 205 ++++++++++++++++++++++--------------------- manifest | 20 ++--- manifest.uuid | 2 +- test/fts3defer2.test | 1 + test/fts3expr.test | 5 ++ test/fts3expr4.test | 57 ++++++++++++ 6 files changed, 178 insertions(+), 112 deletions(-) create mode 100644 test/fts3expr4.test diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index 9c71f26ba1..95a9b1aada 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -185,40 +185,23 @@ static int getNextToken( int rc; sqlite3_tokenizer_cursor *pCursor; Fts3Expr *pRet = 0; - int nConsumed = 0; + int i = 0; - rc = sqlite3Fts3OpenTokenizer(pTokenizer, pParse->iLangid, z, n, &pCursor); + /* Set variable i to the maximum number of bytes of input to tokenize. */ + for(i=0; iiLangid, z, i, &pCursor); if( rc==SQLITE_OK ){ const char *zToken; int nToken = 0, iStart = 0, iEnd = 0, iPosition = 0; int nByte; /* total space to allocate */ rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition); - - if( (rc==SQLITE_OK || rc==SQLITE_DONE) && sqlite3_fts3_enable_parentheses ){ - int i; - if( rc==SQLITE_DONE ) iStart = n; - for(i=0; inNest++; - rc = fts3ExprParse(pParse, &z[i+1], n-i-1, &pRet, &nConsumed); - if( rc==SQLITE_OK && !pRet ){ - rc = SQLITE_DONE; - } - nConsumed = (int)(i + 1 + nConsumed); - break; - } - - if( z[i]==')' ){ - rc = SQLITE_DONE; - pParse->nNest--; - nConsumed = i+1; - break; - } - } - } - - if( nConsumed==0 && rc==SQLITE_OK ){ + if( rc==SQLITE_OK ){ nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken; pRet = (Fts3Expr *)fts3MallocZero(nByte); if( !pRet ){ @@ -252,13 +235,14 @@ static int getNextToken( } } - nConsumed = iEnd; + *pnConsumed = iEnd; + }else if( i && rc==SQLITE_DONE ){ + rc = SQLITE_OK; } pModule->xClose(pCursor); } - *pnConsumed = nConsumed; *ppExpr = pRet; return rc; } @@ -508,6 +492,21 @@ static int getNextNode( return getNextString(pParse, &zInput[1], ii-1, ppExpr); } + if( sqlite3_fts3_enable_parentheses ){ + if( *zInput=='(' ){ + int nConsumed = 0; + pParse->nNest++; + rc = fts3ExprParse(pParse, zInput+1, nInput-1, ppExpr, &nConsumed); + if( rc==SQLITE_OK && !*ppExpr ){ rc = SQLITE_DONE; } + *pnConsumed = (int)(zInput - z) + 1 + nConsumed; + return rc; + }else if( *zInput==')' ){ + pParse->nNest--; + *pnConsumed = (zInput - z) + 1; + *ppExpr = 0; + return SQLITE_DONE; + } + } /* If control flows to this point, this must be a regular token, or ** the end of the input. Read a regular token using the sqlite3_tokenizer @@ -626,96 +625,100 @@ static int fts3ExprParse( while( rc==SQLITE_OK ){ Fts3Expr *p = 0; int nByte = 0; + rc = getNextNode(pParse, zIn, nIn, &p, &nByte); + assert( nByte>0 || (rc!=SQLITE_OK && p==0) ); if( rc==SQLITE_OK ){ - int isPhrase; + if( p ){ + int isPhrase; - if( !sqlite3_fts3_enable_parentheses - && p->eType==FTSQUERY_PHRASE && pParse->isNot - ){ - /* Create an implicit NOT operator. */ - Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr)); - if( !pNot ){ - sqlite3Fts3ExprFree(p); - rc = SQLITE_NOMEM; - goto exprparse_out; - } - pNot->eType = FTSQUERY_NOT; - pNot->pRight = p; - p->pParent = pNot; - if( pNotBranch ){ - pNot->pLeft = pNotBranch; - pNotBranch->pParent = pNot; - } - pNotBranch = pNot; - p = pPrev; - }else{ - int eType = p->eType; - isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft); - - /* The isRequirePhrase variable is set to true if a phrase or - ** an expression contained in parenthesis is required. If a - ** binary operator (AND, OR, NOT or NEAR) is encounted when - ** isRequirePhrase is set, this is a syntax error. - */ - if( !isPhrase && isRequirePhrase ){ - sqlite3Fts3ExprFree(p); - rc = SQLITE_ERROR; - goto exprparse_out; - } - - if( isPhrase && !isRequirePhrase ){ - /* Insert an implicit AND operator. */ - Fts3Expr *pAnd; - assert( pRet && pPrev ); - pAnd = fts3MallocZero(sizeof(Fts3Expr)); - if( !pAnd ){ + if( !sqlite3_fts3_enable_parentheses + && p->eType==FTSQUERY_PHRASE && pParse->isNot + ){ + /* Create an implicit NOT operator. */ + Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr)); + if( !pNot ){ sqlite3Fts3ExprFree(p); rc = SQLITE_NOMEM; goto exprparse_out; } - pAnd->eType = FTSQUERY_AND; - insertBinaryOperator(&pRet, pPrev, pAnd); - pPrev = pAnd; - } + pNot->eType = FTSQUERY_NOT; + pNot->pRight = p; + p->pParent = pNot; + if( pNotBranch ){ + pNot->pLeft = pNotBranch; + pNotBranch->pParent = pNot; + } + pNotBranch = pNot; + p = pPrev; + }else{ + int eType = p->eType; + isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft); - /* This test catches attempts to make either operand of a NEAR - ** operator something other than a phrase. For example, either of - ** the following: - ** - ** (bracketed expression) NEAR phrase - ** phrase NEAR (bracketed expression) - ** - ** Return an error in either case. - */ - if( pPrev && ( + /* The isRequirePhrase variable is set to true if a phrase or + ** an expression contained in parenthesis is required. If a + ** binary operator (AND, OR, NOT or NEAR) is encounted when + ** isRequirePhrase is set, this is a syntax error. + */ + if( !isPhrase && isRequirePhrase ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_ERROR; + goto exprparse_out; + } + + if( isPhrase && !isRequirePhrase ){ + /* Insert an implicit AND operator. */ + Fts3Expr *pAnd; + assert( pRet && pPrev ); + pAnd = fts3MallocZero(sizeof(Fts3Expr)); + if( !pAnd ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_NOMEM; + goto exprparse_out; + } + pAnd->eType = FTSQUERY_AND; + insertBinaryOperator(&pRet, pPrev, pAnd); + pPrev = pAnd; + } + + /* This test catches attempts to make either operand of a NEAR + ** operator something other than a phrase. For example, either of + ** the following: + ** + ** (bracketed expression) NEAR phrase + ** phrase NEAR (bracketed expression) + ** + ** Return an error in either case. + */ + if( pPrev && ( (eType==FTSQUERY_NEAR && !isPhrase && pPrev->eType!=FTSQUERY_PHRASE) || (eType!=FTSQUERY_PHRASE && isPhrase && pPrev->eType==FTSQUERY_NEAR) - )){ - sqlite3Fts3ExprFree(p); - rc = SQLITE_ERROR; - goto exprparse_out; - } - - if( isPhrase ){ - if( pRet ){ - assert( pPrev && pPrev->pLeft && pPrev->pRight==0 ); - pPrev->pRight = p; - p->pParent = pPrev; - }else{ - pRet = p; + )){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_ERROR; + goto exprparse_out; } - }else{ - insertBinaryOperator(&pRet, pPrev, p); + + if( isPhrase ){ + if( pRet ){ + assert( pPrev && pPrev->pLeft && pPrev->pRight==0 ); + pPrev->pRight = p; + p->pParent = pPrev; + }else{ + pRet = p; + } + }else{ + insertBinaryOperator(&pRet, pPrev, p); + } + isRequirePhrase = !isPhrase; } - isRequirePhrase = !isPhrase; + pPrev = p; } assert( nByte>0 ); } assert( rc!=SQLITE_OK || (nByte>0 && nByte<=nIn) ); nIn -= nByte; zIn += nByte; - pPrev = p; } if( rc==SQLITE_DONE && pRet && isRequirePhrase ){ diff --git a/manifest b/manifest index 77f24441e5..98fa92f28f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sthe\sgroup_concat()\sfunction\sreturns\san\sempty\sstring,\snot\sa\sNULL,\s\nif\sit\shas\sat\sleast\sone\sinput\srow.\sFix\sfor\sticket\s[55746f9e65f8587]. -D 2014-05-07T18:23:04.692 +C Fix\sthe\sway\sparenthesis\sin\sMATCH\sexpressions\sare\shandled\sby\sFTS\sif\sthe\stokenizer\sconsiders\sthem\sto\sbe\stoken\scharacters. +D 2014-05-07T19:59:36.446 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -82,7 +82,7 @@ F ext/fts3/fts3.c 41b1920b9a8657963f09cb93b208c2671c5568db F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h bdeb9015405e8facffb8fc7e09174521a2a780f4 F ext/fts3/fts3_aux.c 5c211e17a64885faeb16b9ba7772f9d5445c2365 -F ext/fts3/fts3_expr.c 5165c365cb5a035f5be8bb296f7aa3211d43e4ac +F ext/fts3/fts3_expr.c 2ac35bda474f00c14c19608e49a02c8c7ceb9970 F ext/fts3/fts3_hash.c 29b986e43f4e9dd40110eafa377dc0d63c422c60 F ext/fts3/fts3_hash.h 39cf6874dc239d6b4e30479b1975fe5b22a3caaf F ext/fts3/fts3_icu.c e319e108661147bcca8dd511cd562f33a1ba81b5 @@ -541,13 +541,14 @@ F test/fts3corrupt2.test 6d96efae2f8a6af3eeaf283aba437e6d0e5447ba F test/fts3cov.test e0fb00d8b715ddae4a94c305992dfc3ef70353d7 F test/fts3d.test 597b0b76e41f0d672e2731c4d7b631d628efd13f F test/fts3defer.test 0be4440b73a2e651fc1e472066686d6ada4b9963 -F test/fts3defer2.test a3b6cbeabaf28c9398652a4d101ea224d9358479 +F test/fts3defer2.test e880e3b65bdf999f4746cdaefa65f14a98b9b724 F test/fts3defer3.test dd53fc13223c6d8264a98244e9b19abd35ed71cd F test/fts3drop.test 1b906e293d6773812587b3dc458cb9e8f3f0c297 F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851 -F test/fts3expr.test 06f1a96facc8f3e4b1ad5cc001dc5c8e83e68b9f +F test/fts3expr.test 3401d47b229c4504424caf362cc4ff704cad4162 F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a F test/fts3expr3.test 9e91b8edbcb197bf2e92161aa7696446d96dce5f +F test/fts3expr4.test 0713d94ab951ed88a8c3629a4889a48c55c4067c F test/fts3fault.test cb72dccb0a3b9f730f16c5240f3fcb9303eb1660 F test/fts3fault2.test 3198eef2804deea7cac8403e771d9cbcb752d887 F test/fts3first.test dbdedd20914c8d539aa3206c9b34a23775644641 @@ -1169,8 +1170,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix a94fb9b1b1ef06efc2898975cdfcfa9643731f5e -P 1a0d7d3d9dd54b783e3a805961287dd01f94770c f03fbf3700d9d4a654e3aa2e5caa810a8416bed9 -R aaf6cc2f3f5ec9bee45f0a384deaafae -T +closed f03fbf3700d9d4a654e3aa2e5caa810a8416bed9 -U drh -Z fad451fce86c7077ae7b04e526f1422c +P d01cedaa73d8f9e5502502a1068a9509d1de295c +R 28138eb84e0bfcefa0d0799bd622d421 +U dan +Z 3413f41d3a51a750e0d3b3c404322221 diff --git a/manifest.uuid b/manifest.uuid index ff91e39b9f..d0d7c962bf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d01cedaa73d8f9e5502502a1068a9509d1de295c \ No newline at end of file +e21bf7a2ade6373e94ea403c665f78e1ad22143f \ No newline at end of file diff --git a/test/fts3defer2.test b/test/fts3defer2.test index 608371f15d..87af52461b 100644 --- a/test/fts3defer2.test +++ b/test/fts3defer2.test @@ -59,6 +59,7 @@ do_execsql_test 1.2.1 { SELECT content FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)'; } {{a b c d e f a x y}} + do_execsql_test 1.2.2 { SELECT snippet(t1, '[', ']'), offsets(t1), mit(matchinfo(t1, 'pcxnal')) FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)'; diff --git a/test/fts3expr.test b/test/fts3expr.test index 192219f142..6e23faf633 100644 --- a/test/fts3expr.test +++ b/test/fts3expr.test @@ -509,4 +509,9 @@ do_test fts3expr-8.7 { test_fts3expr "((((blah!))))" } {PHRASE 3 0 blah} do_test fts3expr-8.8 { test_fts3expr "(,(blah-),)" } {PHRASE 3 0 blah} set sqlite_fts3_enable_parentheses 0 + +do_test fts3expr-9.1 { + test_fts3expr "f (e NEAR/2 a)" +} {AND {PHRASE 3 0 f} {NEAR/2 {PHRASE 3 0 e} {PHRASE 3 0 a}}} + finish_test diff --git a/test/fts3expr4.test b/test/fts3expr4.test new file mode 100644 index 0000000000..94737971b4 --- /dev/null +++ b/test/fts3expr4.test @@ -0,0 +1,57 @@ +# 2014 May 7 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS3 module. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix fts3expr4 + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts3||!icu { + finish_test + return +} + +set sqlite_fts3_enable_parentheses 1 + +proc test_icu_fts3expr {expr} { + db one {SELECT fts3_exprtest('icu', $expr, 'a', 'b', 'c')} +} + +proc do_icu_expr_test {tn expr res} { + uplevel [list do_test $tn [list test_icu_fts3expr $expr] $res] +} + +#------------------------------------------------------------------------- +# +do_icu_expr_test 1.1 "abcd" {PHRASE 3 0 abcd} +do_icu_expr_test 1.2 " tag " {PHRASE 3 0 tag} +do_icu_expr_test 1.3 {"x y z"} {PHRASE 3 0 x y z} +do_icu_expr_test 1.4 {x OR y} {OR {PHRASE 3 0 x} {PHRASE 3 0 y}} +do_icu_expr_test 1.5 {(x OR y)} {OR {PHRASE 3 0 x} {PHRASE 3 0 y}} +do_icu_expr_test 1.6 { "(x OR y)" } {PHRASE 3 0 ( x or y )} + +# In "col:word", if "col" is not the name of a column, the entire thing +# is passed to the tokenizer. +# +do_icu_expr_test 1.7 {a:word} {PHRASE 0 0 word} +do_icu_expr_test 1.8 {d:word} {PHRASE 3 0 d:word} + +set sqlite_fts3_enable_parentheses 0 + +do_icu_expr_test 2.1 { + f (e NEAR/2 a) +} {AND {AND {AND {PHRASE 3 0 f} {PHRASE 3 0 (}} {NEAR/2 {PHRASE 3 0 e} {PHRASE 3 0 a}}} {PHRASE 3 0 )}} + +finish_test + From 8009c9b4391db9f64523362cc0ae5a0a4f17e1fb Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 7 May 2014 20:24:00 +0000 Subject: [PATCH 52/71] A better fix for the group_concat() problem. FossilOrigin-Name: 1c086deebd0eb442918972e4b9185a5a0db3b8c3 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/func.c | 4 +--- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 98fa92f28f..2487a4cca5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sway\sparenthesis\sin\sMATCH\sexpressions\sare\shandled\sby\sFTS\sif\sthe\stokenizer\sconsiders\sthem\sto\sbe\stoken\scharacters. -D 2014-05-07T19:59:36.446 +C A\sbetter\sfix\sfor\sthe\sgroup_concat()\sproblem. +D 2014-05-07T20:24:00.613 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -177,7 +177,7 @@ F src/delete.c bcf8f72126cea80fc3d5bc5494cf19b3f8935aaf F src/expr.c 4f9e497c66e2f25a4d139357a778c84d5713207c F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 5269ef07b100763134f71b889327c333bd0989cf -F src/func.c 2077ccd5c77952fb654997eb81bc7be57c955ba2 +F src/func.c 2e16316ec3a6365a0dc3e553c586f91b20f7f6c8 F src/global.c 1d7bb7ea8254ae6a68ed9bfaf65fcb3d1690b486 F src/hash.c d139319967164f139c8d1bb8a11b14db9c4ba3cd F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 @@ -1170,7 +1170,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix a94fb9b1b1ef06efc2898975cdfcfa9643731f5e -P d01cedaa73d8f9e5502502a1068a9509d1de295c -R 28138eb84e0bfcefa0d0799bd622d421 -U dan -Z 3413f41d3a51a750e0d3b3c404322221 +P e21bf7a2ade6373e94ea403c665f78e1ad22143f +R afe7614324621c209bd24609b796c72c +U drh +Z 90528f628504d485125985a9d758e5e7 diff --git a/manifest.uuid b/manifest.uuid index d0d7c962bf..db899c0006 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e21bf7a2ade6373e94ea403c665f78e1ad22143f \ No newline at end of file +1c086deebd0eb442918972e4b9185a5a0db3b8c3 \ No newline at end of file diff --git a/src/func.c b/src/func.c index 91c20ff41e..bc5c78981e 100644 --- a/src/func.c +++ b/src/func.c @@ -1541,9 +1541,7 @@ static void groupConcatStep( } zVal = (char*)sqlite3_value_text(argv[0]); nVal = sqlite3_value_bytes(argv[0]); - if( nVal || (zVal="", firstTerm) ){ - sqlite3StrAccumAppend(pAccum, zVal, nVal); - } + if( zVal ) sqlite3StrAccumAppend(pAccum, zVal, nVal); } } static void groupConcatFinalize(sqlite3_context *context){ From 9ee8810be233e2ad62fd6cc0531cfd1f6b99490c Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 7 May 2014 20:33:17 +0000 Subject: [PATCH 53/71] Remove a superfluous variable initialization. FossilOrigin-Name: 68766f837491cb89c2103f2627eb9e23ab326a68 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 2487a4cca5..8dad091c67 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C A\sbetter\sfix\sfor\sthe\sgroup_concat()\sproblem. -D 2014-05-07T20:24:00.613 +C Remove\sa\ssuperfluous\svariable\sinitialization. +D 2014-05-07T20:33:17.262 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -294,7 +294,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 91dfd382273c3f67fcdc0ac4728f9140f91c6ab3 +F src/where.c fd4b525cd5ab652cea2fbc71ac15c975271ca461 F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1170,7 +1170,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix a94fb9b1b1ef06efc2898975cdfcfa9643731f5e -P e21bf7a2ade6373e94ea403c665f78e1ad22143f -R afe7614324621c209bd24609b796c72c +P 1c086deebd0eb442918972e4b9185a5a0db3b8c3 +R 995ed9b07fad6d84d47ce217aca5ca74 U drh -Z 90528f628504d485125985a9d758e5e7 +Z 2d839c831dc7cae35991010f3444c69e diff --git a/manifest.uuid b/manifest.uuid index db899c0006..4980e6682b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1c086deebd0eb442918972e4b9185a5a0db3b8c3 \ No newline at end of file +68766f837491cb89c2103f2627eb9e23ab326a68 \ No newline at end of file diff --git a/src/where.c b/src/where.c index c5892590bc..2daa9f1536 100644 --- a/src/where.c +++ b/src/where.c @@ -3761,7 +3761,7 @@ static int whereLoopCheaperProperSubset( if( pX->rRun > pY->rRun ) return 0; /* X costs more than Y */ if( pX->nOut > pY->nOut ) return 0; /* X costs more than Y */ } - for(j=0, i=pX->nLTerm-1; i>=0; i--){ + for(i=pX->nLTerm-1; i>=0; i--){ for(j=pY->nLTerm-1; j>=0; j--){ if( pY->aLTerm[j]==pX->aLTerm[i] ) break; } From 2e7be081526084a73bee5ebe778bc83904b87de3 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 7 May 2014 21:16:56 +0000 Subject: [PATCH 54/71] Include sqlite3rtree.h in the tsrc/ pile of source files during target_source in the main.mk makefile. FossilOrigin-Name: 116bed5af664899a73b46dca528ac0c021fc50c3 --- main.mk | 1 + manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/main.mk b/main.mk index da1c30b2fe..3bdbe810f8 100644 --- a/main.mk +++ b/main.mk @@ -210,6 +210,7 @@ SRC += \ $(TOP)/ext/icu/sqliteicu.h \ $(TOP)/ext/icu/icu.c SRC += \ + $(TOP)/ext/rtree/sqlite3rtree.h \ $(TOP)/ext/rtree/rtree.h \ $(TOP)/ext/rtree/rtree.c diff --git a/manifest b/manifest index 8dad091c67..bec4a2d4fe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sa\ssuperfluous\svariable\sinitialization. -D 2014-05-07T20:33:17.262 +C Include\ssqlite3rtree.h\sin\sthe\stsrc/\spile\sof\ssource\sfiles\sduring\ntarget_source\sin\sthe\smain.mk\smakefile. +D 2014-05-07T21:16:56.524 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -144,7 +144,7 @@ F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt f439556c5ce01ced70987e5ee86549a45165d9ff -F main.mk 601f43d91e6c7043127ec192bf85c9a1ae940621 +F main.mk cfa185eed4e0f7e9d28a2e3167cecaa9d6cb39f3 F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea F mkopcodeh.awk c6b3fa301db6ef7ac916b14c60868aeaec1337b5 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 @@ -1170,7 +1170,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix a94fb9b1b1ef06efc2898975cdfcfa9643731f5e -P 1c086deebd0eb442918972e4b9185a5a0db3b8c3 -R 995ed9b07fad6d84d47ce217aca5ca74 +P 68766f837491cb89c2103f2627eb9e23ab326a68 +R 1b75d6e5e9a19ccc14f8e59e21c749f6 U drh -Z 2d839c831dc7cae35991010f3444c69e +Z 3528e73d8f87faaf66e6658213b6d573 diff --git a/manifest.uuid b/manifest.uuid index 4980e6682b..150ca37810 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -68766f837491cb89c2103f2627eb9e23ab326a68 \ No newline at end of file +116bed5af664899a73b46dca528ac0c021fc50c3 \ No newline at end of file From 4712c21287cf757babe805bb1feaeb6be50a7564 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 9 May 2014 20:51:17 +0000 Subject: [PATCH 55/71] When cross-compiling with the MSVC makefile, make sure the correct library path is used. Also, keep track of the required compilation options separately. FossilOrigin-Name: c3dce2e7390eec3a337be1b99f80ad5f721cc647 --- Makefile.msc | 80 ++++++++++++++++++++++++++++++++++----------------- manifest | 14 ++++----- manifest.uuid | 2 +- 3 files changed, 62 insertions(+), 34 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index 2b970e3097..a2d623e653 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1,6 +1,9 @@ # # nmake Makefile for SQLite # +############################################################################### +############################## START OF OPTIONS ############################### +############################################################################### # The toplevel directory of the source tree. This is the directory # that contains this "Makefile.msc". @@ -116,6 +119,19 @@ DEBUG = 0 OPTIMIZATIONS = 2 !ENDIF +# These are the "standard" SQLite compilation options used when compiling for +# the Windows platform. +# +!IFNDEF OPT_FEATURE_FLAGS +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1 +!ENDIF + +############################################################################### +############################### END OF OPTIONS ################################ +############################################################################### + # Check for the predefined command macro CC. This should point to the compiler # binary for the target platform. If it is not defined, simply define it to # the legacy default value 'cl.exe'. @@ -140,6 +156,15 @@ LD = link.exe RC = rc.exe !ENDIF +# Check for the MSVC runtime library path macro. Othertise, this value will +# default to the 'lib' directory underneath the MSVC installation directory. +# +!IFNDEF CRTLIBPATH +CRTLIBPATH = $(VCINSTALLDIR)\lib +!ENDIF + +CRTLIBPATH = $(CRTLIBPATH:\\=\) + # Check for the command macro NCC. This should point to the compiler binary # for the platform the compilation process is taking place on. If it is not # defined, simply define it to have the same value as the CC macro. When @@ -168,8 +193,8 @@ NCC = $(NCC:\\=\) NCC = $(CC) !ENDIF -# Check for the MSVC runtime library path macro. Othertise, this -# value will default to the 'lib' directory underneath the MSVC +# Check for the MSVC native runtime library path macro. Othertise, +# this value will default to the 'lib' directory underneath the MSVC # installation directory. # !IFNDEF NCRTLIBPATH @@ -306,7 +331,6 @@ TCC = $(TCC) -DSQLITE_ENABLE_IOTRACE RCC = $(RCC) -DSQLITE_ENABLE_IOTRACE !ENDIF -# # Prevent warnings about "insecure" MSVC runtime library functions # being used. # @@ -314,28 +338,24 @@ TCC = $(TCC) -D_CRT_SECURE_NO_DEPRECATE -D_CRT_SECURE_NO_WARNINGS BCC = $(BCC) -D_CRT_SECURE_NO_DEPRECATE -D_CRT_SECURE_NO_WARNINGS RCC = $(RCC) -D_CRT_SECURE_NO_DEPRECATE -D_CRT_SECURE_NO_WARNINGS -# # Prevent warnings about "deprecated" POSIX functions being used. # TCC = $(TCC) -D_CRT_NONSTDC_NO_DEPRECATE -D_CRT_NONSTDC_NO_WARNINGS BCC = $(BCC) -D_CRT_NONSTDC_NO_DEPRECATE -D_CRT_NONSTDC_NO_WARNINGS RCC = $(RCC) -D_CRT_NONSTDC_NO_DEPRECATE -D_CRT_NONSTDC_NO_WARNINGS -# # Use the SQLite debugging heap subsystem? # !IF $(MEMDEBUG)!=0 TCC = $(TCC) -DSQLITE_MEMDEBUG=1 RCC = $(RCC) -DSQLITE_MEMDEBUG=1 -# # Use native Win32 heap subsystem instead of malloc/free? # !ELSEIF $(WIN32HEAP)!=0 TCC = $(TCC) -DSQLITE_WIN32_MALLOC=1 RCC = $(RCC) -DSQLITE_WIN32_MALLOC=1 -# # Validate the heap on every call into the native Win32 heap subsystem? # !IF $(DEBUG)>2 @@ -430,25 +450,25 @@ RCC = $(RCC) -DSQLITE_TEMP_STORE=1 # The same set of OMIT and ENABLE flags should be passed to the # LEMON parser generator and the mkkeywordhash tool as well. -# BEGIN standard options -OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1 -OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1 -OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1 -# END standard options +# These are the required SQLite compilation options used when compiling for +# the Windows platform. +# +REQ_FEATURE_FLAGS = $(REQ_FEATURE_FLAGS) -DSQLITE_MAX_TRIGGER_DEPTH=100 -# BEGIN required Windows option -OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_MAX_TRIGGER_DEPTH=100 -# END required Windows option +# Add the required and optional SQLite compilation options into the command +# lines used to invoke the MSVC code and resource compilers. +# +TCC = $(TCC) $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) +RCC = $(RCC) $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) -TCC = $(TCC) $(OPT_FEATURE_FLAGS) -RCC = $(RCC) $(OPT_FEATURE_FLAGS) - -# Add in any optional parameters specified on the make commane line -# ie. make "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1". +# Add in any optional parameters specified on the commane line, e.g. +# nmake /f Makefile.msc all "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1" +# TCC = $(TCC) $(OPTS) RCC = $(RCC) $(OPTS) # If compiling for debugging, add some defines. +# !IF $(DEBUG)>0 TCC = $(TCC) -D_DEBUG BCC = $(BCC) -D_DEBUG @@ -457,6 +477,7 @@ RCC = $(RCC) -D_DEBUG # If optimizations are enabled or disabled (either implicitly or # explicitly), add the necessary flags. +# !IF $(DEBUG)>0 || $(OPTIMIZATIONS)==0 TCC = $(TCC) -Od BCC = $(BCC) -Od @@ -472,12 +493,14 @@ BCC = $(BCC) -O1 !ENDIF # If symbols are enabled (or compiling for debugging), enable PDBs. +# !IF $(DEBUG)>0 || $(SYMBOLS)!=0 TCC = $(TCC) -Zi BCC = $(BCC) -Zi !ENDIF # If ICU support is enabled, add the compiler options for it. +# !IF $(USE_ICU)!=0 TCC = $(TCC) -DSQLITE_ENABLE_ICU=1 RCC = $(RCC) -DSQLITE_ENABLE_ICU=1 @@ -489,6 +512,7 @@ RCC = $(RCC) -I$(ICUINCDIR) # Command line prefixes for compiling code, compiling resources, # linking, etc. +# LTCOMPILE = $(TCC) -Fo$@ LTRCOMPILE = $(RCC) -r LTLIB = lib.exe @@ -512,13 +536,13 @@ LTLINKOPTS = $(LTLINKOPTS) /APPCONTAINER !IF "$(VISUALSTUDIOVERSION)"=="12.0" !IFNDEF STORELIBPATH !IF "$(PLATFORM)"=="x86" -STORELIBPATH = $(NCRTLIBPATH)\store +STORELIBPATH = $(CRTLIBPATH)\store !ELSEIF "$(PLATFORM)"=="x64" -STORELIBPATH = $(NCRTLIBPATH)\store\amd64 +STORELIBPATH = $(CRTLIBPATH)\store\amd64 !ELSEIF "$(PLATFORM)"=="ARM" -STORELIBPATH = $(NCRTLIBPATH)\store\arm +STORELIBPATH = $(CRTLIBPATH)\store\arm !ELSE -STORELIBPATH = $(NCRTLIBPATH)\store +STORELIBPATH = $(CRTLIBPATH)\store !ENDIF !ENDIF STORELIBPATH = $(STORELIBPATH:\\=\) @@ -527,23 +551,27 @@ LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(STORELIBPATH)" !ENDIF # If either debugging or symbols are enabled, enable PDBs. +# !IF $(DEBUG)>0 || $(SYMBOLS)!=0 LDFLAGS = /DEBUG !ENDIF # Start with the Tcl related linker options. +# !IF $(NO_TCL)==0 LTLIBPATHS = /LIBPATH:$(TCLLIBDIR) LTLIBS = $(LIBTCL) !ENDIF # If ICU support is enabled, add the linker options for it. +# !IF $(USE_ICU)!=0 LTLIBPATHS = $(LTLIBPATHS) /LIBPATH:$(ICULIBDIR) LTLIBS = $(LTLIBS) $(LIBICU) !ENDIF # nawk compatible awk. +# !IFNDEF NAWK NAWK = gawk.exe !ENDIF @@ -1222,7 +1250,7 @@ parse.h: parse.c parse.c: $(TOP)\src\parse.y lemon.exe $(TOP)\addopcodes.awk del /Q parse.y parse.h parse.h.temp copy $(TOP)\src\parse.y . - .\lemon.exe $(OPT_FEATURE_FLAGS) $(OPTS) parse.y + .\lemon.exe $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(OPTS) parse.y move parse.h parse.h.temp $(NAWK) -f $(TOP)\addopcodes.awk parse.h.temp > parse.h @@ -1230,7 +1258,7 @@ sqlite3.h: $(TOP)\src\sqlite.h.in $(TOP)\manifest.uuid $(TOP)\VERSION $(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP:\=/) > sqlite3.h mkkeywordhash.exe: $(TOP)\tool\mkkeywordhash.c - $(BCC) -Fe$@ $(OPT_FEATURE_FLAGS) $(OPTS) $(TOP)\tool\mkkeywordhash.c /link $(NLTLINKOPTS) $(NLTLIBPATHS) + $(BCC) -Fe$@ $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(OPTS) $(TOP)\tool\mkkeywordhash.c /link $(NLTLINKOPTS) $(NLTLIBPATHS) keywordhash.h: $(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe .\mkkeywordhash.exe > keywordhash.h diff --git a/manifest b/manifest index bec4a2d4fe..491f33f044 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Include\ssqlite3rtree.h\sin\sthe\stsrc/\spile\sof\ssource\sfiles\sduring\ntarget_source\sin\sthe\smain.mk\smakefile. -D 2014-05-07T21:16:56.524 +C When\scross-compiling\swith\sthe\sMSVC\smakefile,\smake\ssure\sthe\scorrect\slibrary\spath\sis\sused.\s\sAlso,\skeep\strack\sof\sthe\srequired\scompilation\soptions\sseparately. +D 2014-05-09T20:51:17.136 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc 7e6c495d9a145054a09f518781916c7503f7a8e9 +F Makefile.msc f4b4d99d61cfe4ec508600dccd1a0ab2dc8a8b9b F Makefile.vxworks 034289efa9d591b04b1a73598623119c306cbba0 F README.md 64f270c43c38c46de749e419c22f0ae2f4499fe8 F VERSION 9f823c026c6a32fc5f84d212a8aae0a221dba45c @@ -1170,7 +1170,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix a94fb9b1b1ef06efc2898975cdfcfa9643731f5e -P 68766f837491cb89c2103f2627eb9e23ab326a68 -R 1b75d6e5e9a19ccc14f8e59e21c749f6 -U drh -Z 3528e73d8f87faaf66e6658213b6d573 +P 116bed5af664899a73b46dca528ac0c021fc50c3 +R 88c72e40fa185e1eb8f17237cccbb934 +U mistachkin +Z b5868dcc5cdf0f5ef12e0a431c431ca8 diff --git a/manifest.uuid b/manifest.uuid index 150ca37810..a49c4f2f02 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -116bed5af664899a73b46dca528ac0c021fc50c3 \ No newline at end of file +c3dce2e7390eec3a337be1b99f80ad5f721cc647 \ No newline at end of file From d9b3c549b3fe3413aa8f37e175007895d2b00e2c Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 9 May 2014 23:31:55 +0000 Subject: [PATCH 56/71] Further MSVC makefile changes to support Windows Phone 8.1. FossilOrigin-Name: a9c81815e1b7a9c05da61f75edac45cb1a954135 --- Makefile.msc | 39 +++++++++++++++++++++++++++++++++++++++ manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 46 insertions(+), 7 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index a2d623e653..218abe1f8c 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -16,6 +16,12 @@ TOP = . USE_AMALGAMATION = 1 !ENDIF +# Set this non-0 to use the libraries necessary for Windows Phone 8.1. +# +!IFNDEF USE_WP81_OPTS +USE_WP81_OPTS = 0 +!ENDIF + # Set this non-0 to split the SQLite amalgamation file into chunks to # be used for debugging with Visual Studio. # @@ -132,6 +138,12 @@ OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1 ############################### END OF OPTIONS ################################ ############################################################################### +# This assumes that MSVC is always installed in 32-bit Program Files directory +# and sets the variable for use in locating other 32-bit installs accordingly. +# +PROGRAMFILES_X86 = $(VCINSTALLDIR)\..\.. +PROGRAMFILES_X86 = $(PROGRAMFILES_X86:\\=\) + # Check for the predefined command macro CC. This should point to the compiler # binary for the target platform. If it is not defined, simply define it to # the legacy default value 'cl.exe'. @@ -550,6 +562,33 @@ LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(STORELIBPATH)" !ENDIF !ENDIF +# When compiling for Windows Phone 8.1, an extra library path is +# required. +# +!IF $(USE_WP81_OPTS)!=0 +!IFNDEF WP81LIBPATH +!IF "$(PLATFORM)"=="x86" +WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\x86 +!ELSEIF "$(PLATFORM)"=="ARM" +WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\ARM +!ELSE +WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\x86 +!ENDIF +!ENDIF +!ENDIF + +# When compiling for Windows Phone 8.1, some extra linker options +# are also required. +# +!IF $(USE_WP81_OPTS)!=0 +!IFDEF WP81LIBPATH +LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(WP81LIBPATH)" +!ENDIF +LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE +LTLINKOPTS = $(LTLINKOPTS) WindowsPhoneCore.lib RuntimeObject.lib PhoneAppModelHost.lib +LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:kernel32.lib /NODEFAULTLIB:ole32.lib +!ENDIF + # If either debugging or symbols are enabled, enable PDBs. # !IF $(DEBUG)>0 || $(SYMBOLS)!=0 diff --git a/manifest b/manifest index 7f6738b740..3eb473a136 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Merge\supdates\sfrom\strunk. -D 2014-05-09T20:54:07.623 +C Further\sMSVC\smakefile\schanges\sto\ssupport\sWindows\sPhone\s8.1. +D 2014-05-09T23:31:55.574 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc f4b4d99d61cfe4ec508600dccd1a0ab2dc8a8b9b +F Makefile.msc 26164f8288a7380b05587534ca3974c3837e5bd0 F Makefile.vxworks 034289efa9d591b04b1a73598623119c306cbba0 F README.md 64f270c43c38c46de749e419c22f0ae2f4499fe8 F VERSION 9f823c026c6a32fc5f84d212a8aae0a221dba45c @@ -1170,7 +1170,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix a94fb9b1b1ef06efc2898975cdfcfa9643731f5e -P 69698ae90c415cb32d2c144725853483da65f0ff c3dce2e7390eec3a337be1b99f80ad5f721cc647 -R 5a8ec6fb329eaf82ce72337357145445 +P f4fea7bb8a4b118bdceff400a6c49c6291c0d58e +R cc5d4172ed1063b929dacadc942922f5 U mistachkin -Z 30139e1f65fbd864d0f9d58c85a7550e +Z e9a82f6e684a32dc44406c7db15e32b2 diff --git a/manifest.uuid b/manifest.uuid index 8c9091248c..36d319c95e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f4fea7bb8a4b118bdceff400a6c49c6291c0d58e \ No newline at end of file +a9c81815e1b7a9c05da61f75edac45cb1a954135 \ No newline at end of file From 1273614191d1ab5edc0ef4d6556a0b06e8fa3b18 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sat, 10 May 2014 17:28:45 +0000 Subject: [PATCH 57/71] When using Visual Studio 2013, add the appropriate MaxPlatformVersion attribute to the VSIX SDK manifest. FossilOrigin-Name: 0a4f59676bd0ab33b2c86c9a35a2ebbdbaf09ee7 --- manifest | 14 +++++++------- manifest.uuid | 2 +- tool/mkvsix.tcl | 37 ++++++++++++++++++++++++++++++++++++- tool/win/sqlite.vsix | Bin 32820 -> 32825 bytes 4 files changed, 44 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 491f33f044..00a1115ee6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\scross-compiling\swith\sthe\sMSVC\smakefile,\smake\ssure\sthe\scorrect\slibrary\spath\sis\sused.\s\sAlso,\skeep\strack\sof\sthe\srequired\scompilation\soptions\sseparately. -D 2014-05-09T20:51:17.136 +C When\susing\sVisual\sStudio\s2013,\sadd\sthe\sappropriate\sMaxPlatformVersion\sattribute\sto\sthe\sVSIX\sSDK\smanifest. +D 2014-05-10T17:28:45.158 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1142,7 +1142,7 @@ F tool/mksqlite3c-noext.tcl 1712d3d71256ca1f297046619c89e77a4d7c8f6d F tool/mksqlite3c.tcl ba274df71f5e6534b0a913c7c48eabfcbd0934b6 F tool/mksqlite3h.tcl ba24038056f51fde07c0079c41885ab85e2cff12 F tool/mksqlite3internalh.tcl b6514145a7d5321b47e64e19b8116cc44f973eb1 -F tool/mkvsix.tcl 924dcdecda86969686833301c08f84cca2600d94 +F tool/mkvsix.tcl 64805dc0677009f0ba6b6e7199b2f7613cac3ebe F tool/offsets.c fe4262fdfa378e8f5499a42136d17bf3b98f6091 F tool/omittest.tcl 34d7ac01fe4fd18e3637f64abe12c40eca0f6b97 F tool/opcodeDoc.awk b3a2a3d5d3075b8bd90b7afe24283efdd586659c @@ -1169,8 +1169,8 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 -F tool/win/sqlite.vsix a94fb9b1b1ef06efc2898975cdfcfa9643731f5e -P 116bed5af664899a73b46dca528ac0c021fc50c3 -R 88c72e40fa185e1eb8f17237cccbb934 +F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f +P c3dce2e7390eec3a337be1b99f80ad5f721cc647 +R 7cb8e1b3df65a486b753c19d33da3975 U mistachkin -Z b5868dcc5cdf0f5ef12e0a431c431ca8 +Z f2874f77f50d8a6763121ff9dede6833 diff --git a/manifest.uuid b/manifest.uuid index a49c4f2f02..ff047bf0ce 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c3dce2e7390eec3a337be1b99f80ad5f721cc647 \ No newline at end of file +0a4f59676bd0ab33b2c86c9a35a2ebbdbaf09ee7 \ No newline at end of file diff --git a/tool/mkvsix.tcl b/tool/mkvsix.tcl index 712cf32278..378435aab4 100644 --- a/tool/mkvsix.tcl +++ b/tool/mkvsix.tcl @@ -189,6 +189,33 @@ proc getMinVsVersionXmlChunk { vsVersion } { } } +proc getMaxPlatformVersionXmlChunk { packageFlavor vsVersion } { + # + # NOTE: Only Visual Studio 2013 supports this SDK manifest attribute. + # + if {![string equal $vsVersion 2013]} then { + return "" + } + + switch -exact $packageFlavor { + WinRT { + return [appendArgs \ + "\r\n " {MaxPlatformVersion="8.0"}] + } + WinRT81 { + return [appendArgs \ + "\r\n " {MaxPlatformVersion="8.1"}] + } + WP80 { + return [appendArgs \ + "\r\n " {MaxPlatformVersion="8.0"}] + } + default { + return "" + } + } +} + proc getExtraFileListXmlChunk { packageFlavor vsVersion } { # # NOTE: Windows Phone 8.0 does not require any extra attributes in its VSIX @@ -321,7 +348,7 @@ if {[string length $vsVersion] == 0} then { fail "invalid Visual Studio version" } -if {$vsVersion ne "2012" && $vsVersion ne "2013"} then { +if {![string equal $vsVersion 2012] && ![string equal $vsVersion 2013]} then { fail [appendArgs \ "unsupported Visual Studio version, must be one of: " \ [list 2012 2013]] @@ -349,6 +376,8 @@ if {[string equal $packageFlavor WinRT]} then { set targetPlatformIdentifier Windows set targetPlatformVersion v8.0 set minVsVersion [getMinVsVersionXmlChunk $vsVersion] + set maxPlatformVersion \ + [getMaxPlatformVersionXmlChunk $packageFlavor $vsVersion] set extraSdkPath "" set extraFileListAttributes \ [getExtraFileListXmlChunk $packageFlavor $vsVersion] @@ -363,6 +392,8 @@ if {[string equal $packageFlavor WinRT]} then { set targetPlatformIdentifier Windows set targetPlatformVersion v8.1 set minVsVersion [getMinVsVersionXmlChunk $vsVersion] + set maxPlatformVersion \ + [getMaxPlatformVersionXmlChunk $packageFlavor $vsVersion] set extraSdkPath "" set extraFileListAttributes \ [getExtraFileListXmlChunk $packageFlavor $vsVersion] @@ -372,6 +403,8 @@ if {[string equal $packageFlavor WinRT]} then { set targetPlatformIdentifier "Windows Phone" set targetPlatformVersion v8.0 set minVsVersion [getMinVsVersionXmlChunk $vsVersion] + set maxPlatformVersion \ + [getMaxPlatformVersionXmlChunk $packageFlavor $vsVersion] set extraSdkPath "\\..\\$targetPlatformIdentifier" set extraFileListAttributes \ [getExtraFileListXmlChunk $packageFlavor $vsVersion] @@ -381,6 +414,8 @@ if {[string equal $packageFlavor WinRT]} then { set targetPlatformIdentifier Windows set targetPlatformVersion v8.0 set minVsVersion [getMinVsVersionXmlChunk $vsVersion] + set maxPlatformVersion \ + [getMaxPlatformVersionXmlChunk $packageFlavor $vsVersion] set extraSdkPath "" set extraFileListAttributes \ [getExtraFileListXmlChunk $packageFlavor $vsVersion] diff --git a/tool/win/sqlite.vsix b/tool/win/sqlite.vsix index de4d0a1f96ad3b53b899a7bcfbba53f23a1d4990..1450011266bc556131d33a625ca17688ffd19c00 100644 GIT binary patch delta 326 zcmV-M0lEIPfC9OI0OuO<`_nW@U49E_iKh zRgg_f!$1&*?}h${W$D#KZ=zjGK?{PFg3yb4Sd&RJvR}^5nuz4TyCtpFxxK?PJTEg} z^wx+^porcnc~P880j}~jx}B1Vo3l$P7B|z$e1!%cF!FRF^4ww!e@1^jY70uvzUxyQ z)Y=o=yT&WoGKZ_O?7FUq9|jo;PrEYTSajRC9UBfg>V9K1 zH=gWi2|bfOHe50jz9oh@jFP7mLiWaWGK|5%;Vvq9N90NVqo7_=)ngw0|b-5e;BhRdp`sXuTrW+6g`>a-~a#sOp|ecFb1R> YRY!Delc#??0S=Soe@X@vd;kCd0J=VsfdBvi delta 312 zcmV-80muHifC99D0OuO<`_nW@U49E_iKh zRgh0h!$1(l?}dJcWy#e>>ZK#bm%`XWEq3S8|QbemE~Y0oaTn%_((vn3jMK$hu5rMX28f2R6=tSpo|+cp1k z&?E6gzKOLgD~sljMljq5(yIX^5M~`ES(~)C%Z4o z>h=0f?;lVcLX-FbvkrS(1PuUykwjgh(dOF#001|Wc7HGiqdpA6kXn?#Rlj(m- K1`m7y0002+?Tu6b From 0f710546e71337dd680201d61566c1f329abb8a5 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 12 May 2014 15:37:03 +0000 Subject: [PATCH 58/71] In mutex_w32.c, make inclusion of the Windows header file dependent on SQLITE_OS_WIN, not SQLITE_MUTEX_W32. FossilOrigin-Name: 93ffbcc0b1d207950851dbb16a8c101afb949d5f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/mutex_w32.c | 12 +++++++----- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 00a1115ee6..735ec8ef6c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\susing\sVisual\sStudio\s2013,\sadd\sthe\sappropriate\sMaxPlatformVersion\sattribute\sto\sthe\sVSIX\sSDK\smanifest. -D 2014-05-10T17:28:45.158 +C In\smutex_w32.c,\smake\sinclusion\sof\sthe\sWindows\sheader\sfile\sdependent\son\sSQLITE_OS_WIN,\snot\sSQLITE_MUTEX_W32. +D 2014-05-12T15:37:03.806 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -199,7 +199,7 @@ F src/mutex.c d3b66a569368015e0fcb1ac15f81c119f504d3bc F src/mutex.h 5bc526e19dccc412b7ff04642f6fdad3fdfdabea F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553 F src/mutex_unix.c c3a4e00f96ba068a8dbef34084465979aaf369cc -F src/mutex_w32.c ab08c0fc54b71979370ca7c8f42fc64a9f211ebb +F src/mutex_w32.c 9bcab6e699fdb20eae784237c36b299830fb3316 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 60d419395e32a8029fa380a80a3da2e9030f635e @@ -1170,7 +1170,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c3dce2e7390eec3a337be1b99f80ad5f721cc647 -R 7cb8e1b3df65a486b753c19d33da3975 +P 0a4f59676bd0ab33b2c86c9a35a2ebbdbaf09ee7 +R 4aed61add01807e8cd6ee17e1b6f3cd9 U mistachkin -Z f2874f77f50d8a6763121ff9dede6833 +Z 68de491dda45b3ab2f8417b0d5d6b640 diff --git a/manifest.uuid b/manifest.uuid index ff047bf0ce..81a48e539c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0a4f59676bd0ab33b2c86c9a35a2ebbdbaf09ee7 \ No newline at end of file +93ffbcc0b1d207950851dbb16a8c101afb949d5f \ No newline at end of file diff --git a/src/mutex_w32.c b/src/mutex_w32.c index 2eb852413e..f43a152389 100644 --- a/src/mutex_w32.c +++ b/src/mutex_w32.c @@ -13,17 +13,19 @@ */ #include "sqliteInt.h" +#if SQLITE_OS_WIN +/* +** Include the header file for the Windows VFS. +*/ +#include "os_win.h" +#endif + /* ** The code in this file is only used if we are compiling multithreaded ** on a win32 system. */ #ifdef SQLITE_MUTEX_W32 -/* -** Include the header file for the Windows VFS. -*/ -#include "os_win.h" - /* ** Each recursive mutex is an instance of the following structure. */ From 6bb9889ef023ec853e1943c45940d8929f98fa0d Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 12 May 2014 20:04:48 +0000 Subject: [PATCH 59/71] Experimental code to prevent FTS indexes from growing indefinitely as the table is updated. FossilOrigin-Name: b3b505a4dd0c679437a4272109f1188175088cd1 --- ext/fts3/fts3Int.h | 2 +- ext/fts3/fts3_write.c | 161 +++++++++++++++++++++++++++++++---- manifest | 20 +++-- manifest.uuid | 2 +- test/fts4growth.test | 191 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 352 insertions(+), 24 deletions(-) create mode 100644 test/fts4growth.test diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index a5bb2f0041..171a75d124 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -216,7 +216,7 @@ struct Fts3Table { /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. */ - sqlite3_stmt *aStmt[37]; + sqlite3_stmt *aStmt[40]; char *zReadExprlist; char *zWriteExprlist; diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 90d1609226..b54d61390a 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -193,6 +193,7 @@ struct SegmentWriter { int nSize; /* Size of allocation at aData */ int nData; /* Bytes of data in aData */ char *aData; /* Pointer to block from malloc() */ + i64 nLeafData; /* Number of bytes of leaf data written */ }; /* @@ -268,6 +269,10 @@ struct SegmentNode { #define SQL_SELECT_INDEXES 35 #define SQL_SELECT_MXLEVEL 36 +#define SQL_SELECT_LEVEL_RANGE2 37 +#define SQL_UPDATE_LEVEL_IDX 38 +#define SQL_UPDATE_LEVEL 39 + /* ** This function is used to obtain an SQLite prepared statement handle ** for the statement identified by the second argument. If successful, @@ -369,7 +374,17 @@ static int fts3SqlStmt( /* SQL_SELECT_MXLEVEL ** Return the largest relative level in the FTS index or indexes. */ -/* 36 */ "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'" +/* 36 */ "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'", + + /* Return segments in order from oldest to newest.*/ +/* 37 */ "SELECT level, idx, end_block " + "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?" + "ORDER BY level DESC, idx ASC", + + /* Update statements used while promoting segments */ +/* 38 */ "UPDATE %Q.'%q_segdir' SET level=-1,idx=? WHERE level=? AND idx=?", +/* 39 */ "UPDATE %Q.'%q_segdir' SET level=? WHERE level=-1" + }; int rc = SQLITE_OK; sqlite3_stmt *pStmt; @@ -1910,6 +1925,7 @@ static int fts3WriteSegdir( sqlite3_int64 iStartBlock, /* Value for "start_block" field */ sqlite3_int64 iLeafEndBlock, /* Value for "leaves_end_block" field */ sqlite3_int64 iEndBlock, /* Value for "end_block" field */ + sqlite3_int64 nLeafData, /* Bytes of leaf data in segment */ char *zRoot, /* Blob value for "root" field */ int nRoot /* Number of bytes in buffer zRoot */ ){ @@ -1920,7 +1936,13 @@ static int fts3WriteSegdir( sqlite3_bind_int(pStmt, 2, iIdx); sqlite3_bind_int64(pStmt, 3, iStartBlock); sqlite3_bind_int64(pStmt, 4, iLeafEndBlock); - sqlite3_bind_int64(pStmt, 5, iEndBlock); + if( nLeafData==0 ){ + sqlite3_bind_int64(pStmt, 5, iEndBlock); + }else{ + char *zEnd = sqlite3_mprintf("%lld %lld", iEndBlock, nLeafData); + if( !zEnd ) return SQLITE_NOMEM; + sqlite3_bind_text(pStmt, 5, zEnd, -1, sqlite3_free); + } sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC); sqlite3_step(pStmt); rc = sqlite3_reset(pStmt); @@ -2246,6 +2268,9 @@ static int fts3SegWriterAdd( nDoclist; /* Doclist data */ } + /* Increase the total number of bytes written to account for the new entry. */ + pWriter->nLeafData += nReq; + /* If the buffer currently allocated is too small for this entry, realloc ** the buffer to make it large enough. */ @@ -2317,13 +2342,13 @@ static int fts3SegWriterFlush( pWriter->iFirst, pWriter->iFree, &iLast, &zRoot, &nRoot); } if( rc==SQLITE_OK ){ - rc = fts3WriteSegdir( - p, iLevel, iIdx, pWriter->iFirst, iLastLeaf, iLast, zRoot, nRoot); + rc = fts3WriteSegdir(p, iLevel, iIdx, + pWriter->iFirst, iLastLeaf, iLast, pWriter->nLeafData, zRoot, nRoot); } }else{ /* The entire tree fits on the root node. Write it to the segdir table. */ - rc = fts3WriteSegdir( - p, iLevel, iIdx, 0, 0, 0, pWriter->aData, pWriter->nData); + rc = fts3WriteSegdir(p, iLevel, iIdx, + 0, 0, 0, pWriter->nLeafData, pWriter->aData, pWriter->nData); } p->nLeafAdd++; return rc; @@ -2942,6 +2967,106 @@ void sqlite3Fts3SegReaderFinish( } } +/* +** Decode the "end_block" field, selected by column iCol of the SELECT +** statement passed as the first argument. +*/ +static void fts3ReadEndBlockField( + sqlite3_stmt *pStmt, + int iCol, + i64 *piEndBlock, + i64 *pnByte +){ + const unsigned char *zText = sqlite3_column_text(pStmt, iCol); + if( zText ){ + int i; + i64 iVal = 0; + for(i=0; zText[i]>='0' && zText[i]<='9'; i++){ + iVal = iVal*10 + (zText[i] - '0'); + } + *piEndBlock = iVal; + while( zText[i]==' ' ) i++; + iVal = 0; + for(/* no-op */; zText[i]>='0' && zText[i]<='9'; i++){ + iVal = iVal*10 + (zText[i] - '0'); + } + *pnByte = iVal; + } +} + + +/* +** A segment of size nByte bytes has just been written to absolute level +** iAbsLevel. Promote any segments that should be promoted as a result. +*/ +static int fts3PromoteSegments( + Fts3Table *p, /* FTS table handle */ + int iAbsLevel, /* Absolute level just updated */ + sqlite3_int64 nByte /* Size of new segment at iAbsLevel */ +){ + int rc = SQLITE_OK; + sqlite3_stmt *pRange; + + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE2, &pRange, 0); + + if( rc==SQLITE_OK ){ + int bOk = 1; + int iLast = (iAbsLevel/FTS3_SEGDIR_MAXLEVEL + 1) * FTS3_SEGDIR_MAXLEVEL - 1; + + sqlite3_bind_int(pRange, 1, iAbsLevel+1); + sqlite3_bind_int(pRange, 2, iLast); + while( SQLITE_ROW==sqlite3_step(pRange) ){ + i64 nSize, dummy; + fts3ReadEndBlockField(pRange, 2, &dummy, &nSize); + if( nSize>nByte ){ + bOk = 0; + break; + } + } + rc = sqlite3_reset(pRange); + + if( bOk ){ + int iIdx = 0; + sqlite3_stmt *pUpdate1; + sqlite3_stmt *pUpdate2; + + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL_IDX, &pUpdate1, 0); + } + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL, &pUpdate2, 0); + } + + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pRange, 1, iAbsLevel); + while( SQLITE_ROW==sqlite3_step(pRange) ){ + sqlite3_bind_int(pUpdate1, 1, iIdx++); + sqlite3_bind_int(pUpdate1, 2, sqlite3_column_int(pRange, 0)); + sqlite3_bind_int(pUpdate1, 3, sqlite3_column_int(pRange, 1)); + sqlite3_step(pUpdate1); + rc = sqlite3_reset(pUpdate1); + if( rc!=SQLITE_OK ){ + sqlite3_reset(pRange); + break; + } + } + } + if( rc==SQLITE_OK ){ + rc = sqlite3_reset(pRange); + } + + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pUpdate2, 1, iAbsLevel); + sqlite3_step(pUpdate2); + rc = sqlite3_reset(pUpdate2); + } + } + } + + + return rc; +} + /* ** Merge all level iLevel segments in the database into a single ** iLevel+1 segment. Or, if iLevel<0, merge all segments into a @@ -3026,6 +3151,9 @@ static int fts3SegmentMerge( if( rc!=SQLITE_OK ) goto finished; } rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx); + if( rc==SQLITE_OK ){ + rc = fts3PromoteSegments(p, iNewLevel, pWriter->nLeafData); + } finished: fts3SegWriterFree(pWriter); @@ -3035,7 +3163,7 @@ static int fts3SegmentMerge( /* -** Flush the contents of pendingTerms to level 0 segments. +** Flush the contents of pendingTerms to level 0 segments. */ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ int rc = SQLITE_OK; @@ -3426,6 +3554,7 @@ struct IncrmergeWriter { int iIdx; /* Index of *output* segment in iAbsLevel+1 */ sqlite3_int64 iStart; /* Block number of first allocated block */ sqlite3_int64 iEnd; /* Block number of last allocated block */ + sqlite3_int64 nLeafData; /* Bytes of leaf page data so far */ NodeWriter aNodeWriter[FTS_MAX_APPENDABLE_HEIGHT]; }; @@ -3764,8 +3893,8 @@ static int fts3IncrmergeAppend( nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; } + pWriter->nLeafData += nSpace; blobGrowBuffer(&pLeaf->block, pLeaf->block.n + nSpace, &rc); - if( rc==SQLITE_OK ){ if( pLeaf->block.n==0 ){ pLeaf->block.n = 1; @@ -3864,6 +3993,7 @@ static void fts3IncrmergeRelease( pWriter->iStart, /* start_block */ pWriter->aNodeWriter[0].iBlock, /* leaves_end_block */ pWriter->iEnd, /* end_block */ + pWriter->nLeafData, /* end_block */ pRoot->block.a, pRoot->block.n /* root */ ); } @@ -3965,7 +4095,7 @@ static int fts3IncrmergeLoad( if( sqlite3_step(pSelect)==SQLITE_ROW ){ iStart = sqlite3_column_int64(pSelect, 1); iLeafEnd = sqlite3_column_int64(pSelect, 2); - iEnd = sqlite3_column_int64(pSelect, 3); + fts3ReadEndBlockField(pSelect, 3, &iEnd, &pWriter->nLeafData); nRoot = sqlite3_column_bytes(pSelect, 4); aRoot = sqlite3_column_blob(pSelect, 4); }else{ @@ -4566,11 +4696,11 @@ static int fts3IncrmergeHintPop(Blob *pHint, i64 *piAbsLevel, int *pnInput){ /* ** Attempt an incremental merge that writes nMerge leaf blocks. ** -** Incremental merges happen nMin segments at a time. The two -** segments to be merged are the nMin oldest segments (the ones with -** the smallest indexes) in the highest level that contains at least -** nMin segments. Multiple merges might occur in an attempt to write the -** quota of nMerge leaf blocks. +** Incremental merges happen nMin segments at a time. The segments +** to be merged are the nMin oldest segments (the ones with the smallest +** values for the _segdir.idx field) in the highest level that contains +** at least nMin segments. Multiple merges might occur in an attempt to +** write the quota of nMerge leaf blocks. */ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ int rc; /* Return code */ @@ -4687,6 +4817,9 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ } fts3IncrmergeRelease(p, pWriter, &rc); + if( nSeg==0 ){ + fts3PromoteSegments(p, iAbsLevel+1, pWriter->nLeafData); + } } sqlite3Fts3SegReaderFinish(pCsr); diff --git a/manifest b/manifest index 00a1115ee6..1ff6b237e6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\susing\sVisual\sStudio\s2013,\sadd\sthe\sappropriate\sMaxPlatformVersion\sattribute\sto\sthe\sVSIX\sSDK\smanifest. -D 2014-05-10T17:28:45.158 +C Experimental\scode\sto\sprevent\sFTS\sindexes\sfrom\sgrowing\sindefinitely\sas\sthe\stable\sis\supdated. +D 2014-05-12T20:04:48.571 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -80,7 +80,7 @@ F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d F ext/fts3/fts3.c 41b1920b9a8657963f09cb93b208c2671c5568db F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h bdeb9015405e8facffb8fc7e09174521a2a780f4 +F ext/fts3/fts3Int.h b250b154764b86be4ee540f8c233a59858e88f1d F ext/fts3/fts3_aux.c 5c211e17a64885faeb16b9ba7772f9d5445c2365 F ext/fts3/fts3_expr.c 2ac35bda474f00c14c19608e49a02c8c7ceb9970 F ext/fts3/fts3_hash.c 29b986e43f4e9dd40110eafa377dc0d63c422c60 @@ -96,7 +96,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_unicode.c 92391b4b4fb043564c6539ea9b8661e3bcba47b9 F ext/fts3/fts3_unicode2.c 0113d3acf13429e6dc38e0647d1bc71211c31a4d -F ext/fts3/fts3_write.c 74c00329006c3ed6325ba4e5ab7c9b5fc99c8934 +F ext/fts3/fts3_write.c 5fd2aa9d1812387c6254304e20d9ac2b29e16700 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197 @@ -570,6 +570,7 @@ F test/fts4aa.test 0c3152322c7f0b548cc942ad763eaba0da87ccca F test/fts4check.test 66fa274cab2b615f2fb338b257713aba8fad88a8 F test/fts4content.test 2e7252557d6d24afa101d9ba1de710d6140e6d06 F test/fts4docid.test e33c383cfbdff0284685604d256f347a18fdbf01 +F test/fts4growth.test f7eac9fadfe67765c4a0d6202c85f7272766fb9e F test/fts4incr.test 361960ed3550e781f3f313e17e2182ef9cefc0e9 F test/fts4langid.test 24a6e41063b416bbdf371ff6b4476fa41c194aa7 F test/fts4merge.test c424309743fdd203f8e56a1f1cd7872cd66cc0ee @@ -1170,7 +1171,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c3dce2e7390eec3a337be1b99f80ad5f721cc647 -R 7cb8e1b3df65a486b753c19d33da3975 -U mistachkin -Z f2874f77f50d8a6763121ff9dede6833 +P 0a4f59676bd0ab33b2c86c9a35a2ebbdbaf09ee7 +R de45a14af3a90f6b390273a9a2d8cfa9 +T *branch * fts4-experimental +T *sym-fts4-experimental * +T -sym-trunk * +U dan +Z a03d85f9bfa278d813164b1e97a88ff7 diff --git a/manifest.uuid b/manifest.uuid index ff047bf0ce..a9facd1d8e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0a4f59676bd0ab33b2c86c9a35a2ebbdbaf09ee7 \ No newline at end of file +b3b505a4dd0c679437a4272109f1188175088cd1 \ No newline at end of file diff --git a/test/fts4growth.test b/test/fts4growth.test new file mode 100644 index 0000000000..ab2ad62142 --- /dev/null +++ b/test/fts4growth.test @@ -0,0 +1,191 @@ +# 2014 May 12 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS4 module. +# +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix fts4growth + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts3 { + finish_test + return +} + +source $testdir/genesis.tcl + +do_execsql_test 1.1 { CREATE VIRTUAL TABLE x1 USING fts3; } + +do_test 1.2 { + foreach L { + {"See here, young man," said Mulga Bill, "from Walgett to the sea,} + {From Conroy's Gap to Castlereagh, there's none can ride like me.} + {I'm good all round at everything as everybody knows,} + {Although I'm not the one to talk -- I hate a man that blows.} + } { + execsql { INSERT INTO x1 VALUES($L) } + } + execsql { SELECT end_block, length(root) FROM x1_segdir } +} {{0 114} 114 {0 118} 118 {0 95} 95 {0 115} 115} + +do_execsql_test 1.3 { + INSERT INTO x1(x1) VALUES('optimize'); + SELECT level, end_block, length(root) FROM x1_segdir; +} {0 {0 394} 394} + +do_test 1.4 { + foreach L { + {But riding is my special gift, my chiefest, sole delight;} + {Just ask a wild duck can it swim, a wildcat can it fight.} + {There's nothing clothed in hair or hide, or built of flesh or steel,} + {There's nothing walks or jumps, or runs, on axle, hoof, or wheel,} + {But what I'll sit, while hide will hold and girths and straps are tight:} + {I'll ride this here two-wheeled concern right straight away at sight."} + } { + execsql { INSERT INTO x1 VALUES($L) } + } + execsql { + INSERT INTO x1(x1) VALUES('merge=4,4'); + SELECT level, end_block, length(root) FROM x1_segdir; + } +} {0 {0 110} 110 0 {0 132} 132 0 {0 129} 129 1 {128 658} 2} + +do_execsql_test 1.5 { + SELECT length(block) FROM x1_segments; +} {658 {}} + +do_test 1.6 { + foreach L { + {'Twas Mulga Bill, from Eaglehawk, that sought his own abode,} + {That perched above Dead Man's Creek, beside the mountain road.} + {He turned the cycle down the hill and mounted for the fray,} + {But 'ere he'd gone a dozen yards it bolted clean away.} + {It left the track, and through the trees, just like a silver steak,} + {It whistled down the awful slope towards the Dead Man's Creek.} + {It shaved a stump by half an inch, it dodged a big white-box:} + {The very wallaroos in fright went scrambling up the rocks,} + {The wombats hiding in their caves dug deeper underground,} + {As Mulga Bill, as white as chalk, sat tight to every bound.} + {It struck a stone and gave a spring that cleared a fallen tree,} + {It raced beside a precipice as close as close could be;} + {And then as Mulga Bill let out one last despairing shriek} + {It made a leap of twenty feet into the Dead Man's Creek.} + } { + execsql { INSERT INTO x1 VALUES($L) } + } + execsql { + SELECT level, end_block, length(root) FROM x1_segdir; + } +} {1 {128 658} 2 1 {130 1377} 6 0 {0 117} 117} + +do_execsql_test 1.7 { + SELECT sum(length(block)) FROM x1_segments WHERE blockid IN (129, 130); +} {1377} + +#------------------------------------------------------------------------- +# +do_execsql_test 2.1 { + CREATE TABLE t1(docid, words); + CREATE VIRTUAL TABLE x2 USING fts4; +} +fts_kjv_genesis +do_test 2.2 { + foreach id [db eval {SELECT docid FROM t1}] { + execsql { + INSERT INTO x2(docid, content) SELECT $id, words FROM t1 WHERE docid=$id + } + } + foreach id [db eval {SELECT docid FROM t1}] { + execsql { + INSERT INTO x2(docid, content) SELECT NULL, words FROM t1 WHERE docid=$id + } + if {[db one {SELECT count(*) FROM x2_segdir WHERE level<2}]==2} break + } +} {} + +do_execsql_test 2.3 { + SELECT count(*) FROM x2_segdir WHERE level=2; + SELECT count(*) FROM x2_segdir WHERE level=3; +} {6 0} + +do_execsql_test 2.4 { + INSERT INTO x2(x2) VALUES('merge=4,4'); + SELECT count(*) FROM x2_segdir WHERE level=2; + SELECT count(*) FROM x2_segdir WHERE level=3; +} {6 1} + +do_execsql_test 2.5 { + SELECT end_block FROM x2_segdir WHERE level=3; + INSERT INTO x2(x2) VALUES('merge=4,4'); + SELECT end_block FROM x2_segdir WHERE level=3; + INSERT INTO x2(x2) VALUES('merge=4,4'); + SELECT end_block FROM x2_segdir WHERE level=3; +} {{3828 3430} {3828 10191} {3828 14109}} + +do_execsql_test 2.6 { + SELECT sum(length(block)) FROM x2_segdir, x2_segments WHERE + blockid BETWEEN start_block AND leaves_end_block + AND level=3 +} {14109} + +do_execsql_test 2.7 { + INSERT INTO x2(x2) VALUES('merge=1000,4'); + SELECT end_block FROM x2_segdir WHERE level=3; +} {{3828 86120}} + +do_execsql_test 2.8 { + SELECT sum(length(block)) FROM x2_segdir, x2_segments WHERE + blockid BETWEEN start_block AND leaves_end_block + AND level=3 +} {86120} + +#-------------------------------------------------------------------------- + +do_execsql_test 3.1 { + DROP TABLE IF EXISTS x2; + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(docid, words); + CREATE VIRTUAL TABLE x2 USING fts4; +} +fts_kjv_genesis + +proc t1_to_x2 {} { + foreach id [db eval {SELECT docid FROM t1 LIMIT 2}] { + execsql { + DELETE FROM x2 WHERE docid=$id; + INSERT INTO x2(docid, content) SELECT $id, words FROM t1 WHERE docid=$id; + } + } +} + +#do_test 3.2 { + #t1_to_x2 + #execsql {SELECT level, count(*) FROM x2_segdir GROUP BY level} +#} {0 13 1 15 2 5} + +#proc second {x} { lindex $x 1 } +#db func second second +#for {set i 0} {$i <1000} {incr i} { +# t1_to_x2 +# db eval { +# SELECT level, group_concat( second(end_block), ' ' ) AS c FROM x2_segdir GROUP BY level; +# } { +# puts "$i.$level: $c" +# } +#} + + +finish_test + + From a5e2b50d0af831a525cd469c95f17b3acf797d2c Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 12 May 2014 22:36:50 +0000 Subject: [PATCH 60/71] Improve a comment in the MSVC makefile. No changes to code. FossilOrigin-Name: 0901dccedaf5679691ba803ec76d113de170185d --- Makefile.msc | 3 ++- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index 218abe1f8c..3d9d9f2eb2 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -16,7 +16,8 @@ TOP = . USE_AMALGAMATION = 1 !ENDIF -# Set this non-0 to use the libraries necessary for Windows Phone 8.1. +# Set this non-0 to use the library paths and other options necessary for +# Windows Phone 8.1. # !IFNDEF USE_WP81_OPTS USE_WP81_OPTS = 0 diff --git a/manifest b/manifest index 02097b3c0a..b51b7ce0c3 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Add\sVSIX\spackaging\ssupport\sfor\sWindows\sPhone\s8.1\susing\sVisual\sStudio\s2013\sUpdate\s2. -D 2014-05-12T21:12:04.686 +C Improve\sa\scomment\sin\sthe\sMSVC\smakefile.\s\sNo\schanges\sto\scode. +D 2014-05-12T22:36:50.007 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc 26164f8288a7380b05587534ca3974c3837e5bd0 +F Makefile.msc 52911c6bfeaf9b40f47ef4615ab66d98ab0f2b3d F Makefile.vxworks 034289efa9d591b04b1a73598623119c306cbba0 F README.md 64f270c43c38c46de749e419c22f0ae2f4499fe8 F VERSION 9f823c026c6a32fc5f84d212a8aae0a221dba45c @@ -1170,7 +1170,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 93ffbcc0b1d207950851dbb16a8c101afb949d5f f6237a5f190bd5693ceed0ca1f048c3ec2a4da67 -R c51ed0a7c3ef74dbf1467c7d22b0e47a +P 013738f3d531078fb47f9992c18a0d630e771a32 +R 50044157f65da46177a12dafd4a6afe2 U mistachkin -Z 837dec045ed2c03443718363cb8f5388 +Z 53f9d3afbf5bc14f8e3dfdbc370843ae diff --git a/manifest.uuid b/manifest.uuid index d3e843960e..7af790eea2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -013738f3d531078fb47f9992c18a0d630e771a32 \ No newline at end of file +0901dccedaf5679691ba803ec76d113de170185d \ No newline at end of file From c0caea21dee91b1f80745bf9b81a07b237ec314e Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 13 May 2014 20:11:37 +0000 Subject: [PATCH 61/71] Fix a problem preventing delete markers from ever being removed from the FTS index. FossilOrigin-Name: 7f47ae5c5ddb1227484ddae7c6960183932a052a --- ext/fts3/fts3_write.c | 54 ++++++++++------- manifest | 19 +++--- manifest.uuid | 2 +- test/fts3d.test | 19 +++--- test/fts4growth.test | 132 ++++++++++++++++++++++++++++++++++++++---- 5 files changed, 174 insertions(+), 52 deletions(-) diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index b54d61390a..e2093376e6 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -382,8 +382,9 @@ static int fts3SqlStmt( "ORDER BY level DESC, idx ASC", /* Update statements used while promoting segments */ -/* 38 */ "UPDATE %Q.'%q_segdir' SET level=-1,idx=? WHERE level=? AND idx=?", -/* 39 */ "UPDATE %Q.'%q_segdir' SET level=? WHERE level=-1" +/* 38 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=-1,idx=? " + "WHERE level=? AND idx=?", +/* 39 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=? WHERE level=-1" }; int rc = SQLITE_OK; @@ -3091,6 +3092,7 @@ static int fts3SegmentMerge( Fts3SegFilter filter; /* Segment term filter condition */ Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */ int bIgnoreEmpty = 0; /* True to ignore empty segments */ + i64 iMaxLevel = 0; /* Max level number for this index/langid */ assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel==FTS3_SEGCURSOR_PENDING @@ -3102,6 +3104,11 @@ static int fts3SegmentMerge( rc = sqlite3Fts3SegReaderCursor(p, iLangid, iIndex, iLevel, 0, 0, 1, 0, &csr); if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished; + if( iLevel!=FTS3_SEGCURSOR_PENDING ){ + rc = fts3SegmentMaxLevel(p, iLangid, iIndex, &iMaxLevel); + if( rc!=SQLITE_OK ) goto finished; + } + if( iLevel==FTS3_SEGCURSOR_ALL ){ /* This call is to merge all segments in the database to a single ** segment. The level of the new segment is equal to the numerically @@ -3111,21 +3118,21 @@ static int fts3SegmentMerge( rc = SQLITE_DONE; goto finished; } - rc = fts3SegmentMaxLevel(p, iLangid, iIndex, &iNewLevel); + iNewLevel = iMaxLevel; bIgnoreEmpty = 1; - }else if( iLevel==FTS3_SEGCURSOR_PENDING ){ - iNewLevel = getAbsoluteLevel(p, iLangid, iIndex, 0); - rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, 0, &iIdx); }else{ /* This call is to merge all segments at level iLevel. find the next ** available segment index at level iLevel+1. The call to ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to ** a single iLevel+2 segment if necessary. */ - rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, iLevel+1, &iIdx); + assert( FTS3_SEGCURSOR_PENDING==-1 ); iNewLevel = getAbsoluteLevel(p, iLangid, iIndex, iLevel+1); + rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, iLevel+1, &iIdx); + bIgnoreEmpty = (iLevel!=FTS3_SEGCURSOR_PENDING) && (iNewLevel>iMaxLevel); } if( rc!=SQLITE_OK ) goto finished; + assert( csr.nSegment>0 ); assert( iNewLevel>=getAbsoluteLevel(p, iLangid, iIndex, 0) ); assert( iNewLevelnLeafData); + if( pWriter ){ + rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx); + if( rc==SQLITE_OK ){ + rc = fts3PromoteSegments(p, iNewLevel, pWriter->nLeafData); + } } finished: @@ -4725,6 +4734,7 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ const i64 nMod = FTS3_SEGDIR_MAXLEVEL * p->nIndex; sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */ int bUseHint = 0; /* True if attempting to append */ + int iIdx = 0; /* Largest idx in level (iAbsLevel+1) */ /* Search the %_segdir table for the absolute level with the smallest ** relative level number that contains at least nMin segments, if any. @@ -4778,6 +4788,12 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ ** to start work on some other level. */ memset(pWriter, 0, nAlloc); pFilter->flags = FTS3_SEGMENT_REQUIRE_POS; + + if( rc==SQLITE_OK ){ + rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx); + assert( bUseHint==1 || bUseHint==0 ); + if( (iIdx-bUseHint)==0 ) pFilter->flags |= FTS3_SEGMENT_IGNORE_EMPTY; + } if( rc==SQLITE_OK ){ rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr); } @@ -4785,16 +4801,12 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ && SQLITE_OK==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter)) && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr)) ){ - int iIdx = 0; /* Largest idx in level (iAbsLevel+1) */ - rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx); - if( rc==SQLITE_OK ){ - if( bUseHint && iIdx>0 ){ - const char *zKey = pCsr->zTerm; - int nKey = pCsr->nTerm; - rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx-1, zKey, nKey, pWriter); - }else{ - rc = fts3IncrmergeWriter(p, iAbsLevel, iIdx, pCsr, pWriter); - } + if( bUseHint && iIdx>0 ){ + const char *zKey = pCsr->zTerm; + int nKey = pCsr->nTerm; + rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx-1, zKey, nKey, pWriter); + }else{ + rc = fts3IncrmergeWriter(p, iAbsLevel, iIdx, pCsr, pWriter); } if( rc==SQLITE_OK && pWriter->nLeafEst ){ diff --git a/manifest b/manifest index 1ff6b237e6..357a99fd07 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Experimental\scode\sto\sprevent\sFTS\sindexes\sfrom\sgrowing\sindefinitely\sas\sthe\stable\sis\supdated. -D 2014-05-12T20:04:48.571 +C Fix\sa\sproblem\spreventing\sdelete\smarkers\sfrom\sever\sbeing\sremoved\sfrom\sthe\sFTS\sindex. +D 2014-05-13T20:11:37.423 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -96,7 +96,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_unicode.c 92391b4b4fb043564c6539ea9b8661e3bcba47b9 F ext/fts3/fts3_unicode2.c 0113d3acf13429e6dc38e0647d1bc71211c31a4d -F ext/fts3/fts3_write.c 5fd2aa9d1812387c6254304e20d9ac2b29e16700 +F ext/fts3/fts3_write.c 283b24477729129a210d91b48f7c53181583a848 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197 @@ -539,7 +539,7 @@ F test/fts3conf.test ee8500c86dd58ec075e8831a1e216a79989436de F test/fts3corrupt.test 2710b77983cc7789295ddbffea52c1d3b7506dbb F test/fts3corrupt2.test 6d96efae2f8a6af3eeaf283aba437e6d0e5447ba F test/fts3cov.test e0fb00d8b715ddae4a94c305992dfc3ef70353d7 -F test/fts3d.test 597b0b76e41f0d672e2731c4d7b631d628efd13f +F test/fts3d.test 95c17d1b67b33a5eac0bf5a0d11116a0c0ac7a3a F test/fts3defer.test 0be4440b73a2e651fc1e472066686d6ada4b9963 F test/fts3defer2.test e880e3b65bdf999f4746cdaefa65f14a98b9b724 F test/fts3defer3.test dd53fc13223c6d8264a98244e9b19abd35ed71cd @@ -570,7 +570,7 @@ F test/fts4aa.test 0c3152322c7f0b548cc942ad763eaba0da87ccca F test/fts4check.test 66fa274cab2b615f2fb338b257713aba8fad88a8 F test/fts4content.test 2e7252557d6d24afa101d9ba1de710d6140e6d06 F test/fts4docid.test e33c383cfbdff0284685604d256f347a18fdbf01 -F test/fts4growth.test f7eac9fadfe67765c4a0d6202c85f7272766fb9e +F test/fts4growth.test 3b1f8c98b603b38dc9fe4a266f4f5ddb0c73f092 F test/fts4incr.test 361960ed3550e781f3f313e17e2182ef9cefc0e9 F test/fts4langid.test 24a6e41063b416bbdf371ff6b4476fa41c194aa7 F test/fts4merge.test c424309743fdd203f8e56a1f1cd7872cd66cc0ee @@ -1171,10 +1171,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 0a4f59676bd0ab33b2c86c9a35a2ebbdbaf09ee7 -R de45a14af3a90f6b390273a9a2d8cfa9 -T *branch * fts4-experimental -T *sym-fts4-experimental * -T -sym-trunk * +P b3b505a4dd0c679437a4272109f1188175088cd1 +R d1a058fc899c4ecbd09fd7fceab935e7 U dan -Z a03d85f9bfa278d813164b1e97a88ff7 +Z f678f48fb654227badac5ce98e23d62c diff --git a/manifest.uuid b/manifest.uuid index a9facd1d8e..a6856e6014 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b3b505a4dd0c679437a4272109f1188175088cd1 \ No newline at end of file +7f47ae5c5ddb1227484ddae7c6960183932a052a \ No newline at end of file diff --git a/test/fts3d.test b/test/fts3d.test index 2914818d4e..5c04ead0a0 100644 --- a/test/fts3d.test +++ b/test/fts3d.test @@ -213,16 +213,17 @@ do_test fts3d-4.matches { {0 1 0 4 0 2 5 3 0 3 9 1 0 5 11 4} \ {0 0 0 4 0 4 5 2 0 3 8 1 0 5 10 4}] -check_terms_all fts3d-4.1 {a four is one test that this three two was} +puts [db eval {SELECT c FROM t1 } ] +check_terms_all fts3d-4.1 {a four is test that this was} check_doclist_all fts3d-4.1.1 a {[1 0[2]] [2 0[2]] [3 0[2]]} check_doclist_all fts3d-4.1.2 four {} check_doclist_all fts3d-4.1.3 is {[1 0[1]] [3 0[1]]} -check_doclist_all fts3d-4.1.4 one {} +#check_doclist_all fts3d-4.1.4 one {} check_doclist_all fts3d-4.1.5 test {[1 0[3]] [2 0[3]] [3 0[3]]} check_doclist_all fts3d-4.1.6 that {[2 0[0]]} check_doclist_all fts3d-4.1.7 this {[1 0[0]] [3 0[0]]} -check_doclist_all fts3d-4.1.8 three {} -check_doclist_all fts3d-4.1.9 two {} +#check_doclist_all fts3d-4.1.8 three {} +#check_doclist_all fts3d-4.1.9 two {} check_doclist_all fts3d-4.1.10 was {[2 0[1]]} check_terms fts3d-4.2 0 0 {a four test that was} @@ -239,16 +240,16 @@ check_doclist fts3d-4.3.3 0 1 is {[3 0[1]]} check_doclist fts3d-4.3.4 0 1 test {[3 0[3]]} check_doclist fts3d-4.3.5 0 1 this {[3 0[0]]} -check_terms fts3d-4.4 1 0 {a four is one test that this three two was} +check_terms fts3d-4.4 1 0 {a four is test that this was} check_doclist fts3d-4.4.1 1 0 a {[1 0[2]] [2 0[2]] [3 0[2]]} -check_doclist fts3d-4.4.2 1 0 four {[1] [2 0[4]] [3 0[4]]} +check_doclist fts3d-4.4.2 1 0 four {[2 0[4]] [3 0[4]]} check_doclist fts3d-4.4.3 1 0 is {[1 0[1]] [3 0[1]]} -check_doclist fts3d-4.4.4 1 0 one {[1] [2] [3]} +#check_doclist fts3d-4.4.4 1 0 one {[1] [2] [3]} check_doclist fts3d-4.4.5 1 0 test {[1 0[3]] [2 0[3]] [3 0[3]]} check_doclist fts3d-4.4.6 1 0 that {[2 0[0]]} check_doclist fts3d-4.4.7 1 0 this {[1 0[0]] [3 0[0]]} -check_doclist fts3d-4.4.8 1 0 three {[1] [2] [3]} -check_doclist fts3d-4.4.9 1 0 two {[1] [2] [3]} +#check_doclist fts3d-4.4.8 1 0 three {[1] [2] [3]} +#check_doclist fts3d-4.4.9 1 0 two {[1] [2] [3]} check_doclist fts3d-4.4.10 1 0 was {[2 0[1]]} # Optimize should leave the result in the level of the highest-level diff --git a/test/fts4growth.test b/test/fts4growth.test index ab2ad62142..e5b419a389 100644 --- a/test/fts4growth.test +++ b/test/fts4growth.test @@ -151,7 +151,118 @@ do_execsql_test 2.8 { } {86120} #-------------------------------------------------------------------------- +# Test that delete markers are removed from FTS segments when possible. +# It is only possible to remove delete markers when the output of the +# merge operation will become the oldest segment in the index. +# +# 3.1 - when the oldest segment is created by an 'optimize'. +# 3.2 - when the oldest segment is created by an incremental merge. +# 3.3 - by a crisis merge. +# +proc insert_doc {args} { + foreach iDoc $args { + set L [lindex { + {In your eagerness to engage the Trojans,} + {don’t any of you charge ahead of others,} + {trusting in your strength and horsemanship.} + {And don’t lag behind. That will hurt our charge.} + {Any man whose chariot confronts an enemy’s} + {should thrust with his spear at him from there.} + {That’s the most effective tactic, the way} + {men wiped out city strongholds long ago —} + {their chests full of that style and spirit.} + } [expr $iDoc%9]] + execsql { REPLACE INTO x3(docid, content) VALUES($iDoc, $L) } + } +} + +proc delete_doc {args} { + foreach iDoc $args { + execsql { DELETE FROM x3 WHERE docid = $iDoc } + } +} + +proc second {x} { lindex $x 1 } +db func second second + +do_execsql_test 3.0 { CREATE VIRTUAL TABLE x3 USING fts4 } + +do_test 3.1.1 { + db transaction { insert_doc 1 2 3 4 5 6 } + execsql { SELECT level, idx, second(end_block) FROM x3_segdir } +} {0 0 412} +do_test 3.1.2 { + delete_doc 1 2 3 4 5 6 + execsql { SELECT count(*) FROM x3_segdir } +} {0} +do_test 3.1.3 { + db transaction { + insert_doc 1 2 3 4 5 6 7 8 9 + delete_doc 9 8 7 + } + execsql { SELECT level, idx, second(end_block) FROM x3_segdir } +} {0 0 591 0 1 65 0 2 72 0 3 76} +do_test 3.1.4 { + execsql { INSERT INTO x3(x3) VALUES('optimize') } + execsql { SELECT level, idx, second(end_block) FROM x3_segdir } +} {0 0 412} + +do_test 3.2.1 { + execsql { DELETE FROM x3 } + insert_doc 8 7 6 5 4 3 2 1 + delete_doc 7 8 + execsql { SELECT count(*) FROM x3_segdir } +} {10} +do_test 3.2.2 { + execsql { INSERT INTO x3(x3) VALUES('merge=500,10') } + execsql { SELECT level, idx, second(end_block) FROM x3_segdir } +} {1 0 412} + +# This assumes the crisis merge happens when there are already 16 +# segments and one more is added. +# +do_test 3.3.1 { + execsql { DELETE FROM x3 } + insert_doc 1 2 3 4 5 6 7 8 9 10 11 + delete_doc 11 10 9 8 7 + execsql { SELECT count(*) FROM x3_segdir } +} {16} + +do_test 3.3.2 { + insert_doc 12 + execsql { SELECT level, idx, second(end_block) FROM x3_segdir WHERE level=1 } +} {1 0 412} + +#-------------------------------------------------------------------------- +do_execsql_test 4.1 { + DROP TABLE IF EXISTS x4; + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(docid, words); + CREATE VIRTUAL TABLE x4 USING fts4(words); +} +do_test 4.2 { + fts_kjv_genesis + execsql { INSERT INTO x4 SELECT words FROM t1 } + execsql { INSERT INTO x4 SELECT words FROM t1 } +} {} + +do_execsql_test 4.3 { + SELECT level, idx, second(end_block) FROM x4_segdir +} {0 0 117483 0 1 118006} + +do_execsql_test 4.4 { + INSERT INTO x4(x4) VALUES('merge=10,2'); + SELECT count(*) FROM x4_segdir; +} {3} + +breakpoint +do_execsql_test 4.5 { + INSERT INTO x4(x4) VALUES('merge=10,2'); + SELECT count(*) FROM x4_segdir; +} {3} + +if 0 { do_execsql_test 3.1 { DROP TABLE IF EXISTS x2; DROP TABLE IF EXISTS t1; @@ -174,16 +285,17 @@ proc t1_to_x2 {} { #execsql {SELECT level, count(*) FROM x2_segdir GROUP BY level} #} {0 13 1 15 2 5} -#proc second {x} { lindex $x 1 } -#db func second second -#for {set i 0} {$i <1000} {incr i} { -# t1_to_x2 -# db eval { -# SELECT level, group_concat( second(end_block), ' ' ) AS c FROM x2_segdir GROUP BY level; -# } { -# puts "$i.$level: $c" -# } -#} +proc second {x} { lindex $x 1 } +db func second second +for {set i 0} {$i <1000} {incr i} { + t1_to_x2 + db eval { + SELECT level, group_concat( second(end_block), ' ' ) AS c FROM x2_segdir GROUP BY level; + } { + puts "$i.$level: $c" + } +} +} finish_test From 9e83e465eba9417372f45f5d87b2cd3777690a7a Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 14 May 2014 15:58:47 +0000 Subject: [PATCH 62/71] Fix various problems to do with segment promotion. Add test file fts4growth2.test, containing tests to check that the FTS index does not grow indefinitely as the table is updated. FossilOrigin-Name: 21491a9bc686e63bec32f1a67103622f1160a2f9 --- ext/fts3/fts3.c | 9 ++-- ext/fts3/fts3Int.h | 2 +- ext/fts3/fts3_write.c | 60 ++++++++++++++++++--- manifest | 21 ++++---- manifest.uuid | 2 +- test/fts4growth.test | 117 +++++++++++++++++++++++++++++++++++------ test/fts4growth2.test | 89 +++++++++++++++++++++++++++++++ test/permutations.test | 2 + 8 files changed, 262 insertions(+), 40 deletions(-) create mode 100644 test/fts4growth2.test diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index f7b2232902..c421aa9229 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -1333,7 +1333,7 @@ static int fts3InitVtab( p->bHasStat = isFts4; p->bFts4 = isFts4; p->bDescIdx = bDescIdx; - p->bAutoincrmerge = 0xff; /* 0xff means setting unknown */ + p->nAutoincrmerge = 0xff; /* 0xff means setting unknown */ p->zContentTbl = zContent; p->zLanguageid = zLanguageid; zContent = 0; @@ -3302,7 +3302,10 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){ Fts3Table *p = (Fts3Table*)pVtab; int rc = sqlite3Fts3PendingTermsFlush(p); - if( rc==SQLITE_OK && p->bAutoincrmerge==1 && p->nLeafAdd>(nMinMerge/16) ){ + if( rc==SQLITE_OK + && p->nLeafAdd>(nMinMerge/16) + && p->nAutoincrmerge && p->nAutoincrmerge!=0xff + ){ int mxLevel = 0; /* Maximum relative level value in db */ int A; /* Incr-merge parameter A */ @@ -3310,7 +3313,7 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){ assert( rc==SQLITE_OK || mxLevel==0 ); A = p->nLeafAdd * mxLevel; A += (A/2); - if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, 8); + if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, p->nAutoincrmerge); } sqlite3Fts3SegmentsClose(p); return rc; diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 171a75d124..1383102f4c 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -210,7 +210,7 @@ struct Fts3Table { sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ char *zContentTbl; /* content=xxx option, or NULL */ char *zLanguageid; /* languageid=xxx option, or NULL */ - u8 bAutoincrmerge; /* True if automerge=1 */ + int nAutoincrmerge; /* Value configured by 'automerge' */ u32 nLeafAdd; /* Number of leaf blocks added this trans */ /* Precompiled statements used by the implementation. Each of these diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index e2093376e6..1e82187d44 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -2971,16 +2971,23 @@ void sqlite3Fts3SegReaderFinish( /* ** Decode the "end_block" field, selected by column iCol of the SELECT ** statement passed as the first argument. +** +** The "end_block" field may contain either an integer, or a text field +** containing the text representation of two non-negative integers separated +** by one or more space (0x20) characters. In the first case, set *piEndBlock +** to the integer value and *pnByte to zero before returning. In the second, +** set *piEndBlock to the first value and *pnByte to the second. */ static void fts3ReadEndBlockField( sqlite3_stmt *pStmt, int iCol, - i64 *piEndBlock, + i64 *piEndBlock, i64 *pnByte ){ const unsigned char *zText = sqlite3_column_text(pStmt, iCol); if( zText ){ int i; + int iMul = 1; i64 iVal = 0; for(i=0; zText[i]>='0' && zText[i]<='9'; i++){ iVal = iVal*10 + (zText[i] - '0'); @@ -2988,10 +2995,14 @@ static void fts3ReadEndBlockField( *piEndBlock = iVal; while( zText[i]==' ' ) i++; iVal = 0; + if( zText[i]=='-' ){ + i++; + iMul = -1; + } for(/* no-op */; zText[i]>='0' && zText[i]<='9'; i++){ iVal = iVal*10 + (zText[i] - '0'); } - *pnByte = iVal; + *pnByte = (iVal * (i64)iMul); } } @@ -3011,18 +3022,30 @@ static int fts3PromoteSegments( rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE2, &pRange, 0); if( rc==SQLITE_OK ){ - int bOk = 1; + int bOk = 0; int iLast = (iAbsLevel/FTS3_SEGDIR_MAXLEVEL + 1) * FTS3_SEGDIR_MAXLEVEL - 1; + i64 nLimit = (nByte*3)/2; + /* Loop through all entries in the %_segdir table corresponding to + ** segments in this index on levels greater than iAbsLevel. If there is + ** at least one such segment, and it is possible to determine that all + ** such segments are smaller than nLimit bytes in size, they will be + ** promoted to level iAbsLevel. */ sqlite3_bind_int(pRange, 1, iAbsLevel+1); sqlite3_bind_int(pRange, 2, iLast); while( SQLITE_ROW==sqlite3_step(pRange) ){ i64 nSize, dummy; fts3ReadEndBlockField(pRange, 2, &dummy, &nSize); - if( nSize>nByte ){ + if( nSize<=0 || nSize>nLimit ){ + /* If nSize==0, then the %_segdir.end_block field does not not + ** contain a size value. This happens if it was written by an + ** old version of FTS. In this case it is not possible to determine + ** the size of the segment, and so segment promotion does not + ** take place. */ bOk = 0; break; } + bOk = 1; } rc = sqlite3_reset(pRange); @@ -3039,6 +3062,16 @@ static int fts3PromoteSegments( } if( rc==SQLITE_OK ){ + + /* Loop through all %_segdir entries for segments in this index with + ** levels equal to or greater than iAbsLevel. As each entry is visited, + ** updated it to set (level = -1) and (idx = N), where N is 0 for the + ** oldest segment in the range, 1 for the next oldest, and so on. + ** + ** In other words, move all segments being promoted to level -1, + ** setting the "idx" fields as appropriate to keep them in the same + ** order. The contents of level -1 (which is never used, except + ** transiently here), will be moved back to level iAbsLevel below. */ sqlite3_bind_int(pRange, 1, iAbsLevel); while( SQLITE_ROW==sqlite3_step(pRange) ){ sqlite3_bind_int(pUpdate1, 1, iIdx++); @@ -3056,6 +3089,7 @@ static int fts3PromoteSegments( rc = sqlite3_reset(pRange); } + /* Move level -1 to level iAbsLevel */ if( rc==SQLITE_OK ){ sqlite3_bind_int(pUpdate2, 1, iAbsLevel); sqlite3_step(pUpdate2); @@ -3188,14 +3222,15 @@ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ ** estimate the number of leaf blocks of content to be written */ if( rc==SQLITE_OK && p->bHasStat - && p->bAutoincrmerge==0xff && p->nLeafAdd>0 + && p->nAutoincrmerge==0xff && p->nLeafAdd>0 ){ sqlite3_stmt *pStmt = 0; rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); rc = sqlite3_step(pStmt); - p->bAutoincrmerge = (rc==SQLITE_ROW && sqlite3_column_int(pStmt, 0)); + p->nAutoincrmerge = (rc==SQLITE_ROW && sqlite3_column_int(pStmt, 0)); + if( p->nAutoincrmerge==1 ) p->nAutoincrmerge = 8; rc = sqlite3_reset(pStmt); } } @@ -4105,6 +4140,9 @@ static int fts3IncrmergeLoad( iStart = sqlite3_column_int64(pSelect, 1); iLeafEnd = sqlite3_column_int64(pSelect, 2); fts3ReadEndBlockField(pSelect, 3, &iEnd, &pWriter->nLeafData); + if( pWriter->nLeafData<0 ){ + pWriter->nLeafData = pWriter->nLeafData * -1; + } nRoot = sqlite3_column_bytes(pSelect, 4); aRoot = sqlite3_column_blob(pSelect, 4); }else{ @@ -4828,6 +4866,9 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ } } + if( nSeg!=0 ){ + pWriter->nLeafData = pWriter->nLeafData * -1; + } fts3IncrmergeRelease(p, pWriter, &rc); if( nSeg==0 ){ fts3PromoteSegments(p, iAbsLevel+1, pWriter->nLeafData); @@ -4918,7 +4959,10 @@ static int fts3DoAutoincrmerge( ){ int rc = SQLITE_OK; sqlite3_stmt *pStmt = 0; - p->bAutoincrmerge = fts3Getint(&zParam)!=0; + p->nAutoincrmerge = fts3Getint(&zParam); + if( p->nAutoincrmerge==1 || p->nAutoincrmerge>FTS3_MERGE_COUNT ){ + p->nAutoincrmerge = 8; + } if( !p->bHasStat ){ assert( p->bFts4==0 ); sqlite3Fts3CreateStatTable(&rc, p); @@ -4927,7 +4971,7 @@ static int fts3DoAutoincrmerge( rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0); if( rc ) return rc; sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); - sqlite3_bind_int(pStmt, 2, p->bAutoincrmerge); + sqlite3_bind_int(pStmt, 2, p->nAutoincrmerge); sqlite3_step(pStmt); rc = sqlite3_reset(pStmt); return rc; diff --git a/manifest b/manifest index 357a99fd07..276a411569 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\spreventing\sdelete\smarkers\sfrom\sever\sbeing\sremoved\sfrom\sthe\sFTS\sindex. -D 2014-05-13T20:11:37.423 +C Fix\svarious\sproblems\sto\sdo\swith\ssegment\spromotion.\sAdd\stest\sfile\sfts4growth2.test,\scontaining\stests\sto\scheck\sthat\sthe\sFTS\sindex\sdoes\snot\sgrow\sindefinitely\sas\sthe\stable\sis\supdated. +D 2014-05-14T15:58:47.565 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -78,9 +78,9 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 41b1920b9a8657963f09cb93b208c2671c5568db +F ext/fts3/fts3.c e83f894cf1adaf8decd6b1de76bfdcdb79b25507 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h b250b154764b86be4ee540f8c233a59858e88f1d +F ext/fts3/fts3Int.h 16cddf2d7b0e5f3681615ae1d8ca0e45fca44918 F ext/fts3/fts3_aux.c 5c211e17a64885faeb16b9ba7772f9d5445c2365 F ext/fts3/fts3_expr.c 2ac35bda474f00c14c19608e49a02c8c7ceb9970 F ext/fts3/fts3_hash.c 29b986e43f4e9dd40110eafa377dc0d63c422c60 @@ -96,7 +96,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_unicode.c 92391b4b4fb043564c6539ea9b8661e3bcba47b9 F ext/fts3/fts3_unicode2.c 0113d3acf13429e6dc38e0647d1bc71211c31a4d -F ext/fts3/fts3_write.c 283b24477729129a210d91b48f7c53181583a848 +F ext/fts3/fts3_write.c 5e13a15dad44bf516181e9d6251c7d6a27861dee F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197 @@ -570,7 +570,8 @@ F test/fts4aa.test 0c3152322c7f0b548cc942ad763eaba0da87ccca F test/fts4check.test 66fa274cab2b615f2fb338b257713aba8fad88a8 F test/fts4content.test 2e7252557d6d24afa101d9ba1de710d6140e6d06 F test/fts4docid.test e33c383cfbdff0284685604d256f347a18fdbf01 -F test/fts4growth.test 3b1f8c98b603b38dc9fe4a266f4f5ddb0c73f092 +F test/fts4growth.test 50befaa001a6dd4046d32e5d7b3cb75df8360ca8 +F test/fts4growth2.test 19fda88600266629b63a4cece512c2dc6f7c9fc5 F test/fts4incr.test 361960ed3550e781f3f313e17e2182ef9cefc0e9 F test/fts4langid.test 24a6e41063b416bbdf371ff6b4476fa41c194aa7 F test/fts4merge.test c424309743fdd203f8e56a1f1cd7872cd66cc0ee @@ -745,7 +746,7 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54 -F test/permutations.test 40add071ba71aefe1c04f5845308cf46f7de8d04 +F test/permutations.test 975f65293a46cf8deb896ecdaf1ba563ccaca24a F test/pragma.test adb21a90875bc54a880fa939c4d7c46598905aa0 F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -1171,7 +1172,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b3b505a4dd0c679437a4272109f1188175088cd1 -R d1a058fc899c4ecbd09fd7fceab935e7 +P 7f47ae5c5ddb1227484ddae7c6960183932a052a +R a0acf9231899e84a47fa0a8ff85a8e05 U dan -Z f678f48fb654227badac5ce98e23d62c +Z 4a0ad803ddf94719a771980af32709c7 diff --git a/manifest.uuid b/manifest.uuid index a6856e6014..4b5569d5e0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7f47ae5c5ddb1227484ddae7c6960183932a052a \ No newline at end of file +21491a9bc686e63bec32f1a67103622f1160a2f9 \ No newline at end of file diff --git a/test/fts4growth.test b/test/fts4growth.test index e5b419a389..466416e561 100644 --- a/test/fts4growth.test +++ b/test/fts4growth.test @@ -131,7 +131,7 @@ do_execsql_test 2.5 { SELECT end_block FROM x2_segdir WHERE level=3; INSERT INTO x2(x2) VALUES('merge=4,4'); SELECT end_block FROM x2_segdir WHERE level=3; -} {{3828 3430} {3828 10191} {3828 14109}} +} {{3828 -3430} {3828 -10191} {3828 -14109}} do_execsql_test 2.6 { SELECT sum(length(block)) FROM x2_segdir, x2_segments WHERE @@ -235,6 +235,9 @@ do_test 3.3.2 { } {1 0 412} #-------------------------------------------------------------------------- +# Check a theory on a bug in fts4 - that segments with idx==0 were not +# being incrementally merged correctly. Theory turned out to be false. +# do_execsql_test 4.1 { DROP TABLE IF EXISTS x4; DROP TABLE IF EXISTS t1; @@ -256,14 +259,23 @@ do_execsql_test 4.4 { SELECT count(*) FROM x4_segdir; } {3} -breakpoint do_execsql_test 4.5 { INSERT INTO x4(x4) VALUES('merge=10,2'); SELECT count(*) FROM x4_segdir; } {3} -if 0 { -do_execsql_test 3.1 { +do_execsql_test 4.6 { + INSERT INTO x4(x4) VALUES('merge=1000,2'); + SELECT count(*) FROM x4_segdir; +} {1} + + + +#-------------------------------------------------------------------------- +# Check that segments are not promoted if the "end_block" field does not +# contain a size. +# +do_execsql_test 5.1 { DROP TABLE IF EXISTS x2; DROP TABLE IF EXISTS t1; CREATE TABLE t1(docid, words); @@ -271,6 +283,41 @@ do_execsql_test 3.1 { } fts_kjv_genesis +proc first {L} {lindex $L 0} +db func first first + +do_test 5.2 { + foreach r [db eval { SELECT rowid FROM t1 }] { + execsql { + INSERT INTO x2(docid, content) SELECT docid, words FROM t1 WHERE rowid=$r + } + } + foreach d [db eval { SELECT docid FROM t1 LIMIT -1 OFFSET 20 }] { + execsql { DELETE FROM x2 WHERE docid = $d } + } + + execsql { + INSERT INTO x2(x2) VALUES('optimize'); + SELECT level, idx, end_block FROM x2_segdir + } +} {2 0 {752 1926}} + +do_execsql_test 5.3 { + UPDATE x2_segdir SET end_block = CAST( first(end_block) AS INTEGER ); + SELECT end_block, typeof(end_block) FROM x2_segdir; +} {752 integer} + +do_execsql_test 5.4 { + INSERT INTO x2 SELECT words FROM t1 LIMIT 50; + SELECT level, idx, end_block FROM x2_segdir +} {2 0 752 0 0 {758 5174}} + +do_execsql_test 5.5 { + UPDATE x2_segdir SET end_block = end_block || ' 1926' WHERE level=2; + INSERT INTO x2 SELECT words FROM t1 LIMIT 40; + SELECT level, idx, end_block FROM x2_segdir +} {0 0 {752 1926} 0 1 {758 5174} 0 2 {763 4170}} + proc t1_to_x2 {} { foreach id [db eval {SELECT docid FROM t1 LIMIT 2}] { execsql { @@ -280,24 +327,60 @@ proc t1_to_x2 {} { } } +#-------------------------------------------------------------------------- +# Check that segments created by auto-merge are not promoted until they +# are completed. +# + +do_execsql_test 6.1 { + CREATE VIRTUAL TABLE x5 USING fts4; + INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 0; + INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 25; + INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 50; + INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 75; + SELECT count(*) FROM x5_segdir +} {4} + +do_execsql_test 6.2 { + INSERT INTO x5(x5) VALUES('merge=2,4'); + SELECT level, idx, end_block FROM x5_segdir; +} {0 0 {10 9216} 0 1 {21 9330} 0 2 {31 8850} 0 3 {40 8689} 1 0 {1320 -3117}} + +do_execsql_test 6.3 { + INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 100; + SELECT level, idx, end_block FROM x5_segdir; +} { + 0 0 {10 9216} 0 1 {21 9330} 0 2 {31 8850} + 0 3 {40 8689} 1 0 {1320 -3117} 0 4 {1329 8297} +} + +do_execsql_test 6.4 { + INSERT INTO x5(x5) VALUES('merge=200,4'); + SELECT level, idx, end_block FROM x5_segdir; +} {0 0 {1329 8297} 1 0 {1320 28009}} + +do_execsql_test 6.5 { + INSERT INTO x5 SELECT words FROM t1; + SELECT level, idx, end_block FROM x5_segdir; +} { + 0 1 {1329 8297} 0 0 {1320 28009} 0 2 {1449 118006} +} + #do_test 3.2 { #t1_to_x2 #execsql {SELECT level, count(*) FROM x2_segdir GROUP BY level} #} {0 13 1 15 2 5} -proc second {x} { lindex $x 1 } -db func second second -for {set i 0} {$i <1000} {incr i} { - t1_to_x2 - db eval { - SELECT level, group_concat( second(end_block), ' ' ) AS c FROM x2_segdir GROUP BY level; - } { - puts "$i.$level: $c" - } -} -} - +#proc second {x} { lindex $x 1 } +#db func second second +#for {set i 0} {$i <1000} {incr i} { +# t1_to_x2 +# db eval { +# SELECT level, group_concat( second(end_block), ' ' ) AS c FROM x2_segdir GROUP BY level; +# } { +# puts "$i.$level: $c" +# } +#} finish_test - diff --git a/test/fts4growth2.test b/test/fts4growth2.test new file mode 100644 index 0000000000..e56309ab36 --- /dev/null +++ b/test/fts4growth2.test @@ -0,0 +1,89 @@ +# 2014 May 12 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS4 module. +# +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix fts4growth + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts3 { + finish_test + return +} + +source $testdir/genesis.tcl + +do_execsql_test 1.0 { CREATE TABLE t1(docid, words); } +fts_kjv_genesis + +proc tt {val} { + execsql { + BEGIN; + DELETE FROM x1 + WHERE docid IN (SELECT docid FROM t1 WHERE (rowid-1)%4==$val+0); + INSERT INTO x1(docid, content) + SELECT docid, words FROM t1 WHERE (rowid%4)==$val+0; + COMMIT; + } +} + +do_execsql_test 1.1 { + CREATE VIRTUAL TABLE x1 USING fts4; + INSERT INTO x1(x1) VALUES('automerge=2'); +} + +do_test 1.2 { + for {set i 0} {$i < 100} {incr i} { + tt 0 ; tt 1 ; tt 2 ; tt 3 + } + execsql { + SELECT max(level) FROM x1_segdir; + SELECT count(*) FROM x1_segdir WHERE level=4; + } +} {4 1} + +do_test 1.3 { + for {set i 0} {$i < 100} {incr i} { + tt 0 ; tt 1 ; tt 2 ; tt 3 + } + execsql { + SELECT max(level) FROM x1_segdir; + SELECT count(*) FROM x1_segdir WHERE level=4; + } +} {4 1} + +#------------------------------------------------------------------------- +# +do_execsql_test 2.1 { + DELETE FROM t1 WHERE rowid>16; + DROP TABLE IF EXISTS x1; + CREATE VIRTUAL TABLE x1 USING fts4; +} + +db func second second +proc second {L} {lindex $L 1} + +for {set tn 0} {$tn < 40} {incr tn} { + do_test 2.2.$tn { + for {set i 0} {$i < 100} {incr i} { + tt 0 ; tt 1 ; tt 2 ; tt 3 + } + execsql { SELECT max(level) FROM x1_segdir } + } {1} +} + + +finish_test + diff --git a/test/permutations.test b/test/permutations.test index 7f1485f831..b7e1eaf8d2 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -112,6 +112,7 @@ set allquicktests [test_set $alltests -exclude { incrvacuum_ioerr.test autovacuum_crash.test btree8.test shared_err.test vtab_err.test walslow.test walcrash.test walcrash3.test walthread.test rtree3.test indexfault.test securedel2.test + fts3growth2.test }] if {[info exists ::env(QUICKTEST_INCLUDE)]} { set allquicktests [concat $allquicktests $::env(QUICKTEST_INCLUDE)] @@ -196,6 +197,7 @@ test_suite "fts3" -prefix "" -description { fts3corrupt2.test fts3first.test fts4langid.test fts4merge.test fts4check.test fts4unicode.test fts4noti.test fts3varint.test + fts4growth.test fts4growth2.test } test_suite "nofaultsim" -prefix "" -description { From a0b7ca90b71fb47e370ecb4b2ec2f27b01dc7617 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 14 May 2014 18:45:27 +0000 Subject: [PATCH 63/71] Fix a problem to do with loading the value of the FTS automerge setting from the database. FossilOrigin-Name: 7268119f74602929b372b88eca7b3c3b4964f7d5 --- ext/fts3/fts3_write.c | 6 ++++-- manifest | 18 ++++++++--------- manifest.uuid | 2 +- test/fts4growth2.test | 25 ++++++++++++------------ test/fts4merge4.test | 44 ++++++++++++++++++++++++++++++++++++++++++ test/permutations.test | 2 +- 6 files changed, 71 insertions(+), 26 deletions(-) diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 1e82187d44..9bc2091d20 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -3229,8 +3229,10 @@ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ if( rc==SQLITE_OK ){ sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); rc = sqlite3_step(pStmt); - p->nAutoincrmerge = (rc==SQLITE_ROW && sqlite3_column_int(pStmt, 0)); - if( p->nAutoincrmerge==1 ) p->nAutoincrmerge = 8; + if( rc==SQLITE_ROW ){ + p->nAutoincrmerge = sqlite3_column_int(pStmt, 0); + if( p->nAutoincrmerge==1 ) p->nAutoincrmerge = 8; + } rc = sqlite3_reset(pStmt); } } diff --git a/manifest b/manifest index 276a411569..8a696af535 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\svarious\sproblems\sto\sdo\swith\ssegment\spromotion.\sAdd\stest\sfile\sfts4growth2.test,\scontaining\stests\sto\scheck\sthat\sthe\sFTS\sindex\sdoes\snot\sgrow\sindefinitely\sas\sthe\stable\sis\supdated. -D 2014-05-14T15:58:47.565 +C Fix\sa\sproblem\sto\sdo\swith\sloading\sthe\svalue\sof\sthe\sFTS\sautomerge\ssetting\sfrom\sthe\sdatabase. +D 2014-05-14T18:45:27.234 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -96,7 +96,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_unicode.c 92391b4b4fb043564c6539ea9b8661e3bcba47b9 F ext/fts3/fts3_unicode2.c 0113d3acf13429e6dc38e0647d1bc71211c31a4d -F ext/fts3/fts3_write.c 5e13a15dad44bf516181e9d6251c7d6a27861dee +F ext/fts3/fts3_write.c 6cb17b13b08fa2f36c8079726207736246a7176c F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197 @@ -571,13 +571,13 @@ F test/fts4check.test 66fa274cab2b615f2fb338b257713aba8fad88a8 F test/fts4content.test 2e7252557d6d24afa101d9ba1de710d6140e6d06 F test/fts4docid.test e33c383cfbdff0284685604d256f347a18fdbf01 F test/fts4growth.test 50befaa001a6dd4046d32e5d7b3cb75df8360ca8 -F test/fts4growth2.test 19fda88600266629b63a4cece512c2dc6f7c9fc5 +F test/fts4growth2.test 4ea35ea97c7d87e8f19d5ede939ab356f9f75cf0 F test/fts4incr.test 361960ed3550e781f3f313e17e2182ef9cefc0e9 F test/fts4langid.test 24a6e41063b416bbdf371ff6b4476fa41c194aa7 F test/fts4merge.test c424309743fdd203f8e56a1f1cd7872cd66cc0ee F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 F test/fts4merge3.test aab02a09f50fe6baaddc2e159c3eabc116d45fc7 -F test/fts4merge4.test c19c85ca1faa7b6d536832b49c12e1867235f584 +F test/fts4merge4.test d895b1057a7798b67e03455d0fa50e9ea836c47b F test/fts4noti.test aed33ba44808852dcb24bf70fa132e7bf530f057 F test/fts4unicode.test 01ec3fe2a7c3cfff3b4c0581b83caa11b33efa36 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d @@ -746,7 +746,7 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54 -F test/permutations.test 975f65293a46cf8deb896ecdaf1ba563ccaca24a +F test/permutations.test d8946218493c77806ec1029df473d8ed4ad9aef6 F test/pragma.test adb21a90875bc54a880fa939c4d7c46598905aa0 F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -1172,7 +1172,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7f47ae5c5ddb1227484ddae7c6960183932a052a -R a0acf9231899e84a47fa0a8ff85a8e05 +P 21491a9bc686e63bec32f1a67103622f1160a2f9 +R 67348fc4be2e8c129b7e6e89a490a40b U dan -Z 4a0ad803ddf94719a771980af32709c7 +Z e904ec03ef68eb7c18f9e542f40d2d02 diff --git a/manifest.uuid b/manifest.uuid index 4b5569d5e0..698af193d2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -21491a9bc686e63bec32f1a67103622f1160a2f9 \ No newline at end of file +7268119f74602929b372b88eca7b3c3b4964f7d5 \ No newline at end of file diff --git a/test/fts4growth2.test b/test/fts4growth2.test index e56309ab36..023ca8ba1c 100644 --- a/test/fts4growth2.test +++ b/test/fts4growth2.test @@ -15,7 +15,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -set testprefix fts4growth +set testprefix fts4growth2 # If SQLITE_ENABLE_FTS3 is defined, omit this file. ifcapable !fts3 { @@ -30,13 +30,12 @@ fts_kjv_genesis proc tt {val} { execsql { - BEGIN; - DELETE FROM x1 - WHERE docid IN (SELECT docid FROM t1 WHERE (rowid-1)%4==$val+0); - INSERT INTO x1(docid, content) - SELECT docid, words FROM t1 WHERE (rowid%4)==$val+0; - COMMIT; + DELETE FROM x1 + WHERE docid IN (SELECT docid FROM t1 WHERE (rowid-1)%4==$val+0); + INSERT INTO x1(docid, content) + SELECT docid, words FROM t1 WHERE (rowid%4)==$val+0; } + #puts [db eval {SELECT level, idx, end_block FROM x1_segdir}] } do_execsql_test 1.1 { @@ -45,24 +44,24 @@ do_execsql_test 1.1 { } do_test 1.2 { - for {set i 0} {$i < 100} {incr i} { + for {set i 0} {$i < 40} {incr i} { tt 0 ; tt 1 ; tt 2 ; tt 3 } execsql { SELECT max(level) FROM x1_segdir; - SELECT count(*) FROM x1_segdir WHERE level=4; + SELECT count(*) FROM x1_segdir WHERE level=3; } -} {4 1} +} {3 1} do_test 1.3 { - for {set i 0} {$i < 100} {incr i} { + for {set i 0} {$i < 40} {incr i} { tt 0 ; tt 1 ; tt 2 ; tt 3 } execsql { SELECT max(level) FROM x1_segdir; - SELECT count(*) FROM x1_segdir WHERE level=4; + SELECT count(*) FROM x1_segdir WHERE level=2; } -} {4 1} +} {2 1} #------------------------------------------------------------------------- # diff --git a/test/fts4merge4.test b/test/fts4merge4.test index 8e2119de2b..038e460d0e 100644 --- a/test/fts4merge4.test +++ b/test/fts4merge4.test @@ -53,6 +53,50 @@ do_execsql_test 2.2 { SELECT count(*) FROM t1_segdir; } 35 do_execsql_test 2.3 { INSERT INTO t1(t1) VALUES('optimize') } {} do_execsql_test 2.4 { SELECT count(*) FROM t1_segdir; } 1 +#------------------------------------------------------------------------- +# Now test that the automerge=? option appears to work. +# +do_execsql_test 2.1 { CREATE VIRTUAL TABLE t2 USING fts4; } + +set doc "" +foreach c1 "a b c d e f g h i j" { + foreach c2 "a b c d e f g h i j" { + foreach c3 "a b c d e f g h i j" { + lappend doc "$c1$c2$c3" + } + } +} +set doc [string repeat $doc 10] + +foreach {tn am expected} { + 1 {automerge=2} {1 1 2 1 4 1 6 1} + 2 {automerge=4} {1 2 2 1 3 1} + 3 {automerge=8} {0 4 1 3 2 1} + 4 {automerge=1} {0 4 1 3 2 1} +} { + foreach {tn2 openclose} {1 {} 2 { db close ; sqlite3 db test.db }} { + do_test 2.2.$tn.$tn2 { + execsql { DELETE FROM t2 } + execsql { INSERT INTO t2(t2) VALUES($am) }; + + eval $openclose + + for {set i 0} {$i < 100} {incr i} { + execsql { + BEGIN; + INSERT INTO t2 VALUES($doc); + INSERT INTO t2 VALUES($doc); + INSERT INTO t2 VALUES($doc); + INSERT INTO t2 VALUES($doc); + INSERT INTO t2 VALUES($doc); + COMMIT; + } + } + + execsql { SELECT level, count(*) FROM t2_segdir GROUP BY level } + } [list {*}$expected] + } +} sqlite3_enable_shared_cache $::enable_shared_cache finish_test diff --git a/test/permutations.test b/test/permutations.test index b7e1eaf8d2..0e5cc2c8d2 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -112,7 +112,7 @@ set allquicktests [test_set $alltests -exclude { incrvacuum_ioerr.test autovacuum_crash.test btree8.test shared_err.test vtab_err.test walslow.test walcrash.test walcrash3.test walthread.test rtree3.test indexfault.test securedel2.test - fts3growth2.test + fts4growth.test fts4growth2.test }] if {[info exists ::env(QUICKTEST_INCLUDE)]} { set allquicktests [concat $allquicktests $::env(QUICKTEST_INCLUDE)] From 5b34f15a82644307478dc01fdb0c0cda36430d0c Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 14 May 2014 19:49:17 +0000 Subject: [PATCH 64/71] Do not search for promotable segments following an FTS 'optimize' operation or creation of a new segment on the oldest existing level. FossilOrigin-Name: 34f6b4b8e4cb8ea45fd518672dc96335d1ce06ab --- ext/fts3/fts3_write.c | 8 ++++++-- manifest | 14 +++++++------- manifest.uuid | 2 +- test/trace2.test | 1 + 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 9bc2091d20..ce16f0c86a 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -378,7 +378,7 @@ static int fts3SqlStmt( /* Return segments in order from oldest to newest.*/ /* 37 */ "SELECT level, idx, end_block " - "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?" + "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? " "ORDER BY level DESC, idx ASC", /* Update statements used while promoting segments */ @@ -3194,7 +3194,9 @@ static int fts3SegmentMerge( if( pWriter ){ rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx); if( rc==SQLITE_OK ){ - rc = fts3PromoteSegments(p, iNewLevel, pWriter->nLeafData); + if( iLevel==FTS3_SEGCURSOR_PENDING || iNewLevelnLeafData); + } } } @@ -3232,6 +3234,8 @@ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ if( rc==SQLITE_ROW ){ p->nAutoincrmerge = sqlite3_column_int(pStmt, 0); if( p->nAutoincrmerge==1 ) p->nAutoincrmerge = 8; + }else if( rc==SQLITE_DONE ){ + p->nAutoincrmerge = 0; } rc = sqlite3_reset(pStmt); } diff --git a/manifest b/manifest index 8a696af535..2b1578669e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sto\sdo\swith\sloading\sthe\svalue\sof\sthe\sFTS\sautomerge\ssetting\sfrom\sthe\sdatabase. -D 2014-05-14T18:45:27.234 +C Do\snot\ssearch\sfor\spromotable\ssegments\sfollowing\san\sFTS\s'optimize'\soperation\sor\screation\sof\sa\snew\ssegment\son\sthe\soldest\sexisting\slevel. +D 2014-05-14T19:49:17.392 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -96,7 +96,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_unicode.c 92391b4b4fb043564c6539ea9b8661e3bcba47b9 F ext/fts3/fts3_unicode2.c 0113d3acf13429e6dc38e0647d1bc71211c31a4d -F ext/fts3/fts3_write.c 6cb17b13b08fa2f36c8079726207736246a7176c +F ext/fts3/fts3_write.c e58c8df149436dcc85e2face5516b75a01c7a583 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197 @@ -1008,7 +1008,7 @@ F test/tkt4018.test 7c2c9ba4df489c676a0a7a0e809a1fb9b2185bd1 F test/tokenize.test ce430a7aed48fc98301611429595883fdfcab5d7 F test/tpch01.test 8f4ac52f62f3e9f6bce0889105aecdf0275e331b F test/trace.test 4b36a41a3e9c7842151af6da5998f5080cdad9e5 -F test/trace2.test e7a988fdd982cdec62f1f1f34b0360e6476d01a0 +F test/trace2.test 93b47ca6996c66b47f57224cfb146f34e07df382 F test/trans.test 6e1b4c6a42dba31bd65f8fa5e61a2708e08ddde6 F test/trans2.test 62bd045bfc7a1c14c5ba83ba64d21ade31583f76 F test/trans3.test 373ac5183cc56be69f48ae44090e7f672939f732 @@ -1172,7 +1172,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 21491a9bc686e63bec32f1a67103622f1160a2f9 -R 67348fc4be2e8c129b7e6e89a490a40b +P 7268119f74602929b372b88eca7b3c3b4964f7d5 +R 38e0e000cd94731b650020d6321d9a1b U dan -Z e904ec03ef68eb7c18f9e542f40d2d02 +Z 73a15f3066e0e6dc1f73e5ba383c60f7 diff --git a/manifest.uuid b/manifest.uuid index 698af193d2..0656ed2305 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7268119f74602929b372b88eca7b3c3b4964f7d5 \ No newline at end of file +34f6b4b8e4cb8ea45fd518672dc96335d1ce06ab \ No newline at end of file diff --git a/test/trace2.test b/test/trace2.test index 8f68d87585..562c70c538 100644 --- a/test/trace2.test +++ b/test/trace2.test @@ -136,6 +136,7 @@ ifcapable fts3 { "-- SELECT (SELECT max(idx) FROM 'main'.'x1_segdir' WHERE level = ?) + 1" "-- SELECT coalesce((SELECT max(blockid) FROM 'main'.'x1_segments') + 1, 1)" "-- REPLACE INTO 'main'.'x1_segdir' VALUES(?,?,?,?,?,?)" + "-- SELECT level, idx, end_block FROM 'main'.'x1_segdir' WHERE level BETWEEN ? AND ? ORDER BY level DESC, idx ASC" } do_trace_test 2.3 { From 157c957a65887a07495397dc95ac27ac3686d77e Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 15 May 2014 18:36:39 +0000 Subject: [PATCH 65/71] Fix a problem with the handling of delete markers by automerge on large databases. FossilOrigin-Name: 2b09bd17eb85ea3c682e930d2aabc6441f2265e0 --- ext/fts3/fts3_write.c | 40 +++++++++++++++++++++++++++++++++++++++- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 46 insertions(+), 8 deletions(-) diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index ce16f0c86a..2f940eac7a 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -2433,6 +2433,37 @@ static int fts3SegmentMaxLevel( return sqlite3_reset(pStmt); } +/* +** iAbsLevel is an absolute level that may be assumed to exist within +** the database. This function checks if it is the largest level number +** within its index. Assuming no error occurs, *pbMax is set to 1 if +** iAbsLevel is indeed the largest level, or 0 otherwise, and SQLITE_OK +** is returned. If an error occurs, an error code is returned and the +** final value of *pbMax is undefined. +*/ +static int fts3SegmentIsMaxLevel(Fts3Table *p, i64 iAbsLevel, int *pbMax){ + + /* Set pStmt to the compiled version of: + ** + ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? + ** + ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). + */ + sqlite3_stmt *pStmt; + int rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_int64(pStmt, 1, iAbsLevel+1); + sqlite3_bind_int64(pStmt, 2, + ((iAbsLevel/FTS3_SEGDIR_MAXLEVEL)+1) * FTS3_SEGDIR_MAXLEVEL + ); + + *pbMax = 0; + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *pbMax = sqlite3_column_type(pStmt, 0)==SQLITE_NULL; + } + return sqlite3_reset(pStmt); +} + /* ** Delete all entries in the %_segments table associated with the segment ** opened with seg-reader pSeg. This function does not affect the contents @@ -4836,8 +4867,15 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ if( rc==SQLITE_OK ){ rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx); assert( bUseHint==1 || bUseHint==0 ); - if( (iIdx-bUseHint)==0 ) pFilter->flags |= FTS3_SEGMENT_IGNORE_EMPTY; + if( iIdx==0 || (bUseHint && iIdx==1) ){ + int bIgnore; + rc = fts3SegmentIsMaxLevel(p, iAbsLevel+1, &bIgnore); + if( bIgnore ){ + pFilter->flags |= FTS3_SEGMENT_IGNORE_EMPTY; + } + } } + if( rc==SQLITE_OK ){ rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr); } diff --git a/manifest b/manifest index 2b1578669e..8d7389d529 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\ssearch\sfor\spromotable\ssegments\sfollowing\san\sFTS\s'optimize'\soperation\sor\screation\sof\sa\snew\ssegment\son\sthe\soldest\sexisting\slevel. -D 2014-05-14T19:49:17.392 +C Fix\sa\sproblem\swith\sthe\shandling\sof\sdelete\smarkers\sby\sautomerge\son\slarge\sdatabases. +D 2014-05-15T18:36:39.233 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -96,7 +96,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_unicode.c 92391b4b4fb043564c6539ea9b8661e3bcba47b9 F ext/fts3/fts3_unicode2.c 0113d3acf13429e6dc38e0647d1bc71211c31a4d -F ext/fts3/fts3_write.c e58c8df149436dcc85e2face5516b75a01c7a583 +F ext/fts3/fts3_write.c b90173c62460a215498b0da3046f5e429b3a66b0 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197 @@ -1172,7 +1172,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7268119f74602929b372b88eca7b3c3b4964f7d5 -R 38e0e000cd94731b650020d6321d9a1b +P 34f6b4b8e4cb8ea45fd518672dc96335d1ce06ab +R e26158b3998e23a04d5e7968b1e6dfac U dan -Z 73a15f3066e0e6dc1f73e5ba383c60f7 +Z 27219917e93f1cfe1a1e1c19124576cc diff --git a/manifest.uuid b/manifest.uuid index 0656ed2305..8f898f62ee 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -34f6b4b8e4cb8ea45fd518672dc96335d1ce06ab \ No newline at end of file +2b09bd17eb85ea3c682e930d2aabc6441f2265e0 \ No newline at end of file From a7b9e7d2d9e06600d0dc7e4df62e9026ff7053d2 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 16 May 2014 10:30:44 +0000 Subject: [PATCH 66/71] Fix a bug causing an incorrect segment size value to be stored if both an old and new FTS version performed work on the same incremental merge operation. FossilOrigin-Name: a9a2aeab298ba2ac0b8835e61406e2d83bf7e39b --- ext/fts3/fts3_write.c | 6 ++-- manifest | 14 ++++---- manifest.uuid | 2 +- test/fts4growth.test | 79 +++++++++++++++++++++++++++++++++++-------- 4 files changed, 77 insertions(+), 24 deletions(-) diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 2f940eac7a..2cbf084e84 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -3636,6 +3636,7 @@ struct IncrmergeWriter { sqlite3_int64 iStart; /* Block number of first allocated block */ sqlite3_int64 iEnd; /* Block number of last allocated block */ sqlite3_int64 nLeafData; /* Bytes of leaf page data so far */ + u8 bNoLeafData; /* If true, store 0 for segment size */ NodeWriter aNodeWriter[FTS_MAX_APPENDABLE_HEIGHT]; }; @@ -4074,7 +4075,7 @@ static void fts3IncrmergeRelease( pWriter->iStart, /* start_block */ pWriter->aNodeWriter[0].iBlock, /* leaves_end_block */ pWriter->iEnd, /* end_block */ - pWriter->nLeafData, /* end_block */ + (pWriter->bNoLeafData==0 ? pWriter->nLeafData : 0), /* end_block */ pRoot->block.a, pRoot->block.n /* root */ ); } @@ -4180,6 +4181,7 @@ static int fts3IncrmergeLoad( if( pWriter->nLeafData<0 ){ pWriter->nLeafData = pWriter->nLeafData * -1; } + pWriter->bNoLeafData = (pWriter->nLeafData==0); nRoot = sqlite3_column_bytes(pSelect, 4); aRoot = sqlite3_column_blob(pSelect, 4); }else{ @@ -4914,7 +4916,7 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ pWriter->nLeafData = pWriter->nLeafData * -1; } fts3IncrmergeRelease(p, pWriter, &rc); - if( nSeg==0 ){ + if( nSeg==0 && pWriter->bNoLeafData==0 ){ fts3PromoteSegments(p, iAbsLevel+1, pWriter->nLeafData); } } diff --git a/manifest b/manifest index f27c4b1f83..f12f2aa4a8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\schanges\swith\sthis\sbranch. -D 2014-05-15T19:05:55.306 +C Fix\sa\sbug\scausing\san\sincorrect\ssegment\ssize\svalue\sto\sbe\sstored\sif\sboth\san\sold\sand\snew\sFTS\sversion\sperformed\swork\son\sthe\ssame\sincremental\smerge\soperation. +D 2014-05-16T10:30:44.076 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -96,7 +96,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_unicode.c 92391b4b4fb043564c6539ea9b8661e3bcba47b9 F ext/fts3/fts3_unicode2.c 0113d3acf13429e6dc38e0647d1bc71211c31a4d -F ext/fts3/fts3_write.c b90173c62460a215498b0da3046f5e429b3a66b0 +F ext/fts3/fts3_write.c 6681e4855bd24e74e91e2017f3fdc44460449fc1 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197 @@ -570,7 +570,7 @@ F test/fts4aa.test 0c3152322c7f0b548cc942ad763eaba0da87ccca F test/fts4check.test 66fa274cab2b615f2fb338b257713aba8fad88a8 F test/fts4content.test 2e7252557d6d24afa101d9ba1de710d6140e6d06 F test/fts4docid.test e33c383cfbdff0284685604d256f347a18fdbf01 -F test/fts4growth.test 50befaa001a6dd4046d32e5d7b3cb75df8360ca8 +F test/fts4growth.test df10fde9f47cf5c71861e63fd8efcd573c4f7e53 F test/fts4growth2.test 4ea35ea97c7d87e8f19d5ede939ab356f9f75cf0 F test/fts4incr.test 361960ed3550e781f3f313e17e2182ef9cefc0e9 F test/fts4langid.test 24a6e41063b416bbdf371ff6b4476fa41c194aa7 @@ -1172,7 +1172,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 2b09bd17eb85ea3c682e930d2aabc6441f2265e0 0901dccedaf5679691ba803ec76d113de170185d -R bc36878656a1df406ce5faa15b3690cd +P 5809986ffcc5d9cbab64760f8d7fbfbb6b626ee7 +R 12132d8d56290c779558e11739c0e6c3 U dan -Z 7b26c3764a28756f379665cd911542c0 +Z a463ca0b0e1281a342011aafc1248dc1 diff --git a/manifest.uuid b/manifest.uuid index 414ca82c34..0c30c7f481 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5809986ffcc5d9cbab64760f8d7fbfbb6b626ee7 \ No newline at end of file +a9a2aeab298ba2ac0b8835e61406e2d83bf7e39b \ No newline at end of file diff --git a/test/fts4growth.test b/test/fts4growth.test index 466416e561..aa5f251f95 100644 --- a/test/fts4growth.test +++ b/test/fts4growth.test @@ -366,21 +366,72 @@ do_execsql_test 6.5 { 0 1 {1329 8297} 0 0 {1320 28009} 0 2 {1449 118006} } -#do_test 3.2 { - #t1_to_x2 - #execsql {SELECT level, count(*) FROM x2_segdir GROUP BY level} -#} {0 13 1 15 2 5} +#-------------------------------------------------------------------------- +# Ensure that if part of an incremental merge is performed by an old +# version that does not support storing segment sizes in the end_block +# field, no size is stored in the final segment (as it would be incorrect). +# +do_execsql_test 7.1 { + CREATE VIRTUAL TABLE x6 USING fts4; + INSERT INTO x6 SELECT words FROM t1; + INSERT INTO x6 SELECT words FROM t1; + INSERT INTO x6 SELECT words FROM t1; + INSERT INTO x6 SELECT words FROM t1; + INSERT INTO x6 SELECT words FROM t1; + INSERT INTO x6 SELECT words FROM t1; + SELECT level, idx, end_block FROM x6_segdir; +} { + 0 0 {118 117483} 0 1 {238 118006} 0 2 {358 118006} + 0 3 {478 118006} 0 4 {598 118006} 0 5 {718 118006} +} + +do_execsql_test 7.2 { + INSERT INTO x6(x6) VALUES('merge=25,4'); + SELECT level, idx, end_block FROM x6_segdir; +} { + 0 0 {118 117483} 0 1 {238 118006} 0 2 {358 118006} + 0 3 {478 118006} 0 4 {598 118006} 0 5 {718 118006} + 1 0 {16014 -51226} +} + +do_execsql_test 7.3 { + UPDATE x6_segdir SET end_block = first(end_block) WHERE level=1; + SELECT level, idx, end_block FROM x6_segdir; +} { + 0 0 {118 117483} 0 1 {238 118006} 0 2 {358 118006} + 0 3 {478 118006} 0 4 {598 118006} 0 5 {718 118006} + 1 0 16014 +} + +do_execsql_test 7.4 { + INSERT INTO x6(x6) VALUES('merge=25,4'); + SELECT level, idx, end_block FROM x6_segdir; +} { + 0 0 {118 117483} 0 1 {238 118006} 0 2 {358 118006} + 0 3 {478 118006} 0 4 {598 118006} 0 5 {718 118006} + 1 0 16014 +} + +do_execsql_test 7.5 { + INSERT INTO x6(x6) VALUES('merge=2500,4'); + SELECT level, idx, end_block FROM x6_segdir; +} { + 0 0 {598 118006} 0 1 {718 118006} 1 0 16014 +} + +do_execsql_test 7.6 { + INSERT INTO x6(x6) VALUES('merge=2500,2'); + SELECT level, idx, start_block, leaves_end_block, end_block FROM x6_segdir; +} { + 2 0 23695 24147 {41262 633507} +} + +do_execsql_test 7.7 { + SELECT sum(length(block)) FROM x6_segments + WHERE blockid BETWEEN 23695 AND 24147 +} {633507} + -#proc second {x} { lindex $x 1 } -#db func second second -#for {set i 0} {$i <1000} {incr i} { -# t1_to_x2 -# db eval { -# SELECT level, group_concat( second(end_block), ' ' ) AS c FROM x2_segdir GROUP BY level; -# } { -# puts "$i.$level: $c" -# } -#} finish_test From c007f61bb02c2f20a3a240daeb0f2c88d68df85e Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 16 May 2014 14:17:01 +0000 Subject: [PATCH 67/71] Repurpose the SQLITE_TESTCTRL_FAULT_INSTALL test-control to register a callback to be invoked by sqlite3FaultSim(). That test-control has been unused since 2008-06-20 and was never used in any official release. FossilOrigin-Name: 0d43a7ad9abe821e33e0bf83a997aa4461b1e3f2 --- manifest | 24 +++++++------- manifest.uuid | 2 +- src/global.c | 13 ++++++-- src/main.c | 17 ++++++++++ src/sqliteInt.h | 13 ++++++-- src/test2.c | 88 +++++++++++++++++++++++++++++++++++++++++++++++-- src/util.c | 18 ++++++++++ test/misc1.test | 16 +++++++++ 8 files changed, 171 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index b51b7ce0c3..716155c3c3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sa\scomment\sin\sthe\sMSVC\smakefile.\s\sNo\schanges\sto\scode. -D 2014-05-12T22:36:50.007 +C Repurpose\sthe\sSQLITE_TESTCTRL_FAULT_INSTALL\stest-control\sto\sregister\sa\ncallback\sto\sbe\sinvoked\sby\ssqlite3FaultSim().\s\sThat\stest-control\shas\sbeen\nunused\ssince\s2008-06-20\sand\swas\snever\sused\sin\sany\sofficial\srelease. +D 2014-05-16T14:17:01.206 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -178,7 +178,7 @@ F src/expr.c 4f9e497c66e2f25a4d139357a778c84d5713207c F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 5269ef07b100763134f71b889327c333bd0989cf F src/func.c 2e16316ec3a6365a0dc3e553c586f91b20f7f6c8 -F src/global.c 1d7bb7ea8254ae6a68ed9bfaf65fcb3d1690b486 +F src/global.c 1e4bd956dc2f608f87d2a929abc4a20db65f30e4 F src/hash.c d139319967164f139c8d1bb8a11b14db9c4ba3cd F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 @@ -187,7 +187,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 867c7b330b740c6c917af9956b13b81d0a048303 -F src/main.c 0a8cfb6b2899649880e5874ac9bfd1c6f7c6a4a3 +F src/main.c bf60d71edc33a48a6e4f7494da64aaf15e6542fd F src/malloc.c 0203ebce9152c6a0e5de520140b8ba65187350be F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b @@ -225,13 +225,13 @@ F src/shell.c 2afe7a7154e97be0c74c5feacf09626bda8493be F src/sqlite.h.in 564fc23db33870b5096b20d72df7491ce0b8b74f F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h b2947801eccefd7ba3e5f14e1353289351a83cf3 +F src/sqliteInt.h 1977f44283be0d3b40da315fb4fd9e21c79fadcc F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c e87c99e28a145943666b51b212dacae35fcea0bd F src/test1.c 899bddeb0c7fb2b8062de6f03af4a4e4f48f9df5 -F src/test2.c 7355101c085304b90024f2261e056cdff13c6c35 +F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df F src/test5.c 5a34feec76d9b3a86aab30fd4f6cc9c48cbab4c1 @@ -279,7 +279,7 @@ F src/tokenize.c 6da2de6e12218ccb0aea5184b56727d011f4bee7 F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c -F src/util.c 2b5fb283a190aacdb286f7835a447c45b345b83c +F src/util.c 049fe1d3c0e2209c1bee107aec2fcff6285f909f F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 F src/vdbe.c 7f359193bf2366cc914a9ece093ebf284e56acdc F src/vdbe.h 394464909ed682334aa3d5831aae0c2fe2abef94 @@ -698,7 +698,7 @@ F test/minmax.test 42fbad0e81afaa6e0de41c960329f2b2c3526efd F test/minmax2.test b44bae787fc7b227597b01b0ca5575c7cb54d3bc F test/minmax3.test cc1e8b010136db0d01a6f2a29ba5a9f321034354 F test/minmax4.test 536a3360470633a177e42fbc19660d146b51daef -F test/misc1.test 441a0fafc7087f841db09fbfca54e7aea9f5a84c +F test/misc1.test 1201a037c24f982cc0e956cdaa34fcaf6439c417 F test/misc2.test 00d7de54eda90e237fc9a38b9e5ccc769ebf6d4d F test/misc3.test cf3dda47d5dda3e53fc5804a100d3c82be736c9d F test/misc4.test 9c078510fbfff05a9869a0b6d8b86a623ad2c4f6 @@ -1170,7 +1170,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 013738f3d531078fb47f9992c18a0d630e771a32 -R 50044157f65da46177a12dafd4a6afe2 -U mistachkin -Z 53f9d3afbf5bc14f8e3dfdbc370843ae +P 0901dccedaf5679691ba803ec76d113de170185d +R 67689d62fcdc72fe4681824f9ba5e0cd +U drh +Z 053659c56a956daeb64b7fcb6d47db2b diff --git a/manifest.uuid b/manifest.uuid index 7af790eea2..e1375063e0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0901dccedaf5679691ba803ec76d113de170185d \ No newline at end of file +0d43a7ad9abe821e33e0bf83a997aa4461b1e3f2 \ No newline at end of file diff --git a/src/global.c b/src/global.c index 1ee3f6436f..2c14b58abd 100644 --- a/src/global.c +++ b/src/global.c @@ -173,15 +173,22 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { 0, /* isMutexInit */ 0, /* isMallocInit */ 0, /* isPCacheInit */ - 0, /* pInitMutex */ 0, /* nRefInitMutex */ + 0, /* pInitMutex */ 0, /* xLog */ 0, /* pLogArg */ - 0, /* bLocaltimeFault */ #ifdef SQLITE_ENABLE_SQLLOG 0, /* xSqllog */ - 0 /* pSqllogArg */ + 0, /* pSqllogArg */ #endif +#ifdef SQLITE_VDBE_COVERAGE + 0, /* xVdbeBranch */ + 0, /* pVbeBranchArg */ +#endif +#ifndef SQLITE_OMIT_BUILTIN_TEST + 0, /* xTestCallback */ +#endif + 0 /* bLocaltimeFault */ }; /* diff --git a/src/main.c b/src/main.c index 65521f4466..b3bc66e849 100644 --- a/src/main.c +++ b/src/main.c @@ -3114,6 +3114,23 @@ int sqlite3_test_control(int op, ...){ break; } + /* + ** sqlite3_test_control(FAULT_INSTALL, xCallback) + ** + ** Arrange to invoke xCallback() whenever sqlite3FaultSim() is called, + ** if xCallback is not NULL. + ** + ** As a test of the fault simulator mechanism itself, sqlite3FaultSim(0) + ** is called immediately after installing the new callback and the return + ** value from sqlite3FaultSim(0) becomes the return from + ** sqlite3_test_control(). + */ + case SQLITE_TESTCTRL_FAULT_INSTALL: { + sqlite3Config.xTestCallback = va_arg(ap, int(*)(int)); + rc = sqlite3FaultSim(0); + break; + } + /* ** sqlite3_test_control(BENIGN_MALLOC_HOOKS, xBegin, xEnd) ** diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 4bd65a690f..bb8763ccbb 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2709,11 +2709,10 @@ struct Sqlite3Config { int isMutexInit; /* True after mutexes are initialized */ int isMallocInit; /* True after malloc is initialized */ int isPCacheInit; /* True after malloc is initialized */ - sqlite3_mutex *pInitMutex; /* Mutex used by sqlite3_initialize() */ int nRefInitMutex; /* Number of users of pInitMutex */ + sqlite3_mutex *pInitMutex; /* Mutex used by sqlite3_initialize() */ void (*xLog)(void*,int,const char*); /* Function for logging */ void *pLogArg; /* First argument to xLog() */ - int bLocaltimeFault; /* True to fail localtime() calls */ #ifdef SQLITE_ENABLE_SQLLOG void(*xSqllog)(void*,sqlite3*,const char*, int); void *pSqllogArg; @@ -2725,6 +2724,10 @@ struct Sqlite3Config { void (*xVdbeBranch)(void*,int iSrcLine,u8 eThis,u8 eMx); /* Callback */ void *pVdbeBranchArg; /* 1st argument */ #endif +#ifndef SQLITE_OMIT_BUILTIN_TEST + int (*xTestCallback)(int); /* Invoked by sqlite3FaultSim() */ +#endif + int bLocaltimeFault; /* True to fail localtime() calls */ }; /* @@ -3026,6 +3029,12 @@ int sqlite3ParseUri(const char*,const char*,unsigned int*, Btree *sqlite3DbNameToBtree(sqlite3*,const char*); int sqlite3CodeOnce(Parse *); +#ifdef SQLITE_OMIT_BUILTIN_TEST +# define sqlite3FaultSim(X) SQLITE_OK +#else + int sqlite3FaultSim(int); +#endif + Bitvec *sqlite3BitvecCreate(u32); int sqlite3BitvecTest(Bitvec*, u32); int sqlite3BitvecSet(Bitvec*, u32); diff --git a/src/test2.c b/src/test2.c index d130e9d01b..58f271ff27 100644 --- a/src/test2.c +++ b/src/test2.c @@ -568,7 +568,90 @@ static int testPendingByte( rc = sqlite3_test_control(SQLITE_TESTCTRL_PENDING_BYTE, pbyte); Tcl_SetObjResult(interp, Tcl_NewIntObj(rc)); return TCL_OK; -} +} + +/* +** The sqlite3FaultSim() callback: +*/ +static Tcl_Interp *faultSimInterp = 0; +static int faultSimScriptSize = 0; +static char *faultSimScript; +static int faultSimCallback(int x){ + char zInt[30]; + int i; + int isNeg; + int rc; + if( x==0 ){ + memcpy(faultSimScript+faultSimScriptSize, "0", 2); + }else{ + /* Convert x to text without using any sqlite3 routines */ + if( x<0 ){ + isNeg = 1; + x = -x; + }else{ + isNeg = 0; + } + zInt[sizeof(zInt)-1] = 0; + for(i=sizeof(zInt)-2; i>0 && x>0; i--, x /= 10){ + zInt[i] = (x%10) + '0'; + } + if( isNeg ) zInt[i--] = '-'; + memcpy(faultSimScript+faultSimScriptSize, zInt+i+1, sizeof(zInt)-i); + } + rc = Tcl_Eval(faultSimInterp, faultSimScript); + if( rc ){ + fprintf(stderr, "fault simulator script failed: [%s]", faultSimScript); + rc = SQLITE_ERROR; + }else{ + rc = atoi(Tcl_GetStringResult(faultSimInterp)); + } + Tcl_ResetResult(faultSimInterp); + return rc; +} + +/* +** sqlite3_test_control_fault_install SCRIPT +** +** Arrange to invoke SCRIPT with the integer argument to sqlite3FaultSim() +** appended, whenever sqlite3FaultSim() is called. Or, if SCRIPT is the +** empty string, cancel the sqlite3FaultSim() callback. +*/ +static int faultInstallCmd( + void *NotUsed, + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + const char **argv /* Text of each argument */ +){ + const char *zScript; + int nScript; + int rc; + if( argc!=1 && argc!=2 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], + " SCRIPT\"", (void*)0); + } + zScript = argc==2 ? argv[1] : ""; + nScript = (int)strlen(zScript); + if( faultSimScript ){ + free(faultSimScript); + faultSimScript = 0; + } + if( nScript==0 ){ + rc = sqlite3_test_control(SQLITE_TESTCTRL_FAULT_INSTALL, 0); + }else{ + faultSimScript = malloc( nScript+100 ); + if( faultSimScript==0 ){ + Tcl_AppendResult(interp, "out of memory", (void*)0); + return SQLITE_ERROR; + } + memcpy(faultSimScript, zScript, nScript); + faultSimScript[nScript] = ' '; + faultSimScriptSize = nScript+1; + faultSimInterp = interp; + rc = sqlite3_test_control(SQLITE_TESTCTRL_FAULT_INSTALL, faultSimCallback); + } + Tcl_SetObjResult(interp, Tcl_NewIntObj(rc)); + return SQLITE_OK; +} /* ** sqlite3BitvecBuiltinTest SIZE PROGRAM @@ -638,7 +721,8 @@ int Sqlitetest2_Init(Tcl_Interp *interp){ { "fake_big_file", (Tcl_CmdProc*)fake_big_file }, #endif { "sqlite3BitvecBuiltinTest",(Tcl_CmdProc*)testBitvecBuiltinTest }, - { "sqlite3_test_control_pending_byte", (Tcl_CmdProc*)testPendingByte }, + { "sqlite3_test_control_pending_byte", (Tcl_CmdProc*)testPendingByte }, + { "sqlite3_test_control_fault_install", (Tcl_CmdProc*)faultInstallCmd }, }; int i; for(i=0; i Date: Fri, 16 May 2014 15:48:17 +0000 Subject: [PATCH 68/71] Add extra test to backcompat.test to ensure that old and new versions of FTS may work together on the same incremental merge operation. FossilOrigin-Name: 3997d47bb9c8593a2cd30317ea65163b99db1832 --- manifest | 12 +++++----- manifest.uuid | 2 +- test/backcompat.test | 54 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index f12f2aa4a8..543d66fc2f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbug\scausing\san\sincorrect\ssegment\ssize\svalue\sto\sbe\sstored\sif\sboth\san\sold\sand\snew\sFTS\sversion\sperformed\swork\son\sthe\ssame\sincremental\smerge\soperation. -D 2014-05-16T10:30:44.076 +C Add\sextra\stest\sto\sbackcompat.test\sto\sensure\sthat\sold\sand\snew\sversions\sof\sFTS\smay\swork\stogether\son\sthe\ssame\sincremental\smerge\soperation. +D 2014-05-16T15:48:17.688 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -336,7 +336,7 @@ F test/autoindex1.test 762ff3f8e25d852aae55c6462ca166a80c0cde61 F test/autovacuum.test 941892505d2c0f410a0cb5970dfa1c7c4e5f6e74 F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4 F test/avtrans.test 0252654f4295ddda3b2cce0e894812259e655a85 -F test/backcompat.test 5f8ad58b3eaebc78cd2c66c65476a42e6f32b2ad +F test/backcompat.test 19a1f337c68419b020a7481dd272a472c4ad8ef4 F test/backup.test c9cdd23a495864b9edf75a9fa66f5cb7e10fcf62 F test/backup2.test 34986ef926ea522911a51dfdb2f8e99b7b75ebcf F test/backup4.test 2a2e4a64388090b152de753fd9e123f28f6a3bd4 @@ -1172,7 +1172,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 5809986ffcc5d9cbab64760f8d7fbfbb6b626ee7 -R 12132d8d56290c779558e11739c0e6c3 +P a9a2aeab298ba2ac0b8835e61406e2d83bf7e39b +R 03d2cea8f8e26027c7bf9b05231d6fb8 U dan -Z a463ca0b0e1281a342011aafc1248dc1 +Z ba763e498b0d066dca73ae5326487971 diff --git a/manifest.uuid b/manifest.uuid index 0c30c7f481..1f33001479 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a9a2aeab298ba2ac0b8835e61406e2d83bf7e39b \ No newline at end of file +3997d47bb9c8593a2cd30317ea65163b99db1832 \ No newline at end of file diff --git a/test/backcompat.test b/test/backcompat.test index fdc2410b7a..ea7e6a9eed 100644 --- a/test/backcompat.test +++ b/test/backcompat.test @@ -58,12 +58,24 @@ proc do_backcompat_test {rv bin1 bin2 script} { code1 { sqlite3 db test.db } code2 { sqlite3 db test.db } + foreach c {code1 code2} { + $c { + set v [split [db version] .] + if {[llength $v]==3} {lappend v 0} + set ::sqlite_libversion [format \ + "%d%.2d%.2d%2d" [lindex $v 0] [lindex $v 1] [lindex $v 2] [lindex $v 3] + ] + } + } + uplevel $script catch { code1 { db close } } catch { code2 { db close } } catch { close $::bc_chan2 } catch { close $::bc_chan1 } + + } array set ::incompatible [list] @@ -381,6 +393,48 @@ ifcapable fts3 { } { do_test backcompat-3.7 [list sql1 $q] [sql2 $q] } + + # Now test that an incremental merge can be started by one version + # and finished by another. And that the integrity-check still + # passes. + do_test backcompat-3.8 { + sql1 { + DROP TABLE IF EXISTS t1; + DROP TABLE IF EXISTS t2; + CREATE TABLE t1(docid, words); + CREATE VIRTUAL TABLE t2 USING fts3(words); + } + code1 [list source $testdir/genesis.tcl] + code1 { fts_kjv_genesis } + sql1 { + INSERT INTO t2 SELECT words FROM t1; + INSERT INTO t2 SELECT words FROM t1; + INSERT INTO t2 SELECT words FROM t1; + INSERT INTO t2 SELECT words FROM t1; + INSERT INTO t2 SELECT words FROM t1; + INSERT INTO t2 SELECT words FROM t1; + SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level; + } + } {0 {0 1 2 3 4 5}} + + if {[code1 { set ::sqlite_libversion }] >=3071200 + && [code2 { set ::sqlite_libversion }] >=3071200 + } { + do_test backcompat-3.9 { + sql1 { INSERT INTO t2(t2) VALUES('merge=100,4'); } + sql2 { INSERT INTO t2(t2) VALUES('merge=100,4'); } + sql1 { INSERT INTO t2(t2) VALUES('merge=100,4'); } + sql2 { INSERT INTO t2(t2) VALUES('merge=2500,4'); } + sql2 { + SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level; + } + } {0 {0 1} 1 0} + + do_test backcompat-3.10 { + sql1 { INSERT INTO t2(t2) VALUES('integrity-check') } + sql2 { INSERT INTO t2(t2) VALUES('integrity-check') } + } {} + } } } } From 77a90ce811107805c414f381285d5a818e5e34ea Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 16 May 2014 23:15:50 +0000 Subject: [PATCH 69/71] Work around compilation issue with MSVC. FossilOrigin-Name: 9623a29c11e3a90513016ea3306c085d2bf91b62 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/main.c | 7 ++++++- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 91c09ac09c..361e70cb17 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fixes\sto\sprevent\san\sFTS\sindex\sfrom\sgrowing\sindefinitely\sas\sthe\scorresponding\stable\sis\supdated.\sChange\sthe\sFTS\s'automerge'\soption\sto\sallow\sthe\suser\sto\sspecify\sthe\snumber\sof\ssegments\sthat\sshould\sbe\smerged\ssimultaneously\sby\sauto-merges. -D 2014-05-16T16:16:59.006 +C Work\saround\scompilation\sissue\swith\sMSVC. +D 2014-05-16T23:15:50.277 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -187,7 +187,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 867c7b330b740c6c917af9956b13b81d0a048303 -F src/main.c bf60d71edc33a48a6e4f7494da64aaf15e6542fd +F src/main.c 1c164c7b742f0cd60290b34dc9086df9f939f490 F src/malloc.c 0203ebce9152c6a0e5de520140b8ba65187350be F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b @@ -1172,7 +1172,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 0d43a7ad9abe821e33e0bf83a997aa4461b1e3f2 3997d47bb9c8593a2cd30317ea65163b99db1832 -R 01618b664df3c3b19706db5d70667d8a -U dan -Z fa66c3fc7c4b02244be8e29ed55dc1c5 +P a75f1800021d30b712053373440041b7b355a36a +R e44c4f2d8b2a74316856f263d26bcc16 +U mistachkin +Z 7ea453b006be693b9823cb9b81301836 diff --git a/manifest.uuid b/manifest.uuid index d4cefb8a54..92cf7444c6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a75f1800021d30b712053373440041b7b355a36a \ No newline at end of file +9623a29c11e3a90513016ea3306c085d2bf91b62 \ No newline at end of file diff --git a/src/main.c b/src/main.c index b3bc66e849..d150d19d3c 100644 --- a/src/main.c +++ b/src/main.c @@ -3126,7 +3126,12 @@ int sqlite3_test_control(int op, ...){ ** sqlite3_test_control(). */ case SQLITE_TESTCTRL_FAULT_INSTALL: { - sqlite3Config.xTestCallback = va_arg(ap, int(*)(int)); + /* MSVC is picky about pulling func ptrs from va lists. + ** http://support.microsoft.com/kb/47961 + ** sqlite3Config.xTestCallback = va_arg(ap, int(*)(int)); + */ + typedef int(*TESTCALLBACKFUNC_t)(int); + sqlite3Config.xTestCallback = va_arg(ap, TESTCALLBACKFUNC_t); rc = sqlite3FaultSim(0); break; } From 02c1e264a9fa144181b3898e010ee1050b6658b9 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 16 May 2014 23:38:59 +0000 Subject: [PATCH 70/71] Fix harmless compiler warnings. FossilOrigin-Name: 7e60347e21a7473974061965a8d1f2eadde987e2 --- ext/fts3/fts3_write.c | 12 ++++++------ manifest | 15 +++++++++------ manifest.uuid | 2 +- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 2cbf084e84..ea3c08802a 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -3044,7 +3044,7 @@ static void fts3ReadEndBlockField( */ static int fts3PromoteSegments( Fts3Table *p, /* FTS table handle */ - int iAbsLevel, /* Absolute level just updated */ + sqlite3_int64 iAbsLevel, /* Absolute level just updated */ sqlite3_int64 nByte /* Size of new segment at iAbsLevel */ ){ int rc = SQLITE_OK; @@ -3054,7 +3054,7 @@ static int fts3PromoteSegments( if( rc==SQLITE_OK ){ int bOk = 0; - int iLast = (iAbsLevel/FTS3_SEGDIR_MAXLEVEL + 1) * FTS3_SEGDIR_MAXLEVEL - 1; + i64 iLast = (iAbsLevel/FTS3_SEGDIR_MAXLEVEL + 1) * FTS3_SEGDIR_MAXLEVEL - 1; i64 nLimit = (nByte*3)/2; /* Loop through all entries in the %_segdir table corresponding to @@ -3062,8 +3062,8 @@ static int fts3PromoteSegments( ** at least one such segment, and it is possible to determine that all ** such segments are smaller than nLimit bytes in size, they will be ** promoted to level iAbsLevel. */ - sqlite3_bind_int(pRange, 1, iAbsLevel+1); - sqlite3_bind_int(pRange, 2, iLast); + sqlite3_bind_int64(pRange, 1, iAbsLevel+1); + sqlite3_bind_int64(pRange, 2, iLast); while( SQLITE_ROW==sqlite3_step(pRange) ){ i64 nSize, dummy; fts3ReadEndBlockField(pRange, 2, &dummy, &nSize); @@ -3103,7 +3103,7 @@ static int fts3PromoteSegments( ** setting the "idx" fields as appropriate to keep them in the same ** order. The contents of level -1 (which is never used, except ** transiently here), will be moved back to level iAbsLevel below. */ - sqlite3_bind_int(pRange, 1, iAbsLevel); + sqlite3_bind_int64(pRange, 1, iAbsLevel); while( SQLITE_ROW==sqlite3_step(pRange) ){ sqlite3_bind_int(pUpdate1, 1, iIdx++); sqlite3_bind_int(pUpdate1, 2, sqlite3_column_int(pRange, 0)); @@ -3122,7 +3122,7 @@ static int fts3PromoteSegments( /* Move level -1 to level iAbsLevel */ if( rc==SQLITE_OK ){ - sqlite3_bind_int(pUpdate2, 1, iAbsLevel); + sqlite3_bind_int64(pUpdate2, 1, iAbsLevel); sqlite3_step(pUpdate2); rc = sqlite3_reset(pUpdate2); } diff --git a/manifest b/manifest index 361e70cb17..0cdba0d21b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Work\saround\scompilation\sissue\swith\sMSVC. -D 2014-05-16T23:15:50.277 +C Fix\sharmless\scompiler\swarnings. +D 2014-05-16T23:38:59.727 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -96,7 +96,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_unicode.c 92391b4b4fb043564c6539ea9b8661e3bcba47b9 F ext/fts3/fts3_unicode2.c 0113d3acf13429e6dc38e0647d1bc71211c31a4d -F ext/fts3/fts3_write.c 6681e4855bd24e74e91e2017f3fdc44460449fc1 +F ext/fts3/fts3_write.c 9e4e8579ad26cff3ee7743aaf5c3fe937fc441b4 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197 @@ -1172,7 +1172,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a75f1800021d30b712053373440041b7b355a36a -R e44c4f2d8b2a74316856f263d26bcc16 +P 9623a29c11e3a90513016ea3306c085d2bf91b62 +R 5ded43b85153ff3080cc0993c5b9bd75 +T *branch * ftsWarnings +T *sym-ftsWarnings * +T -sym-trunk * U mistachkin -Z 7ea453b006be693b9823cb9b81301836 +Z 3f5cc6ff89fc3bbade4957ecc9f7a199 diff --git a/manifest.uuid b/manifest.uuid index 92cf7444c6..7ca9a7ea7b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9623a29c11e3a90513016ea3306c085d2bf91b62 \ No newline at end of file +7e60347e21a7473974061965a8d1f2eadde987e2 \ No newline at end of file From 83e05b814eb8a625fdb6ee6d3b630408d3dd19d5 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 17 May 2014 15:10:37 +0000 Subject: [PATCH 71/71] Fix a broken test case in fts4growth2.test. FossilOrigin-Name: 9fde5b960af39b187474e5b2909ff0e56d5e0bf9 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/fts4growth2.test | 11 ++++++++--- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 361e70cb17..48cfe8e5aa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Work\saround\scompilation\sissue\swith\sMSVC. -D 2014-05-16T23:15:50.277 +C Fix\sa\sbroken\stest\scase\sin\sfts4growth2.test. +D 2014-05-17T15:10:37.982 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -571,7 +571,7 @@ F test/fts4check.test 66fa274cab2b615f2fb338b257713aba8fad88a8 F test/fts4content.test 2e7252557d6d24afa101d9ba1de710d6140e6d06 F test/fts4docid.test e33c383cfbdff0284685604d256f347a18fdbf01 F test/fts4growth.test df10fde9f47cf5c71861e63fd8efcd573c4f7e53 -F test/fts4growth2.test 4ea35ea97c7d87e8f19d5ede939ab356f9f75cf0 +F test/fts4growth2.test 2f063be1902a73cd087355837c52fed42ac11a5d F test/fts4incr.test 361960ed3550e781f3f313e17e2182ef9cefc0e9 F test/fts4langid.test 24a6e41063b416bbdf371ff6b4476fa41c194aa7 F test/fts4merge.test c424309743fdd203f8e56a1f1cd7872cd66cc0ee @@ -1172,7 +1172,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a75f1800021d30b712053373440041b7b355a36a -R e44c4f2d8b2a74316856f263d26bcc16 -U mistachkin -Z 7ea453b006be693b9823cb9b81301836 +P 9623a29c11e3a90513016ea3306c085d2bf91b62 +R 7fbf0d1c98f8f088bba70604a90180b5 +U dan +Z 9d3390acf23fdcf50586fd0f9fba7a93 diff --git a/manifest.uuid b/manifest.uuid index 92cf7444c6..99244e27d5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9623a29c11e3a90513016ea3306c085d2bf91b62 \ No newline at end of file +9fde5b960af39b187474e5b2909ff0e56d5e0bf9 \ No newline at end of file diff --git a/test/fts4growth2.test b/test/fts4growth2.test index 023ca8ba1c..af41d51c18 100644 --- a/test/fts4growth2.test +++ b/test/fts4growth2.test @@ -28,14 +28,19 @@ source $testdir/genesis.tcl do_execsql_test 1.0 { CREATE TABLE t1(docid, words); } fts_kjv_genesis +proc structure {} { + puts [ db eval {SELECT level, count(*) FROM x1_segdir GROUP BY level} ] +} + proc tt {val} { execsql { DELETE FROM x1 WHERE docid IN (SELECT docid FROM t1 WHERE (rowid-1)%4==$val+0); + } + execsql { INSERT INTO x1(docid, content) SELECT docid, words FROM t1 WHERE (rowid%4)==$val+0; } - #puts [db eval {SELECT level, idx, end_block FROM x1_segdir}] } do_execsql_test 1.1 { @@ -49,9 +54,9 @@ do_test 1.2 { } execsql { SELECT max(level) FROM x1_segdir; - SELECT count(*) FROM x1_segdir WHERE level=3; + SELECT count(*) FROM x1_segdir WHERE level=2; } -} {3 1} +} {2 1} do_test 1.3 { for {set i 0} {$i < 40} {incr i} {